diff --git a/.backportrc.json b/.backportrc.json index 77b06cd419275..db4418e180190 100644 --- a/.backportrc.json +++ b/.backportrc.json @@ -1,9 +1,10 @@ { "upstream" : "elastic/elasticsearch", - "targetBranchChoices" : [ "main", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ], + "targetBranchChoices" : [ "main", "8.x", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ], "targetPRLabels" : [ "backport" ], "branchLabelMapping" : { - "^v8.16.0$" : "main", + "^v9.0.0$" : "main", + "^v8.18.0$" : "8.x", "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2" } } \ No newline at end of file diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index b6b730fc3de8b..f25092bc6d42f 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -16,10 +16,10 @@ export COMPOSE_HTTP_TIMEOUT JOB_BRANCH="$BUILDKITE_BRANCH" export JOB_BRANCH -GRADLEW="./gradlew --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/" +GRADLEW="./gradlew --console=plain --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/" export GRADLEW -GRADLEW_BAT="./gradlew.bat --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/" +GRADLEW_BAT="./gradlew.bat --console=plain --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/" export GRADLEW_BAT export $(cat .ci/java-versions.properties | grep '=' | xargs) @@ -47,6 +47,8 @@ export GRADLE_BUILD_CACHE_PASSWORD BUILDKITE_API_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/buildkite-api-token) export BUILDKITE_API_TOKEN +export GH_TOKEN="$VAULT_GITHUB_TOKEN" + if [[ "${USE_LUCENE_SNAPSHOT_CREDS:-}" == "true" ]]; then data=$(.buildkite/scripts/get-legacy-secret.sh aws-elastic/creds/lucene-snapshots) @@ -78,11 +80,15 @@ if [[ "${USE_SNYK_CREDENTIALS:-}" == "true" ]]; then fi if [[ "${USE_PROD_DOCKER_CREDENTIALS:-}" == "true" ]]; then - DOCKER_REGISTRY_USERNAME="$(vault read -field=username secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)" - export DOCKER_REGISTRY_USERNAME + if which docker > /dev/null 2>&1; then + DOCKER_REGISTRY_USERNAME="$(vault read -field=username secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)" + export DOCKER_REGISTRY_USERNAME + + DOCKER_REGISTRY_PASSWORD="$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)" + export DOCKER_REGISTRY_PASSWORD - DOCKER_REGISTRY_PASSWORD="$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)" - export DOCKER_REGISTRY_PASSWORD + docker login --username "$DOCKER_REGISTRY_USERNAME" --password "$DOCKER_REGISTRY_PASSWORD" docker.elastic.co + fi fi if [[ "$BUILDKITE_AGENT_META_DATA_PROVIDER" != *"k8s"* ]]; then @@ -105,3 +111,13 @@ EOF Agent information from gobld EOF fi + +# Amazon Linux 2 has DNS resolution issues with resource-based hostnames in EC2 +# We have many functional tests that try to lookup and resolve the hostname of the local machine in a particular way +# And they fail. This sets up a manual entry for the hostname in dnsmasq. +if [[ -f /etc/os-release ]] && grep -q '"Amazon Linux 2"' /etc/os-release; then + echo "$(hostname -i | cut -d' ' -f 2) $(hostname -f)." | sudo tee /etc/dnsmasq.hosts + sudo systemctl restart dnsmasq.service +fi + +.buildkite/scripts/get-latest-test-mutes.sh diff --git a/.buildkite/hooks/pre-command.bat b/.buildkite/hooks/pre-command.bat index fe7c2371de0e5..752c2bf23eb14 100644 --- a/.buildkite/hooks/pre-command.bat +++ b/.buildkite/hooks/pre-command.bat @@ -15,9 +15,12 @@ set BUILD_NUMBER=%BUILDKITE_BUILD_NUMBER% set COMPOSE_HTTP_TIMEOUT=120 set JOB_BRANCH=%BUILDKITE_BRANCH% +set GH_TOKEN=%VAULT_GITHUB_TOKEN% + set GRADLE_BUILD_CACHE_USERNAME=vault read -field=username secret/ci/elastic-elasticsearch/migrated/gradle-build-cache set GRADLE_BUILD_CACHE_PASSWORD=vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache bash.exe -c "nohup bash .buildkite/scripts/setup-monitoring.sh /dev/null 2>&1 &" +bash.exe -c "bash .buildkite/scripts/get-latest-test-mutes.sh" exit /b 0 diff --git a/.buildkite/pipelines/dra-workflow.yml b/.buildkite/pipelines/dra-workflow.yml index bcc6c9c57d756..25477c8541fa9 100644 --- a/.buildkite/pipelines/dra-workflow.yml +++ b/.buildkite/pipelines/dra-workflow.yml @@ -6,7 +6,8 @@ steps: provider: gcp image: family/elasticsearch-ubuntu-2204 machineType: custom-32-98304 - buildDirectory: /dev/shm/bk + localSsds: 1 + localSsdInterface: nvme diskSizeGb: 350 - wait # The hadoop build depends on the ES artifact diff --git a/.buildkite/pipelines/intake.template.yml b/.buildkite/pipelines/intake.template.yml index 1a513971b2c10..78752ab597ed6 100644 --- a/.buildkite/pipelines/intake.template.yml +++ b/.buildkite/pipelines/intake.template.yml @@ -1,63 +1,57 @@ steps: - label: sanity-check - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files precommit + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints precommit timeout_in_minutes: 300 agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - wait - label: part1 - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart1 + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart1 timeout_in_minutes: 300 agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: part2 - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2 + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart2 timeout_in_minutes: 300 agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: part3 - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart3 + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart3 timeout_in_minutes: 300 agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: part4 - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4 + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart4 timeout_in_minutes: 300 agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: part5 - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart5 + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart5 timeout_in_minutes: 300 agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - group: bwc-snapshots steps: - label: "{{matrix.BWC_VERSION}} / bwc-snapshots" - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files v$$BWC_VERSION#bwcTest + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints v$$BWC_VERSION#bwcTest timeout_in_minutes: 300 matrix: setup: @@ -67,18 +61,16 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: "{{matrix.BWC_VERSION}}" - label: rest-compat - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkRestCompat + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkRestCompat timeout_in_minutes: 300 agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - wait - trigger: elasticsearch-dra-workflow label: Trigger DRA snapshot workflow diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index f698f722d977e..f2fa3565ff85c 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -1,85 +1,77 @@ # This file is auto-generated. See .buildkite/pipelines/intake.template.yml steps: - label: sanity-check - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files precommit + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints precommit timeout_in_minutes: 300 agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - wait - label: part1 - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart1 + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart1 timeout_in_minutes: 300 agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: part2 - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2 + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart2 timeout_in_minutes: 300 agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: part3 - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart3 + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart3 timeout_in_minutes: 300 agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: part4 - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4 + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart4 timeout_in_minutes: 300 agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: part5 - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart5 + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart5 timeout_in_minutes: 300 agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - group: bwc-snapshots steps: - label: "{{matrix.BWC_VERSION}} / bwc-snapshots" - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files v$$BWC_VERSION#bwcTest + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints v$$BWC_VERSION#bwcTest timeout_in_minutes: 300 matrix: setup: - BWC_VERSION: ["7.17.25", "8.15.2", "8.16.0"] + BWC_VERSION: ["7.17.27", "8.16.3", "8.17.1", "8.18.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: "{{matrix.BWC_VERSION}}" - label: rest-compat - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkRestCompat + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkRestCompat timeout_in_minutes: 300 agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - wait - trigger: elasticsearch-dra-workflow label: Trigger DRA snapshot workflow diff --git a/.buildkite/pipelines/lucene-snapshot/build-snapshot.yml b/.buildkite/pipelines/lucene-snapshot/build-snapshot.yml index 1f69b8faa7ab4..8cf2a8aacbece 100644 --- a/.buildkite/pipelines/lucene-snapshot/build-snapshot.yml +++ b/.buildkite/pipelines/lucene-snapshot/build-snapshot.yml @@ -15,7 +15,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - wait - trigger: "elasticsearch-lucene-snapshot-tests" build: diff --git a/.buildkite/pipelines/lucene-snapshot/run-tests.yml b/.buildkite/pipelines/lucene-snapshot/run-tests.yml index 49c3396488d82..d25fa7b2067c1 100644 --- a/.buildkite/pipelines/lucene-snapshot/run-tests.yml +++ b/.buildkite/pipelines/lucene-snapshot/run-tests.yml @@ -1,63 +1,57 @@ steps: - label: sanity-check - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files precommit + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints precommit timeout_in_minutes: 300 agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - wait: null - label: part1 - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart1 + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart1 timeout_in_minutes: 300 agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: part2 - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2 + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart2 timeout_in_minutes: 300 agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: part3 - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart3 + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart3 timeout_in_minutes: 300 agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: part4 - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4 + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart4 timeout_in_minutes: 300 agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: part5 - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart5 + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart5 timeout_in_minutes: 300 agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - group: bwc-snapshots steps: - label: "{{matrix.BWC_VERSION}} / bwc-snapshots" - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files v$$BWC_VERSION#bwcTest + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints v$$BWC_VERSION#bwcTest timeout_in_minutes: 300 matrix: setup: @@ -70,15 +64,13 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: "{{matrix.BWC_VERSION}}" - label: rest-compat - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkRestCompat + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkRestCompat timeout_in_minutes: 300 agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/periodic-packaging.bwc.template.yml b/.buildkite/pipelines/periodic-packaging.bwc.template.yml index 8a6fa2553b204..b06bc80d3535d 100644 --- a/.buildkite/pipelines/periodic-packaging.bwc.template.yml +++ b/.buildkite/pipelines/periodic-packaging.bwc.template.yml @@ -11,6 +11,5 @@ image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: $BWC_VERSION diff --git a/.buildkite/pipelines/periodic-packaging.template.yml b/.buildkite/pipelines/periodic-packaging.template.yml index 64c5fa5060e6c..38e4a71ccea93 100644 --- a/.buildkite/pipelines/periodic-packaging.template.yml +++ b/.buildkite/pipelines/periodic-packaging.template.yml @@ -3,27 +3,29 @@ steps: steps: - label: "{{matrix.image}} / packaging-tests-unix" command: ./.ci/scripts/packaging-test.sh destructivePackagingTest - timeout_in_minutes: 300 + timeout_in_minutes: 420 matrix: setup: image: - - centos-7 - - debian-10 - debian-11 + - debian-12 - opensuse-leap-15 - oraclelinux-7 - oraclelinux-8 + - oraclelinux-9 - sles-12 - sles-15 - ubuntu-1804 - ubuntu-2004 - ubuntu-2204 + - ubuntu-2404 - rocky-8 - rocky-9 - rhel-7 - rhel-8 - rhel-9 - almalinux-8 + - almalinux-9 agents: provider: gcp image: family/elasticsearch-{{matrix.image}} diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 3c98dd4b30e74..1242e1890ab25 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -4,27 +4,29 @@ steps: steps: - label: "{{matrix.image}} / packaging-tests-unix" command: ./.ci/scripts/packaging-test.sh destructivePackagingTest - timeout_in_minutes: 300 + timeout_in_minutes: 420 matrix: setup: image: - - centos-7 - - debian-10 - debian-11 + - debian-12 - opensuse-leap-15 - oraclelinux-7 - oraclelinux-8 + - oraclelinux-9 - sles-12 - sles-15 - ubuntu-1804 - ubuntu-2004 - ubuntu-2204 + - ubuntu-2404 - rocky-8 - rocky-9 - rhel-7 - rhel-8 - rhel-9 - almalinux-8 + - almalinux-9 agents: provider: gcp image: family/elasticsearch-{{matrix.image}} @@ -46,7 +48,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 7.0.1 @@ -63,7 +64,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 7.1.1 @@ -80,7 +80,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 7.2.1 @@ -97,7 +96,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 7.3.2 @@ -114,7 +112,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 7.4.2 @@ -131,7 +128,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 7.5.2 @@ -148,7 +144,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 7.6.2 @@ -165,7 +160,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 7.7.1 @@ -182,7 +176,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 7.8.1 @@ -199,7 +192,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 7.9.3 @@ -216,7 +208,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 7.10.2 @@ -233,7 +224,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 7.11.2 @@ -250,7 +240,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 7.12.1 @@ -267,7 +256,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 7.13.4 @@ -284,7 +272,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 7.14.2 @@ -301,7 +288,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 7.15.2 @@ -318,12 +304,11 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 7.16.3 - - label: "{{matrix.image}} / 7.17.25 / packaging-tests-upgrade" - command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.25 + - label: "{{matrix.image}} / 7.17.27 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.27 timeout_in_minutes: 300 matrix: setup: @@ -335,9 +320,8 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: - BWC_VERSION: 7.17.25 + BWC_VERSION: 7.17.27 - label: "{{matrix.image}} / 8.0.1 / packaging-tests-upgrade" command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.0.1 @@ -352,7 +336,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.0.1 @@ -369,7 +352,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.1.3 @@ -386,7 +368,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.2.3 @@ -403,7 +384,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.3.3 @@ -420,7 +400,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.4.3 @@ -437,7 +416,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.5.3 @@ -454,7 +432,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.6.2 @@ -471,7 +448,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.7.1 @@ -488,7 +464,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.8.2 @@ -505,7 +480,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.9.2 @@ -522,7 +496,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.10.4 @@ -539,7 +512,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.11.4 @@ -556,7 +528,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.12.2 @@ -573,7 +544,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.13.4 @@ -590,12 +560,43 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.14.3 - - label: "{{matrix.image}} / 8.15.2 / packaging-tests-upgrade" - command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.2 + - label: "{{matrix.image}} / 8.15.5 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.5 + timeout_in_minutes: 300 + matrix: + setup: + image: + - rocky-8 + - ubuntu-2004 + agents: + provider: gcp + image: family/elasticsearch-{{matrix.image}} + machineType: custom-16-32768 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 8.15.5 + + - label: "{{matrix.image}} / 8.16.3 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.3 + timeout_in_minutes: 300 + matrix: + setup: + image: + - rocky-8 + - ubuntu-2004 + agents: + provider: gcp + image: family/elasticsearch-{{matrix.image}} + machineType: custom-16-32768 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 8.16.3 + + - label: "{{matrix.image}} / 8.17.1 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.1 timeout_in_minutes: 300 matrix: setup: @@ -607,12 +608,11 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: - BWC_VERSION: 8.15.2 + BWC_VERSION: 8.17.1 - - label: "{{matrix.image}} / 8.16.0 / packaging-tests-upgrade" - command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.0 + - label: "{{matrix.image}} / 8.18.0 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.0 timeout_in_minutes: 300 matrix: setup: @@ -624,9 +624,8 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: - BWC_VERSION: 8.16.0 + BWC_VERSION: 8.18.0 - group: packaging-tests-windows steps: diff --git a/.buildkite/pipelines/periodic-platform-support.yml b/.buildkite/pipelines/periodic-platform-support.yml index 867ebe41ed6af..e930e53b0ccd8 100644 --- a/.buildkite/pipelines/periodic-platform-support.yml +++ b/.buildkite/pipelines/periodic-platform-support.yml @@ -7,30 +7,31 @@ steps: matrix: setup: image: - - centos-7 - - debian-10 - debian-11 + - debian-12 - opensuse-leap-15 - oraclelinux-7 - oraclelinux-8 + - oraclelinux-9 - sles-12 - sles-15 - ubuntu-1804 - ubuntu-2004 - ubuntu-2204 + - ubuntu-2404 - rocky-8 - rocky-9 - rhel-7 - rhel-8 - rhel-9 - almalinux-8 + - almalinux-9 agents: provider: gcp image: family/elasticsearch-{{matrix.image}} localSsds: 1 localSsdInterface: nvme machineType: custom-32-98304 - diskSizeGb: 250 env: {} - group: platform-support-windows steps: @@ -69,6 +70,7 @@ steps: image: - almalinux-8-aarch64 - ubuntu-2004-aarch64 + - ubuntu-2404-aarch64 GRADLE_TASK: - checkPart1 - checkPart2 diff --git a/.buildkite/pipelines/periodic.bwc.template.yml b/.buildkite/pipelines/periodic.bwc.template.yml index b22270dbf221c..43a0a7438d656 100644 --- a/.buildkite/pipelines/periodic.bwc.template.yml +++ b/.buildkite/pipelines/periodic.bwc.template.yml @@ -7,7 +7,6 @@ machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: $BWC_VERSION retry: diff --git a/.buildkite/pipelines/periodic.template.yml b/.buildkite/pipelines/periodic.template.yml index 87e30a0ea73ba..110226d44c90e 100644 --- a/.buildkite/pipelines/periodic.template.yml +++ b/.buildkite/pipelines/periodic.template.yml @@ -25,7 +25,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: example-plugins command: |- cd $$WORKSPACE/plugins/examples @@ -37,7 +36,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - group: java-fips-matrix steps: - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.GRADLE_TASK}} / java-fips-matrix" @@ -59,7 +57,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" GRADLE_TASK: "{{matrix.GRADLE_TASK}}" @@ -76,7 +73,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" BWC_VERSION: "{{matrix.BWC_VERSION}}" @@ -105,7 +101,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" GRADLE_TASK: "{{matrix.GRADLE_TASK}}" @@ -126,7 +121,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" BWC_VERSION: "{{matrix.BWC_VERSION}}" @@ -162,7 +156,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: third-party / azure command: | export azure_storage_container=elasticsearch-ci-thirdparty @@ -177,7 +170,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: third-party / gcs command: | export google_storage_bucket=elasticsearch-ci-thirdparty @@ -192,7 +184,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: third-party / geoip command: | .ci/scripts/run-gradle.sh :modules:ingest-geoip:internalClusterTest -Dtests.jvm.argline="-Dgeoip_use_service=true" @@ -202,7 +193,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: third-party / s3 command: | export amazon_s3_bucket=elasticsearch-ci.us-west-2 @@ -217,7 +207,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: Upload Snyk Dependency Graph command: .ci/scripts/run-gradle.sh uploadSnykDependencyGraph -PsnykTargetReference=$BUILDKITE_BRANCH env: @@ -228,8 +217,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - if: build.branch == "main" || build.branch == "7.17" + if: build.branch == "main" || build.branch == "8.x" || build.branch == "7.17" - label: check-branch-consistency command: .ci/scripts/run-gradle.sh branchConsistency timeout_in_minutes: 15 @@ -237,7 +225,6 @@ steps: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-2 - diskSizeGb: 250 - label: check-branch-protection-rules command: .buildkite/scripts/branch-protection.sh timeout_in_minutes: 5 diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index 4f862911a2d8c..4b56b18e4cd83 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -11,7 +11,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 7.0.1 retry: @@ -31,7 +30,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 7.1.1 retry: @@ -51,7 +49,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 7.2.1 retry: @@ -71,7 +68,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 7.3.2 retry: @@ -91,7 +87,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 7.4.2 retry: @@ -111,7 +106,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 7.5.2 retry: @@ -131,7 +125,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 7.6.2 retry: @@ -151,7 +144,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 7.7.1 retry: @@ -171,7 +163,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 7.8.1 retry: @@ -191,7 +182,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 7.9.3 retry: @@ -211,7 +201,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 7.10.2 retry: @@ -231,7 +220,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 7.11.2 retry: @@ -251,7 +239,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 7.12.1 retry: @@ -271,7 +258,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 7.13.4 retry: @@ -291,7 +277,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 7.14.2 retry: @@ -311,7 +296,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 7.15.2 retry: @@ -331,7 +315,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 7.16.3 retry: @@ -342,8 +325,8 @@ steps: - signal_reason: agent_stop limit: 3 - - label: 7.17.25 / bwc - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.25#bwcTest + - label: 7.17.27 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.27#bwcTest timeout_in_minutes: 300 agents: provider: gcp @@ -351,9 +334,8 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: - BWC_VERSION: 7.17.25 + BWC_VERSION: 7.17.27 retry: automatic: - exit_status: "-1" @@ -371,7 +353,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.0.1 retry: @@ -391,7 +372,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.1.3 retry: @@ -411,7 +391,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.2.3 retry: @@ -431,7 +410,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.3.3 retry: @@ -451,7 +429,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.4.3 retry: @@ -471,7 +448,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.5.3 retry: @@ -491,7 +467,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.6.2 retry: @@ -511,7 +486,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.7.1 retry: @@ -531,7 +505,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.8.2 retry: @@ -551,7 +524,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.9.2 retry: @@ -571,7 +543,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.10.4 retry: @@ -591,7 +562,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.11.4 retry: @@ -611,7 +581,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.12.2 retry: @@ -631,7 +600,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.13.4 retry: @@ -651,7 +619,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.14.3 retry: @@ -662,8 +629,8 @@ steps: - signal_reason: agent_stop limit: 3 - - label: 8.15.2 / bwc - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.2#bwcTest + - label: 8.15.5 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.5#bwcTest timeout_in_minutes: 300 agents: provider: gcp @@ -671,9 +638,8 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: - BWC_VERSION: 8.15.2 + BWC_VERSION: 8.15.5 retry: automatic: - exit_status: "-1" @@ -682,8 +648,8 @@ steps: - signal_reason: agent_stop limit: 3 - - label: 8.16.0 / bwc - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.0#bwcTest + - label: 8.16.3 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.3#bwcTest timeout_in_minutes: 300 agents: provider: gcp @@ -691,9 +657,46 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: - BWC_VERSION: 8.16.0 + BWC_VERSION: 8.16.3 + retry: + automatic: + - exit_status: "-1" + limit: 3 + signal_reason: none + - signal_reason: agent_stop + limit: 3 + + - label: 8.17.1 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.1#bwcTest + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: n1-standard-32 + buildDirectory: /dev/shm/bk + preemptible: true + env: + BWC_VERSION: 8.17.1 + retry: + automatic: + - exit_status: "-1" + limit: 3 + signal_reason: none + - signal_reason: agent_stop + limit: 3 + + - label: 8.18.0 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.0#bwcTest + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: n1-standard-32 + buildDirectory: /dev/shm/bk + preemptible: true + env: + BWC_VERSION: 8.18.0 retry: automatic: - exit_status: "-1" @@ -726,7 +729,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: example-plugins command: |- cd $$WORKSPACE/plugins/examples @@ -738,7 +740,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - group: java-fips-matrix steps: - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.GRADLE_TASK}} / java-fips-matrix" @@ -760,7 +761,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" GRADLE_TASK: "{{matrix.GRADLE_TASK}}" @@ -771,13 +771,12 @@ steps: setup: ES_RUNTIME_JAVA: - openjdk17 - BWC_VERSION: ["7.17.25", "8.15.2", "8.16.0"] + BWC_VERSION: ["7.17.27", "8.16.3", "8.17.1", "8.18.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" BWC_VERSION: "{{matrix.BWC_VERSION}}" @@ -806,7 +805,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" GRADLE_TASK: "{{matrix.GRADLE_TASK}}" @@ -821,13 +819,12 @@ steps: - openjdk21 - openjdk22 - openjdk23 - BWC_VERSION: ["7.17.25", "8.15.2", "8.16.0"] + BWC_VERSION: ["7.17.27", "8.16.3", "8.17.1", "8.18.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" BWC_VERSION: "{{matrix.BWC_VERSION}}" @@ -863,7 +860,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: third-party / azure command: | export azure_storage_container=elasticsearch-ci-thirdparty @@ -878,7 +874,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: third-party / gcs command: | export google_storage_bucket=elasticsearch-ci-thirdparty @@ -893,7 +888,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: third-party / geoip command: | .ci/scripts/run-gradle.sh :modules:ingest-geoip:internalClusterTest -Dtests.jvm.argline="-Dgeoip_use_service=true" @@ -903,7 +897,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: third-party / s3 command: | export amazon_s3_bucket=elasticsearch-ci.us-west-2 @@ -918,7 +911,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: Upload Snyk Dependency Graph command: .ci/scripts/run-gradle.sh uploadSnykDependencyGraph -PsnykTargetReference=$BUILDKITE_BRANCH env: @@ -929,8 +921,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - if: build.branch == "main" || build.branch == "7.17" + if: build.branch == "main" || build.branch == "8.x" || build.branch == "7.17" - label: check-branch-consistency command: .ci/scripts/run-gradle.sh branchConsistency timeout_in_minutes: 15 @@ -938,7 +929,6 @@ steps: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-2 - diskSizeGb: 250 - label: check-branch-protection-rules command: .buildkite/scripts/branch-protection.sh timeout_in_minutes: 5 diff --git a/.buildkite/pipelines/pull-request/build-benchmark.yml b/.buildkite/pipelines/pull-request/build-benchmark.yml index 96330bee03638..8d3215b8393ce 100644 --- a/.buildkite/pipelines/pull-request/build-benchmark.yml +++ b/.buildkite/pipelines/pull-request/build-benchmark.yml @@ -22,4 +22,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/bwc-snapshots.yml b/.buildkite/pipelines/pull-request/bwc-snapshots.yml index 8f59e593b286f..5a9fc2d938ac0 100644 --- a/.buildkite/pipelines/pull-request/bwc-snapshots.yml +++ b/.buildkite/pipelines/pull-request/bwc-snapshots.yml @@ -18,4 +18,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/cloud-deploy.yml b/.buildkite/pipelines/pull-request/cloud-deploy.yml index 2932f874c5cf8..ce8e8206d51ff 100644 --- a/.buildkite/pipelines/pull-request/cloud-deploy.yml +++ b/.buildkite/pipelines/pull-request/cloud-deploy.yml @@ -11,4 +11,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/docs-check.yml b/.buildkite/pipelines/pull-request/docs-check.yml index 3bf1e43697a7c..2201eb2d1e4ea 100644 --- a/.buildkite/pipelines/pull-request/docs-check.yml +++ b/.buildkite/pipelines/pull-request/docs-check.yml @@ -12,4 +12,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/eql-correctness.yml b/.buildkite/pipelines/pull-request/eql-correctness.yml index d85827d10e886..8f7ca6942c0e9 100644 --- a/.buildkite/pipelines/pull-request/eql-correctness.yml +++ b/.buildkite/pipelines/pull-request/eql-correctness.yml @@ -7,4 +7,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/example-plugins.yml b/.buildkite/pipelines/pull-request/example-plugins.yml index fb4a17fb214cb..18d0de6594980 100644 --- a/.buildkite/pipelines/pull-request/example-plugins.yml +++ b/.buildkite/pipelines/pull-request/example-plugins.yml @@ -16,4 +16,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/full-bwc.yml b/.buildkite/pipelines/pull-request/full-bwc.yml index c404069bd0e60..d3fa8eccaf7d9 100644 --- a/.buildkite/pipelines/pull-request/full-bwc.yml +++ b/.buildkite/pipelines/pull-request/full-bwc.yml @@ -13,4 +13,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/packaging-tests-unix.yml b/.buildkite/pipelines/pull-request/packaging-tests-unix.yml index d5c937aa4b5a2..ddcdf531764d5 100644 --- a/.buildkite/pipelines/pull-request/packaging-tests-unix.yml +++ b/.buildkite/pipelines/pull-request/packaging-tests-unix.yml @@ -3,88 +3,36 @@ config: steps: - group: packaging-tests-unix steps: - - label: "{{matrix.image}} / docker / packaging-tests-unix" - key: "packaging-tests-unix-docker" - command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.docker + - label: "{{matrix.image}} / {{matrix.PACKAGING_TASK}} / packaging-tests-unix" + key: "packaging-tests-unix" + command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.{{matrix.PACKAGING_TASK}} timeout_in_minutes: 300 matrix: setup: image: - - centos-7 - - debian-10 - - debian-11 - - opensuse-leap-15 - - oraclelinux-7 - - oraclelinux-8 - - sles-12 - - sles-15 - - ubuntu-1804 - - ubuntu-2004 - - ubuntu-2204 - - rocky-8 - - rocky-9 - - rhel-7 - - rhel-8 - - rhel-9 - - almalinux-8 - agents: - provider: gcp - image: family/elasticsearch-{{matrix.image}} - diskSizeGb: 350 - machineType: custom-16-32768 - - label: "{{matrix.image}} / packages / packaging-tests-unix" - key: "packaging-tests-unix-packages" - command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.packages - timeout_in_minutes: 300 - matrix: - setup: - image: - - centos-7 - - debian-10 - - debian-11 - - opensuse-leap-15 - - oraclelinux-7 - - oraclelinux-8 - - sles-12 - - sles-15 - - ubuntu-1804 - - ubuntu-2004 - - ubuntu-2204 - - rocky-8 - - rocky-9 - - rhel-7 - - rhel-8 - - rhel-9 - - almalinux-8 - agents: - provider: gcp - image: family/elasticsearch-{{matrix.image}} - diskSizeGb: 350 - machineType: custom-16-32768 - - label: "{{matrix.image}} / archives / packaging-tests-unix" - key: "packaging-tests-unix-archives" - command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.archives - timeout_in_minutes: 300 - matrix: - setup: - image: - - centos-7 - - debian-10 - debian-11 + - debian-12 - opensuse-leap-15 - oraclelinux-7 - oraclelinux-8 + - oraclelinux-9 - sles-12 - sles-15 - ubuntu-1804 - ubuntu-2004 - ubuntu-2204 + - ubuntu-2404 - rocky-8 - rocky-9 - rhel-7 - rhel-8 - rhel-9 - almalinux-8 + - almalinux-9 + PACKAGING_TASK: + - docker-cloud-ess + - packages + - archives agents: provider: gcp image: family/elasticsearch-{{matrix.image}} diff --git a/.buildkite/pipelines/pull-request/packaging-upgrade-tests.yml b/.buildkite/pipelines/pull-request/packaging-upgrade-tests.yml index 970dafbb28647..c62cf23310422 100644 --- a/.buildkite/pipelines/pull-request/packaging-upgrade-tests.yml +++ b/.buildkite/pipelines/pull-request/packaging-upgrade-tests.yml @@ -18,6 +18,5 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: $BWC_VERSION diff --git a/.buildkite/pipelines/pull-request/part-1-fips.yml b/.buildkite/pipelines/pull-request/part-1-fips.yml index 99544e7f5a80b..42f930c1bde9a 100644 --- a/.buildkite/pipelines/pull-request/part-1-fips.yml +++ b/.buildkite/pipelines/pull-request/part-1-fips.yml @@ -9,4 +9,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-1.yml b/.buildkite/pipelines/pull-request/part-1.yml index b4b9d5469ec41..7a09e2a162ff8 100644 --- a/.buildkite/pipelines/pull-request/part-1.yml +++ b/.buildkite/pipelines/pull-request/part-1.yml @@ -1,10 +1,11 @@ steps: - label: part-1 - command: .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart1 + command: | + .buildkite/scripts/spotless.sh # This doesn't have to be part of part-1, it was just a convenient place to put it + .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart1 timeout_in_minutes: 300 agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-2-fips.yml b/.buildkite/pipelines/pull-request/part-2-fips.yml index 36a9801547d78..6a3647ceb50ae 100644 --- a/.buildkite/pipelines/pull-request/part-2-fips.yml +++ b/.buildkite/pipelines/pull-request/part-2-fips.yml @@ -9,4 +9,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-2.yml b/.buildkite/pipelines/pull-request/part-2.yml index 12bd78cf895fd..43de69bbcd945 100644 --- a/.buildkite/pipelines/pull-request/part-2.yml +++ b/.buildkite/pipelines/pull-request/part-2.yml @@ -7,4 +7,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-3-fips.yml b/.buildkite/pipelines/pull-request/part-3-fips.yml index 4a2df3026e782..cee3ea153acb9 100644 --- a/.buildkite/pipelines/pull-request/part-3-fips.yml +++ b/.buildkite/pipelines/pull-request/part-3-fips.yml @@ -9,4 +9,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-3.yml b/.buildkite/pipelines/pull-request/part-3.yml index 6991c05da85c6..12abae7634822 100644 --- a/.buildkite/pipelines/pull-request/part-3.yml +++ b/.buildkite/pipelines/pull-request/part-3.yml @@ -9,4 +9,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-4-fips.yml b/.buildkite/pipelines/pull-request/part-4-fips.yml index 734f8af816895..11a50456ca4c0 100644 --- a/.buildkite/pipelines/pull-request/part-4-fips.yml +++ b/.buildkite/pipelines/pull-request/part-4-fips.yml @@ -9,4 +9,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-4.yml b/.buildkite/pipelines/pull-request/part-4.yml index 59f2f2898a590..af11f08953d07 100644 --- a/.buildkite/pipelines/pull-request/part-4.yml +++ b/.buildkite/pipelines/pull-request/part-4.yml @@ -9,4 +9,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-5-fips.yml b/.buildkite/pipelines/pull-request/part-5-fips.yml index 801b812bb99c0..4e193ac751086 100644 --- a/.buildkite/pipelines/pull-request/part-5-fips.yml +++ b/.buildkite/pipelines/pull-request/part-5-fips.yml @@ -9,4 +9,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-5.yml b/.buildkite/pipelines/pull-request/part-5.yml index c7e50631d1cdd..306ce7533d0ed 100644 --- a/.buildkite/pipelines/pull-request/part-5.yml +++ b/.buildkite/pipelines/pull-request/part-5.yml @@ -9,4 +9,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/precommit.yml b/.buildkite/pipelines/pull-request/precommit.yml index 8d1458b1b60c8..1763758932581 100644 --- a/.buildkite/pipelines/pull-request/precommit.yml +++ b/.buildkite/pipelines/pull-request/precommit.yml @@ -3,11 +3,12 @@ config: skip-labels: [] steps: - label: precommit - command: .ci/scripts/run-gradle.sh -Dignore.tests.seed precommit + command: | + .buildkite/scripts/spotless.sh + .ci/scripts/run-gradle.sh -Dignore.tests.seed precommit timeout_in_minutes: 300 agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/rest-compatibility.yml b/.buildkite/pipelines/pull-request/rest-compatibility.yml index 16144a2a0780f..a69810e23d960 100644 --- a/.buildkite/pipelines/pull-request/rest-compatibility.yml +++ b/.buildkite/pipelines/pull-request/rest-compatibility.yml @@ -9,4 +9,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/validate-changelogs.yml b/.buildkite/pipelines/pull-request/validate-changelogs.yml index 296ef11637118..9451d321a9b39 100644 --- a/.buildkite/pipelines/pull-request/validate-changelogs.yml +++ b/.buildkite/pipelines/pull-request/validate-changelogs.yml @@ -7,4 +7,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/scripts/dra-workflow.sh b/.buildkite/scripts/dra-workflow.sh index ecfb8088072a0..f2dc40ca1927f 100755 --- a/.buildkite/scripts/dra-workflow.sh +++ b/.buildkite/scripts/dra-workflow.sh @@ -6,7 +6,7 @@ WORKFLOW="${DRA_WORKFLOW:-snapshot}" BRANCH="${BUILDKITE_BRANCH:-}" # Don't publish main branch to staging -if [[ "$BRANCH" == "main" && "$WORKFLOW" == "staging" ]]; then +if [[ ("$BRANCH" == "main" || "$BRANCH" == *.x) && "$WORKFLOW" == "staging" ]]; then exit 0 fi @@ -22,6 +22,7 @@ if [[ "$BRANCH" == "main" ]]; then fi ES_VERSION=$(grep elasticsearch build-tools-internal/version.properties | sed "s/elasticsearch *= *//g") +echo "ES_VERSION=$ES_VERSION" VERSION_SUFFIX="" if [[ "$WORKFLOW" == "snapshot" ]]; then @@ -29,7 +30,10 @@ if [[ "$WORKFLOW" == "snapshot" ]]; then fi BEATS_BUILD_ID="$(./.ci/scripts/resolve-dra-manifest.sh beats "$RM_BRANCH" "$ES_VERSION" "$WORKFLOW")" +echo "BEATS_BUILD_ID=$BEATS_BUILD_ID" + ML_CPP_BUILD_ID="$(./.ci/scripts/resolve-dra-manifest.sh ml-cpp "$RM_BRANCH" "$ES_VERSION" "$WORKFLOW")" +echo "ML_CPP_BUILD_ID=$ML_CPP_BUILD_ID" LICENSE_KEY_ARG="" BUILD_SNAPSHOT_ARG="" diff --git a/.buildkite/scripts/get-latest-test-mutes.sh b/.buildkite/scripts/get-latest-test-mutes.sh new file mode 100755 index 0000000000000..5721e29f1b773 --- /dev/null +++ b/.buildkite/scripts/get-latest-test-mutes.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +if [[ ! "${BUILDKITE_PULL_REQUEST:-}" || "${BUILDKITE_AGENT_META_DATA_PROVIDER:-}" == "k8s" ]]; then + exit 0 +fi + +testMuteBranch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}" +testMuteFile="$(mktemp)" + +# If this PR contains changes to muted-tests.yml, we disable this functionality +# Otherwise, we wouldn't be able to test unmutes +if [[ ! $(gh pr diff "$BUILDKITE_PULL_REQUEST" --name-only | grep 'muted-tests.yml') ]]; then + gh api -H 'Accept: application/vnd.github.v3.raw' "repos/elastic/elasticsearch/contents/muted-tests.yml?ref=$testMuteBranch" > "$testMuteFile" + + if [[ -s "$testMuteFile" ]]; then + mkdir -p ~/.gradle + # This is using gradle.properties instead of an env var so that it's easily compatible with the Windows pre-command hook + echo "org.gradle.project.org.elasticsearch.additional.muted.tests=$testMuteFile" >> ~/.gradle/gradle.properties + fi +fi diff --git a/.buildkite/scripts/lucene-snapshot/update-branch.sh b/.buildkite/scripts/lucene-snapshot/update-branch.sh index 6a2d1e3df05f7..a2a80824c984f 100755 --- a/.buildkite/scripts/lucene-snapshot/update-branch.sh +++ b/.buildkite/scripts/lucene-snapshot/update-branch.sh @@ -7,12 +7,21 @@ if [[ "$BUILDKITE_BRANCH" != "lucene_snapshot"* ]]; then exit 1 fi -echo --- Updating "$BUILDKITE_BRANCH" branch with main +if [[ "$BUILDKITE_BRANCH" == "lucene_snapshot_10" ]]; then + UPSTREAM="main" +elif [[ "$BUILDKITE_BRANCH" == "lucene_snapshot" ]]; then + UPSTREAM="8.x" +else + echo "Error: unknown branch: $BUILDKITE_BRANCH" + exit 1 +fi + +echo --- Updating "$BUILDKITE_BRANCH" branch with "$UPSTREAM" git config --global user.name elasticsearchmachine git config --global user.email 'infra-root+elasticsearchmachine@elastic.co' git checkout "$BUILDKITE_BRANCH" -git fetch origin main -git merge --no-edit origin/main +git fetch origin "$UPSTREAM" +git merge --no-edit "origin/$UPSTREAM" git push origin "$BUILDKITE_BRANCH" diff --git a/.buildkite/scripts/spotless.sh b/.buildkite/scripts/spotless.sh new file mode 100755 index 0000000000000..b9e6094edb2c7 --- /dev/null +++ b/.buildkite/scripts/spotless.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +if [[ -z "${BUILDKITE_PULL_REQUEST:-}" ]]; then + echo "Not a pull request, skipping spotless" + exit 0 +fi + +if ! git diff --exit-code; then + echo "Changes are present before running spotless, not running" + git status + exit 0 +fi + +NEW_COMMIT_MESSAGE="[CI] Auto commit changes from spotless" +PREVIOUS_COMMIT_MESSAGE="$(git log -1 --pretty=%B)" + +echo "--- Running spotless" +.ci/scripts/run-gradle.sh -Dscan.tag.NESTED spotlessApply + +if git diff --exit-code; then + echo "No changes found after running spotless. Don't need to auto commit." + exit 0 +fi + +if [[ "$NEW_COMMIT_MESSAGE" == "$PREVIOUS_COMMIT_MESSAGE" ]]; then + echo "Changes found after running spotless" + echo "CI already attempted to commit these changes, but the file(s) seem to have changed again." + echo "Please review and fix manually." + exit 1 +fi + +git config --global user.name elasticsearchmachine +git config --global user.email 'infra-root+elasticsearchmachine@elastic.co' + +gh pr checkout "${BUILDKITE_PULL_REQUEST}" +git add -u . +git commit -m "$NEW_COMMIT_MESSAGE" +git push + +# After the git push, the new commit will trigger a new build within a few seconds and this build should get cancelled +# So, let's just sleep to give the build time to cancel itself without an error +# If it doesn't get cancelled for some reason, then exit with an error, because we don't want this build to be green (we just don't want it to generate an error either) +sleep 300 +exit 1 diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 6c5aaa38717ef..3a66c90650b34 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -16,7 +16,7 @@ BWC_VERSION: - "7.14.2" - "7.15.2" - "7.16.3" - - "7.17.25" + - "7.17.27" - "8.0.1" - "8.1.3" - "8.2.3" @@ -32,5 +32,7 @@ BWC_VERSION: - "8.12.2" - "8.13.4" - "8.14.3" - - "8.15.2" - - "8.16.0" + - "8.15.5" + - "8.16.3" + - "8.17.1" + - "8.18.0" diff --git a/.ci/dockerOnLinuxExclusions b/.ci/dockerOnLinuxExclusions index c150cca590f7d..715ed86188dd5 100644 --- a/.ci/dockerOnLinuxExclusions +++ b/.ci/dockerOnLinuxExclusions @@ -15,6 +15,7 @@ sles-15.2 sles-15.3 sles-15.4 sles-15.5 +sles-15.6 # These OSes are deprecated and filtered starting with 8.0.0, but need to be excluded # for PR checks diff --git a/.ci/scripts/packaging-test.sh b/.ci/scripts/packaging-test.sh index 1626255c30b4f..10e20d9138bb0 100755 --- a/.ci/scripts/packaging-test.sh +++ b/.ci/scripts/packaging-test.sh @@ -77,5 +77,6 @@ sudo -E env \ --unset=ES_JAVA_HOME \ --unset=JAVA_HOME \ SYSTEM_JAVA_HOME=`readlink -f -n $BUILD_JAVA_HOME` \ - ./gradlew -g $HOME/.gradle --scan --parallel --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ --continue $@ + DOCKER_CONFIG="${HOME}/.docker" \ + ./gradlew -g $HOME/.gradle --console=plain --scan --parallel --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ --continue $@ diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions index f00be923db67c..193be7ecc4edc 100644 --- a/.ci/snapshotBwcVersions +++ b/.ci/snapshotBwcVersions @@ -1,4 +1,5 @@ BWC_VERSION: - - "7.17.25" - - "8.15.2" - - "8.16.0" + - "7.17.27" + - "8.16.3" + - "8.17.1" + - "8.18.0" diff --git a/.editorconfig b/.editorconfig index cf4f74744d2b4..774fd201ef8d5 100644 --- a/.editorconfig +++ b/.editorconfig @@ -209,7 +209,7 @@ indent_size = 4 max_line_length = 140 ij_java_class_count_to_use_import_on_demand = 999 ij_java_names_count_to_use_import_on_demand = 999 -ij_java_imports_layout = *,|,com.**,|,org.**,|,java.**,javax.**,|,$* +ij_java_imports_layout = *,|,com.**,|,org.**,|,java.**,|,javax.**,|,$* [*.json] indent_size = 2 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index f0d9068820029..0000000000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1,76 +0,0 @@ -# GitHub CODEOWNERS definition -# Identify which groups will be pinged by changes to different parts of the codebase. -# For more info, see https://help.github.com/articles/about-codeowners/ - -# Stack Monitoring index templates -x-pack/plugin/core/template-resources/src/main/resources/monitoring-alerts-7.json @elastic/stack-monitoring -x-pack/plugin/core/template-resources/src/main/resources/monitoring-beats-mb.json @elastic/stack-monitoring -x-pack/plugin/core/template-resources/src/main/resources/monitoring-beats.json @elastic/stack-monitoring -x-pack/plugin/core/template-resources/src/main/resources/monitoring-ent-search-mb.json @elastic/stack-monitoring -x-pack/plugin/core/template-resources/src/main/resources/monitoring-es-mb.json @elastic/stack-monitoring -x-pack/plugin/core/template-resources/src/main/resources/monitoring-es.json @elastic/stack-monitoring -x-pack/plugin/core/template-resources/src/main/resources/monitoring-kibana-mb.json @elastic/stack-monitoring -x-pack/plugin/core/template-resources/src/main/resources/monitoring-kibana.json @elastic/stack-monitoring -x-pack/plugin/core/template-resources/src/main/resources/monitoring-logstash-mb.json @elastic/stack-monitoring -x-pack/plugin/core/template-resources/src/main/resources/monitoring-logstash.json @elastic/stack-monitoring -x-pack/plugin/core/template-resources/src/main/resources/monitoring-mb-ilm-policy.json @elastic/stack-monitoring -x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java @elastic/stack-monitoring - -# Fleet -x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet @elastic/fleet -x-pack/plugin/core/src/main/resources/fleet-* @elastic/fleet - -# Logstash -libs/logstash-bridge @elastic/logstash - -# Kibana Security -x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @elastic/kibana-security - -# APM Data index templates, etc. -x-pack/plugin/apm-data/src/main/resources @elastic/obs-ds-intake-services -x-pack/plugin/apm-data/src/yamlRestTest/resources @elastic/obs-ds-intake-services - -# OTel -x-pack/plugin/otel-data/src/main/resources @elastic/obs-ds-intake-services -x-pack/plugin/otel-data/src/yamlRestTest/resources @elastic/obs-ds-intake-services - -# Delivery -gradle @elastic/es-delivery -build-conventions @elastic/es-delivery -build-tools @elastic/es-delivery -build-tools-internal @elastic/es-delivery -*.gradle @elastic/es-delivery -.buildkite @elastic/es-delivery -.ci @elastic/es-delivery -.idea @elastic/es-delivery -distribution/src @elastic/es-delivery -distribution/packages/src @elastic/es-delivery -distribution/docker/src @elastic/es-delivery - -# Core/Infra -distribution/tools @elastic/es-core-infra -libs/core @elastic/es-core-infra -libs/logging @elastic/es-core-infra -libs/native @elastic/es-core-infra -libs/plugin-analysis-api @elastic/es-core-infra -libs/plugin-api @elastic/es-core-infra -libs/plugin-classloader @elastic/es-core-infra -libs/plugin-scanner @elastic/es-core-infra -libs/x-content @elastic/es-core-infra -modules/lang-expression @elastic/es-core-infra -modules/lang-mustache @elastic/es-core-infra -modules/lang-painless @elastic/es-core-infra -modules/rest-root @elastic/es-core-infra -modules/systemd @elastic/es-core-infra -server/src/main/java/org/elasticsearch/bootstrap @elastic/es-core-infra -server/src/main/java/org/elasticsearch/node @elastic/es-core-infra -server/src/main/java/org/elasticsearch/plugins @elastic/es-core-infra -server/src/main/java/org/elasticsearch/threadpool @elastic/es-core-infra - -# Security -x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege @elastic/es-security -x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java @elastic/es-security - -# Analytical engine -x-pack/plugin/esql @elastic/es-analytical-engine -x-pack/plugin/esql-core @elastic/es-analytical-engine diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5b68fd807220e..66da179d48c6a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -378,10 +378,11 @@ license header unless instructed otherwise: /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ The top-level `x-pack` directory contains code covered by the [Elastic @@ -659,51 +660,11 @@ node cannot continue to operate as a member of the cluster: Errors like this should be very rare. When in doubt, prefer `WARN` to `ERROR`. -### Version numbers in the Elasticsearch codebase - -Starting in 8.8.0, we have separated out the version number representations -of various aspects of Elasticsearch into their own classes, using their own -numbering scheme separate to release version. The main ones are -`TransportVersion` and `IndexVersion`, representing the version of the -inter-node binary protocol and index data + metadata respectively. - -Separated version numbers are comprised of an integer number. The semantic -meaning of a version number are defined within each `*Version` class. There -is no direct mapping between separated version numbers and the release version. -The versions used by any particular instance of Elasticsearch can be obtained -by querying `/_nodes/info` on the node. - -#### Using separated version numbers - -Whenever a change is made to a component versioned using a separated version -number, there are a few rules that need to be followed: - -1. Each version number represents a specific modification to that component, - and should not be modified once it is defined. Each version is immutable - once merged into `main`. -2. To create a new component version, add a new constant to the respective class - with a descriptive name of the change being made. Increment the integer - number according to the particular `*Version` class. - -If your pull request has a conflict around your new version constant, -you need to update your PR from `main` and change your PR to use the next -available version number. - -### Checking for cluster features - -As part of developing a new feature or change, you might need to determine -if all nodes in a cluster have been upgraded to support your new feature. -This can be done using `FeatureService`. To define and check for a new -feature in a cluster: - -1. Define a new `NodeFeature` constant with a unique id for the feature - in a class related to the change you're doing. -2. Return that constant from an instance of `FeatureSpecification.getFeatures`, - either an existing implementation or a new implementation. Make sure - the implementation is added as an SPI implementation in `module-info.java` - and `META-INF/services`. -3. To check if all nodes in the cluster support the new feature, call -`FeatureService.clusterHasFeature(ClusterState, NodeFeature)` +### Versioning Elasticsearch + +There are various concepts used to identify running node versions, +and the capabilities and compatibility of those nodes. For more information, +see `docs/internal/Versioning.md` ### Creating a distribution diff --git a/LICENSE.txt b/LICENSE.txt index 530576692e0ed..060d0d0e8f13c 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,7 +1,9 @@ -Source code in this repository is covered by (i) a dual license under the Server -Side Public License, v 1 and the Elastic License 2.0 or (ii) an Apache License -2.0 compatible license or (iii) solely under the Elastic License 2.0, in each -case, as noted in the applicable header. The default throughout the repository -is a dual license under the Server Side Public License, v 1 and the Elastic -License 2.0, unless the header specifies another license. Code that is licensed -solely under the Elastic License 2.0 is found only in the x-pack folder. +Source code in this repository is covered by (i) a triple license under the "GNU +Affero General Public License v3.0 only", "the Server Side Public License, v 1", +and the "Elastic License 2.0", or (ii) an "Apache License 2.0" compatible +license or (iii) solely under the "Elastic License 2.0", in each case, as noted +in the applicable header. The default throughout the repository is a triple +license under the "GNU Affero General Public License v3.0 only", "the Server +Side Public License, v 1", and the "Elastic License 2.0", unless the header +specifies another license. Code that is licensed solely under the "Elastic +License 2.0" is found only in the x-pack folder. diff --git a/README.asciidoc b/README.asciidoc index c1945e56b025b..a1ccbefe3bdbc 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -33,9 +33,11 @@ https://www.elastic.co/downloads/elasticsearch[elastic.co/downloads/elasticsearc === Run Elasticsearch locally //// -IMPORTANT: This content is replicated in the Elasticsearch guide. See `run-elasticsearch-locally.asciidoc`. -Both will soon be replaced by a quickstart script. -//// +IMPORTANT: This content is replicated in `run-elasticsearch-locally.asciidoc`. +Ensure both files are in sync. + +https://github.com/elastic/start-local is the source of truth. +//// [WARNING] ==== @@ -44,89 +46,67 @@ DO NOT USE THESE INSTRUCTIONS FOR PRODUCTION DEPLOYMENTS. This setup is intended for local development and testing only. ==== -The following commands help you very quickly spin up a single-node Elasticsearch cluster, together with Kibana in Docker. -Use this setup for local development or testing. +Quickly set up Elasticsearch and Kibana in Docker for local development or testing, using the https://github.com/elastic/start-local?tab=readme-ov-file#-try-elasticsearch-and-kibana-locally[`start-local` script]. -==== Prerequisites +ℹ️ For more detailed information about the `start-local` setup, refer to the https://github.com/elastic/start-local[README on GitHub]. -If you don't have Docker installed, https://www.docker.com/products/docker-desktop[download and install Docker Desktop] for your operating system. +==== Prerequisites -==== Set environment variables +- If you don't have Docker installed, https://www.docker.com/products/docker-desktop[download and install Docker Desktop] for your operating system. +- If you're using Microsoft Windows, then install https://learn.microsoft.com/en-us/windows/wsl/install[Windows Subsystem for Linux (WSL)]. -Configure the following environment variables. +==== Trial license -[source,sh] ----- -export ELASTIC_PASSWORD="" # password for "elastic" username -export KIBANA_PASSWORD="" # Used internally by Kibana, must be at least 6 characters long ----- +This setup comes with a one-month trial of the Elastic *Platinum* license. +After the trial period, the license reverts to *Free and open - Basic*. +Refer to https://www.elastic.co/subscriptions[Elastic subscriptions] for more information. -==== Create a Docker network +==== Run `start-local` -To run both Elasticsearch and Kibana, you'll need to create a Docker network: +To set up Elasticsearch and Kibana locally, run the `start-local` script: [source,sh] ---- -docker network create elastic-net +curl -fsSL https://elastic.co/start-local | sh ---- +// NOTCONSOLE -==== Run Elasticsearch +This script creates an `elastic-start-local` folder containing configuration files and starts both Elasticsearch and Kibana using Docker. -Start the Elasticsearch container with the following command: +After running the script, you can access Elastic services at the following endpoints: -[source,sh] ----- -docker run -p 127.0.0.1:9200:9200 -d --name elasticsearch --network elastic-net \ - -e ELASTIC_PASSWORD=$ELASTIC_PASSWORD \ - -e "discovery.type=single-node" \ - -e "xpack.security.http.ssl.enabled=false" \ - -e "xpack.license.self_generated.type=trial" \ - docker.elastic.co/elasticsearch/elasticsearch:{version} ----- +* *Elasticsearch*: http://localhost:9200 +* *Kibana*: http://localhost:5601 -==== Run Kibana (optional) +The script generates a random password for the `elastic` user, which is displayed at the end of the installation and stored in the `.env` file. -To run Kibana, you must first set the `kibana_system` password in the Elasticsearch container. +[CAUTION] +==== +This setup is for local testing only. HTTPS is disabled, and Basic authentication is used for Elasticsearch. For security, Elasticsearch and Kibana are accessible only through `localhost`. +==== -[source,sh] ----- -# configure the Kibana password in the ES container -curl -u elastic:$ELASTIC_PASSWORD \ - -X POST \ - http://localhost:9200/_security/user/kibana_system/_password \ - -d '{"password":"'"$KIBANA_PASSWORD"'"}' \ - -H 'Content-Type: application/json' ----- -// NOTCONSOLE +==== API access -Start the Kibana container with the following command: +An API key for Elasticsearch is generated and stored in the `.env` file as `ES_LOCAL_API_KEY`. +Use this key to connect to Elasticsearch with a https://www.elastic.co/guide/en/elasticsearch/client/index.html[programming language client] or the https://www.elastic.co/guide/en/elasticsearch/reference/current/rest-apis.html[REST API]. + +From the `elastic-start-local` folder, check the connection to Elasticsearch using `curl`: [source,sh] +---- +source .env +curl $ES_LOCAL_URL -H "Authorization: ApiKey ${ES_LOCAL_API_KEY}" ---- -docker run -p 127.0.0.1:5601:5601 -d --name kibana --network elastic-net \ - -e ELASTICSEARCH_URL=http://elasticsearch:9200 \ - -e ELASTICSEARCH_HOSTS=http://elasticsearch:9200 \ - -e ELASTICSEARCH_USERNAME=kibana_system \ - -e ELASTICSEARCH_PASSWORD=$KIBANA_PASSWORD \ - -e "xpack.security.enabled=false" \ - -e "xpack.license.self_generated.type=trial" \ - docker.elastic.co/kibana/kibana:{version} ----- - -.Trial license -[%collapsible] -==== -The service is started with a trial license. The trial license enables all features of Elasticsearch for a trial period of 30 days. After the trial period expires, the license is downgraded to a basic license, which is free forever. If you prefer to skip the trial and use the basic license, set the value of the `xpack.license.self_generated.type` variable to basic instead. For a detailed feature comparison between the different licenses, refer to our https://www.elastic.co/subscriptions[subscriptions page]. -==== +// NOTCONSOLE -==== Send requests to Elasticsearch +=== Send requests to Elasticsearch You send data and other requests to Elasticsearch through REST APIs. You can interact with Elasticsearch using any client that sends HTTP requests, such as the https://www.elastic.co/guide/en/elasticsearch/client/index.html[Elasticsearch language clients] and https://curl.se[curl]. -===== Using curl +==== Using curl Here's an example curl command to create a new Elasticsearch index, using basic auth: @@ -139,7 +119,7 @@ curl -u elastic:$ELASTIC_PASSWORD \ ---- // NOTCONSOLE -===== Using a language client +==== Using a language client To connect to your local dev Elasticsearch cluster with a language client, you can use basic authentication with the `elastic` username and the password you set in the environment variable. @@ -167,7 +147,7 @@ client = Elasticsearch( print(client.info()) ---- -===== Using the Dev Tools Console +==== Using the Dev Tools Console Kibana's developer console provides an easy way to experiment and test requests. To access the console, open Kibana, then go to **Management** > **Dev Tools**. diff --git a/TESTING.asciidoc b/TESTING.asciidoc index 2c205f9090ba8..f6a0670f33590 100644 --- a/TESTING.asciidoc +++ b/TESTING.asciidoc @@ -555,8 +555,8 @@ Sometimes a backward compatibility change spans two versions. A common case is a new functionality that needs a BWC bridge in an unreleased versioned of a release branch (for example, 5.x). Another use case, since the introduction of serverless, is to test BWC against main in addition to the other released branches. To do so, specify the `bwc.refspec` remote and branch to use for the BWC build as `origin/main`. -To test against main, you will also need to create a new version in link:./server/src/main/java/org/elasticsearch/Version.java[Version.java], -increment `elasticsearch` in link:./build-tools-internal/version.properties[version.properties], and hard-code the `project.version` for ml-cpp +To test against main, you will also need to create a new version in link:./server/src/main/java/org/elasticsearch/Version.java[Version.java], +increment `elasticsearch` in link:./build-tools-internal/version.properties[version.properties], and hard-code the `project.version` for ml-cpp in link:./x-pack/plugin/ml/build.gradle[ml/build.gradle]. In general, to test the changes, you can instruct Gradle to build the BWC version from another remote/branch combination instead of pulling the release branch from GitHub. @@ -616,7 +616,7 @@ For specific YAML rest tests one can use For disabling entire types of tests for subprojects, one can use for example: ------------------------------------------------ -if (BuildParams.inFipsJvm){ +if (buildParams.inFipsJvm) { // This test cluster is using a BASIC license and FIPS 140 mode is not supported in BASIC tasks.named("javaRestTest").configure{enabled = false } } diff --git a/Vagrantfile b/Vagrantfile index cdabd3a7bacd9..74fc4db48c552 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -5,10 +5,11 @@ # vagrant section in TESTING.asciidoc. # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". define_opts = { autostart: false diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle index b16621aaaa471..25cfae6c9803a 100644 --- a/benchmarks/build.gradle +++ b/benchmarks/build.gradle @@ -1,12 +1,12 @@ -import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.internal.test.TestUtil /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: org.elasticsearch.gradle.internal.ElasticsearchJavaBasePlugin @@ -39,15 +39,15 @@ dependencies { // us to invoke the JMH uberjar as usual. exclude group: 'net.sf.jopt-simple', module: 'jopt-simple' } - api(project(':libs:elasticsearch-h3')) + api(project(':libs:h3')) api(project(':modules:aggregations')) api(project(':x-pack:plugin:esql-core')) api(project(':x-pack:plugin:esql')) api(project(':x-pack:plugin:esql:compute')) - implementation project(path: ':libs:elasticsearch-simdvec') + implementation project(path: ':libs:simdvec') expression(project(path: ':modules:lang-expression', configuration: 'zip')) painless(project(path: ':modules:lang-painless', configuration: 'zip')) - nativeLib(project(':libs:elasticsearch-native')) + nativeLib(project(':libs:native')) api "org.openjdk.jmh:jmh-core:$versions.jmh" annotationProcessor "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh" // Dependencies of JMH @@ -77,7 +77,7 @@ tasks.register("copyPainless", Copy) { } tasks.named("run").configure { - executable = "${BuildParams.runtimeJavaHome}/bin/java" + executable = "${buildParams.runtimeJavaHome.get()}/bin/java" args << "-Dplugins.dir=${buildDir}/plugins" << "-Dtests.index=${buildDir}/index" dependsOn "copyExpression", "copyPainless", configurations.nativeLib systemProperty 'es.nativelibs.path', TestUtil.getTestLibraryPath(file("../libs/native/libraries/build/platform/").toString()) diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/BytesArrayReadLongBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/BytesArrayReadLongBenchmark.java index 9e830362f3a76..35c15eca45a01 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/BytesArrayReadLongBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/BytesArrayReadLongBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.bytes; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/BytesArrayReadVLongBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/BytesArrayReadVLongBenchmark.java index 9c8c44b9f4ed5..1512a0b8e8c48 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/BytesArrayReadVLongBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/BytesArrayReadVLongBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.bytes; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/PagedBytesReferenceReadLongBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/PagedBytesReferenceReadLongBenchmark.java index 79f43919b6566..24ce3b6fe1c64 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/PagedBytesReferenceReadLongBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/PagedBytesReferenceReadLongBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.bytes; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/PagedBytesReferenceReadVIntBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/PagedBytesReferenceReadVIntBenchmark.java index e28a0218177a6..3bb8516b22fc7 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/PagedBytesReferenceReadVIntBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/PagedBytesReferenceReadVIntBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.bytes; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/PagedBytesReferenceReadVLongBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/PagedBytesReferenceReadVLongBenchmark.java index e24b5ed554dc9..df0b0880fc3c6 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/PagedBytesReferenceReadVLongBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/PagedBytesReferenceReadVLongBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.bytes; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/common/util/IntArrayBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/common/util/IntArrayBenchmark.java index 973155f69effd..a412ef26d36f1 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/common/util/IntArrayBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/common/util/IntArrayBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.common.util; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java index 8b22e50e4e8c9..652defa7b39cd 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.compute.operator; @@ -16,6 +17,7 @@ import org.elasticsearch.compute.aggregation.CountAggregatorFunction; import org.elasticsearch.compute.aggregation.CountDistinctDoubleAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.CountDistinctLongAggregatorFunctionSupplier; +import org.elasticsearch.compute.aggregation.FilteredAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MaxDoubleAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MaxLongAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MinDoubleAggregatorFunctionSupplier; @@ -26,14 +28,19 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.data.ElementType; import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.OrdinalBytesRefVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.AggregationOperator; import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.compute.operator.HashAggregationOperator; import org.elasticsearch.compute.operator.Operator; import org.openjdk.jmh.annotations.Benchmark; @@ -74,7 +81,10 @@ public class AggregatorBenchmark { private static final String DOUBLES = "doubles"; private static final String BOOLEANS = "booleans"; private static final String BYTES_REFS = "bytes_refs"; + private static final String ORDINALS = "ordinals"; private static final String TWO_LONGS = "two_" + LONGS; + private static final String TWO_BYTES_REFS = "two_" + BYTES_REFS; + private static final String TWO_ORDINALS = "two_" + ORDINALS; private static final String LONGS_AND_BYTES_REFS = LONGS + "_and_" + BYTES_REFS; private static final String TWO_LONGS_AND_BYTES_REFS = "two_" + LONGS + "_and_" + BYTES_REFS; @@ -93,13 +103,20 @@ public class AggregatorBenchmark { private static final String NONE = "none"; + private static final String CONSTANT_TRUE = "constant_true"; + private static final String ALL_TRUE = "all_true"; + private static final String HALF_TRUE = "half_true"; + private static final String CONSTANT_FALSE = "constant_false"; + static { // Smoke test all the expected values and force loading subclasses more like prod try { for (String grouping : AggregatorBenchmark.class.getField("grouping").getAnnotationsByType(Param.class)[0].value()) { for (String op : AggregatorBenchmark.class.getField("op").getAnnotationsByType(Param.class)[0].value()) { for (String blockType : AggregatorBenchmark.class.getField("blockType").getAnnotationsByType(Param.class)[0].value()) { - run(grouping, op, blockType, 50); + for (String filter : AggregatorBenchmark.class.getField("filter").getAnnotationsByType(Param.class)[0].value()) { + run(grouping, op, blockType, filter, 10); + } } } } @@ -108,7 +125,21 @@ public class AggregatorBenchmark { } } - @Param({ NONE, LONGS, INTS, DOUBLES, BOOLEANS, BYTES_REFS, TWO_LONGS, LONGS_AND_BYTES_REFS, TWO_LONGS_AND_BYTES_REFS }) + @Param( + { + NONE, + LONGS, + INTS, + DOUBLES, + BOOLEANS, + BYTES_REFS, + ORDINALS, + TWO_LONGS, + TWO_BYTES_REFS, + TWO_ORDINALS, + LONGS_AND_BYTES_REFS, + TWO_LONGS_AND_BYTES_REFS } + ) public String grouping; @Param({ COUNT, COUNT_DISTINCT, MIN, MAX, SUM }) @@ -117,10 +148,14 @@ public class AggregatorBenchmark { @Param({ VECTOR_LONGS, HALF_NULL_LONGS, VECTOR_DOUBLES, HALF_NULL_DOUBLES }) public String blockType; - private static Operator operator(DriverContext driverContext, String grouping, String op, String dataType) { + @Param({ NONE, CONSTANT_TRUE, ALL_TRUE, HALF_TRUE, CONSTANT_FALSE }) + public String filter; + + private static Operator operator(DriverContext driverContext, String grouping, String op, String dataType, String filter) { + if (grouping.equals("none")) { return new AggregationOperator( - List.of(supplier(op, dataType, 0).aggregatorFactory(AggregatorMode.SINGLE).apply(driverContext)), + List.of(supplier(op, dataType, filter, 0).aggregatorFactory(AggregatorMode.SINGLE).apply(driverContext)), driverContext ); } @@ -129,8 +164,12 @@ private static Operator operator(DriverContext driverContext, String grouping, S case INTS -> List.of(new BlockHash.GroupSpec(0, ElementType.INT)); case DOUBLES -> List.of(new BlockHash.GroupSpec(0, ElementType.DOUBLE)); case BOOLEANS -> List.of(new BlockHash.GroupSpec(0, ElementType.BOOLEAN)); - case BYTES_REFS -> List.of(new BlockHash.GroupSpec(0, ElementType.BYTES_REF)); + case BYTES_REFS, ORDINALS -> List.of(new BlockHash.GroupSpec(0, ElementType.BYTES_REF)); case TWO_LONGS -> List.of(new BlockHash.GroupSpec(0, ElementType.LONG), new BlockHash.GroupSpec(1, ElementType.LONG)); + case TWO_BYTES_REFS, TWO_ORDINALS -> List.of( + new BlockHash.GroupSpec(0, ElementType.BYTES_REF), + new BlockHash.GroupSpec(1, ElementType.BYTES_REF) + ); case LONGS_AND_BYTES_REFS -> List.of( new BlockHash.GroupSpec(0, ElementType.LONG), new BlockHash.GroupSpec(1, ElementType.BYTES_REF) @@ -143,14 +182,14 @@ private static Operator operator(DriverContext driverContext, String grouping, S default -> throw new IllegalArgumentException("unsupported grouping [" + grouping + "]"); }; return new HashAggregationOperator( - List.of(supplier(op, dataType, groups.size()).groupingAggregatorFactory(AggregatorMode.SINGLE)), + List.of(supplier(op, dataType, filter, groups.size()).groupingAggregatorFactory(AggregatorMode.SINGLE)), () -> BlockHash.build(groups, driverContext.blockFactory(), 16 * 1024, false), driverContext ); } - private static AggregatorFunctionSupplier supplier(String op, String dataType, int dataChannel) { - return switch (op) { + private static AggregatorFunctionSupplier supplier(String op, String dataType, String filter, int dataChannel) { + return filtered(switch (op) { case COUNT -> CountAggregatorFunction.supplier(List.of(dataChannel)); case COUNT_DISTINCT -> switch (dataType) { case LONGS -> new CountDistinctLongAggregatorFunctionSupplier(List.of(dataChannel), 3000); @@ -173,10 +212,22 @@ private static AggregatorFunctionSupplier supplier(String op, String dataType, i default -> throw new IllegalArgumentException("unsupported data type [" + dataType + "]"); }; default -> throw new IllegalArgumentException("unsupported op [" + op + "]"); - }; + }, filter); } - private static void checkExpected(String grouping, String op, String blockType, String dataType, Page page, int opCount) { + private static void checkExpected( + String grouping, + String op, + String blockType, + String filter, + String dataType, + Page page, + int opCount + ) { + if (filter.equals(CONSTANT_FALSE) || filter.equals(HALF_TRUE)) { + // We don't verify these because it's hard to get the right answer. + return; + } String prefix = String.format("[%s][%s][%s] ", grouping, op, blockType); if (grouping.equals("none")) { checkUngrouped(prefix, op, dataType, page, opCount); @@ -191,6 +242,10 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri checkGroupingBlock(prefix, LONGS, page.getBlock(0)); checkGroupingBlock(prefix, LONGS, page.getBlock(1)); } + case TWO_BYTES_REFS, TWO_ORDINALS -> { + checkGroupingBlock(prefix, BYTES_REFS, page.getBlock(0)); + checkGroupingBlock(prefix, BYTES_REFS, page.getBlock(1)); + } case LONGS_AND_BYTES_REFS -> { checkGroupingBlock(prefix, LONGS, page.getBlock(0)); checkGroupingBlock(prefix, BYTES_REFS, page.getBlock(1)); @@ -352,7 +407,7 @@ private static void checkGroupingBlock(String prefix, String grouping, Block blo throw new AssertionError(prefix + "bad group expected [true] but was [" + groups.getBoolean(1) + "]"); } } - case BYTES_REFS -> { + case BYTES_REFS, ORDINALS -> { BytesRefBlock groups = (BytesRefBlock) block; for (int g = 0; g < GROUPS; g++) { if (false == groups.getBytesRef(g, new BytesRef()).equals(bytesGroup(g))) { @@ -481,6 +536,8 @@ private static Block dataBlock(BlockFactory blockFactory, String blockType) { private static List groupingBlocks(String grouping, String blockType) { return switch (grouping) { case TWO_LONGS -> List.of(groupingBlock(LONGS, blockType), groupingBlock(LONGS, blockType)); + case TWO_BYTES_REFS -> List.of(groupingBlock(BYTES_REFS, blockType), groupingBlock(BYTES_REFS, blockType)); + case TWO_ORDINALS -> List.of(groupingBlock(ORDINALS, blockType), groupingBlock(ORDINALS, blockType)); case LONGS_AND_BYTES_REFS -> List.of(groupingBlock(LONGS, blockType), groupingBlock(BYTES_REFS, blockType)); case TWO_LONGS_AND_BYTES_REFS -> List.of( groupingBlock(LONGS, blockType), @@ -543,6 +600,19 @@ private static Block groupingBlock(String grouping, String blockType) { } yield builder.build(); } + case ORDINALS -> { + IntVector.Builder ordinals = blockFactory.newIntVectorBuilder(BLOCK_LENGTH * valuesPerGroup); + for (int i = 0; i < BLOCK_LENGTH; i++) { + for (int v = 0; v < valuesPerGroup; v++) { + ordinals.appendInt(i % GROUPS); + } + } + BytesRefVector.Builder bytes = blockFactory.newBytesRefVectorBuilder(BLOCK_LENGTH * valuesPerGroup); + for (int i = 0; i < GROUPS; i++) { + bytes.appendBytesRef(bytesGroup(i)); + } + yield new OrdinalBytesRefVector(ordinals.build(), bytes.build()).asBlock(); + } default -> throw new UnsupportedOperationException("unsupported grouping [" + grouping + "]"); }; } @@ -558,13 +628,59 @@ private static BytesRef bytesGroup(int group) { }); } + private static AggregatorFunctionSupplier filtered(AggregatorFunctionSupplier agg, String filter) { + if (filter.equals("none")) { + return agg; + } + BooleanBlock mask = mask(filter).asBlock(); + return new FilteredAggregatorFunctionSupplier(agg, context -> new EvalOperator.ExpressionEvaluator() { + @Override + public Block eval(Page page) { + mask.incRef(); + return mask; + } + + @Override + public void close() { + mask.close(); + } + }); + } + + private static BooleanVector mask(String filter) { + // Usually BLOCK_LENGTH is the count of positions, but sometimes the blocks are longer + int positionCount = BLOCK_LENGTH * 10; + return switch (filter) { + case CONSTANT_TRUE -> blockFactory.newConstantBooleanVector(true, positionCount); + case ALL_TRUE -> { + try (BooleanVector.Builder builder = blockFactory.newBooleanVectorFixedBuilder(positionCount)) { + for (int i = 0; i < positionCount; i++) { + builder.appendBoolean(true); + } + yield builder.build(); + } + } + case HALF_TRUE -> { + try (BooleanVector.Builder builder = blockFactory.newBooleanVectorFixedBuilder(positionCount)) { + for (int i = 0; i < positionCount; i++) { + builder.appendBoolean(i % 2 == 0); + } + yield builder.build(); + } + } + case CONSTANT_FALSE -> blockFactory.newConstantBooleanVector(false, positionCount); + default -> throw new IllegalArgumentException("unsupported filter [" + filter + "]"); + }; + } + @Benchmark @OperationsPerInvocation(OP_COUNT * BLOCK_LENGTH) public void run() { - run(grouping, op, blockType, OP_COUNT); + run(grouping, op, blockType, filter, OP_COUNT); } - private static void run(String grouping, String op, String blockType, int opCount) { + private static void run(String grouping, String op, String blockType, String filter, int opCount) { + // System.err.printf("[%s][%s][%s][%s][%s]\n", grouping, op, blockType, filter, opCount); String dataType = switch (blockType) { case VECTOR_LONGS, HALF_NULL_LONGS -> LONGS; case VECTOR_DOUBLES, HALF_NULL_DOUBLES -> DOUBLES; @@ -572,13 +688,13 @@ private static void run(String grouping, String op, String blockType, int opCoun }; DriverContext driverContext = driverContext(); - try (Operator operator = operator(driverContext, grouping, op, dataType)) { + try (Operator operator = operator(driverContext, grouping, op, dataType, filter)) { Page page = page(driverContext.blockFactory(), grouping, blockType); for (int i = 0; i < opCount; i++) { operator.addInput(page.shallowCopy()); } operator.finish(); - checkExpected(grouping, op, blockType, dataType, operator.getOutput(), opCount); + checkExpected(grouping, op, blockType, filter, dataType, operator.getOutput(), opCount); } } diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockBenchmark.java index 59fdfff3025a1..71d858f370c6a 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.compute.operator; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockKeepMaskBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockKeepMaskBenchmark.java index 23048ad188a37..5b9643d697686 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockKeepMaskBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockKeepMaskBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.compute.operator; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockReadBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockReadBenchmark.java index 327dcfcff3a28..76fc7c0c0899c 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockReadBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockReadBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.compute.operator; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java index 7335dfbd8f239..9aab4a3e3210f 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.compute.operator; @@ -24,6 +25,7 @@ import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.compute.operator.Operator; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern; @@ -31,6 +33,7 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.evaluator.EvalMapper; +import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case; import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateTrunc; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Abs; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin; @@ -52,6 +55,7 @@ import java.time.Duration; import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -90,6 +94,8 @@ public class EvalBenchmark { "abs", "add", "add_double", + "case_1_eager", + "case_1_lazy", "date_trunc", "equal_to_const", "long_equal_to_long", @@ -124,6 +130,18 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) { layout(doubleField) ).get(driverContext); } + case "case_1_eager", "case_1_lazy" -> { + FieldAttribute f1 = longField(); + FieldAttribute f2 = longField(); + Expression condition = new Equals(Source.EMPTY, f1, new Literal(Source.EMPTY, 1L, DataType.LONG)); + Expression lhs = f1; + Expression rhs = f2; + if (operation.endsWith("lazy")) { + lhs = new Add(Source.EMPTY, lhs, new Literal(Source.EMPTY, 1L, DataType.LONG)); + rhs = new Add(Source.EMPTY, rhs, new Literal(Source.EMPTY, 1L, DataType.LONG)); + } + yield EvalMapper.toEvaluator(new Case(Source.EMPTY, condition, List.of(lhs, rhs)), layout(f1, f2)).get(driverContext); + } case "date_trunc" -> { FieldAttribute timestamp = new FieldAttribute( Source.EMPTY, @@ -215,6 +233,28 @@ private static void checkExpected(String operation, Page actual) { } } } + case "case_1_eager" -> { + LongVector f1 = actual.getBlock(0).asVector(); + LongVector f2 = actual.getBlock(1).asVector(); + LongVector result = actual.getBlock(2).asVector(); + for (int i = 0; i < BLOCK_LENGTH; i++) { + long expected = f1.getLong(i) == 1 ? f1.getLong(i) : f2.getLong(i); + if (result.getLong(i) != expected) { + throw new AssertionError("[" + operation + "] expected [" + expected + "] but was [" + result.getLong(i) + "]"); + } + } + } + case "case_1_lazy" -> { + LongVector f1 = actual.getBlock(0).asVector(); + LongVector f2 = actual.getBlock(1).asVector(); + LongVector result = actual.getBlock(2).asVector(); + for (int i = 0; i < BLOCK_LENGTH; i++) { + long expected = 1 + (f1.getLong(i) == 1 ? f1.getLong(i) : f2.getLong(i)); + if (result.getLong(i) != expected) { + throw new AssertionError("[" + operation + "] expected [" + expected + "] but was [" + result.getLong(i) + "]"); + } + } + } case "date_trunc" -> { LongVector v = actual.getBlock(1).asVector(); long oneDay = TimeValue.timeValueHours(24).millis(); @@ -279,6 +319,15 @@ private static Page page(String operation) { } yield new Page(builder.build()); } + case "case_1_eager", "case_1_lazy" -> { + var f1 = blockFactory.newLongBlockBuilder(BLOCK_LENGTH); + var f2 = blockFactory.newLongBlockBuilder(BLOCK_LENGTH); + for (int i = 0; i < BLOCK_LENGTH; i++) { + f1.appendLong(i); + f2.appendLong(-i); + } + yield new Page(f1.build(), f2.build()); + } case "long_equal_to_long" -> { var lhs = blockFactory.newLongBlockBuilder(BLOCK_LENGTH); var rhs = blockFactory.newLongBlockBuilder(BLOCK_LENGTH); diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/MultivalueDedupeBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/MultivalueDedupeBenchmark.java index 744da521b96c3..86721a7eed18c 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/MultivalueDedupeBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/MultivalueDedupeBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.compute.operator; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/TopNBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/TopNBenchmark.java index 3d5a36ea288b4..cbb6fd682bdae 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/TopNBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/TopNBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.compute.operator; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java index 2185c6d1df611..9ed5e1accef59 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.compute.operator; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/fs/AvailableIndexFoldersBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/fs/AvailableIndexFoldersBenchmark.java index cb5a7670ceced..6361981a790e6 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/fs/AvailableIndexFoldersBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/fs/AvailableIndexFoldersBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.fs; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/h3/H3Benchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/h3/H3Benchmark.java index 2441acab7d405..47fd06cdb0e39 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/h3/H3Benchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/h3/H3Benchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.h3; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/h3/H3State.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/h3/H3State.java index 5707e692a0750..b7ad281c637e5 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/h3/H3State.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/h3/H3State.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.h3; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/DecodeConstantIntegerBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/DecodeConstantIntegerBenchmark.java index 0bfd58954d43f..4ade83728b73b 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/DecodeConstantIntegerBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/DecodeConstantIntegerBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.index.codec.tsdb; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/DecodeDecreasingIntegerBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/DecodeDecreasingIntegerBenchmark.java index 3411124096651..f41ae3a912a94 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/DecodeDecreasingIntegerBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/DecodeDecreasingIntegerBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.index.codec.tsdb; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/DecodeIncreasingIntegerBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/DecodeIncreasingIntegerBenchmark.java index e73fb627957e3..40d0d41b4aefc 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/DecodeIncreasingIntegerBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/DecodeIncreasingIntegerBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.index.codec.tsdb; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/DecodeNonSortedIntegerBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/DecodeNonSortedIntegerBenchmark.java index 4fc38cab24b18..7eea2d9d7a70e 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/DecodeNonSortedIntegerBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/DecodeNonSortedIntegerBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.index.codec.tsdb; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/EncodeConstantIntegerBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/EncodeConstantIntegerBenchmark.java index 93f8dbf27347e..47579b3acd410 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/EncodeConstantIntegerBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/EncodeConstantIntegerBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.index.codec.tsdb; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/EncodeDecreasingIntegerBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/EncodeDecreasingIntegerBenchmark.java index 2f736cb7a21b4..73d0e13a76a77 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/EncodeDecreasingIntegerBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/EncodeDecreasingIntegerBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.index.codec.tsdb; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/EncodeIncreasingIntegerBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/EncodeIncreasingIntegerBenchmark.java index 8a4a990d0d375..f9e164972bebb 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/EncodeIncreasingIntegerBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/EncodeIncreasingIntegerBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.index.codec.tsdb; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/EncodeNonSortedIntegerBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/EncodeNonSortedIntegerBenchmark.java index 39f63e5644158..f195228a723e5 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/EncodeNonSortedIntegerBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/EncodeNonSortedIntegerBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.index.codec.tsdb; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/AbstractDocValuesForUtilBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/AbstractDocValuesForUtilBenchmark.java index ed033b962c688..53723f05728b5 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/AbstractDocValuesForUtilBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/AbstractDocValuesForUtilBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.index.codec.tsdb.internal; @@ -20,7 +21,7 @@ public abstract class AbstractDocValuesForUtilBenchmark { protected final int blockSize; public AbstractDocValuesForUtilBenchmark() { - this.forUtil = new DocValuesForUtil(); + this.forUtil = new DocValuesForUtil(ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE); this.blockSize = ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE; } diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/AbstractLongArraySupplier.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/AbstractLongArraySupplier.java index 300363d8446b3..e03563743b803 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/AbstractLongArraySupplier.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/AbstractLongArraySupplier.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.index.codec.tsdb.internal; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/ConstantIntegerSupplier.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/ConstantIntegerSupplier.java index 43e94c76ae54a..34acd82d812c8 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/ConstantIntegerSupplier.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/ConstantIntegerSupplier.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.index.codec.tsdb.internal; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/DecodeBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/DecodeBenchmark.java index 0159c35450fe7..284324b3d9206 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/DecodeBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/DecodeBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.index.codec.tsdb.internal; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/DecreasingIntegerSupplier.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/DecreasingIntegerSupplier.java index 7e4ed87896a46..7813260298bef 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/DecreasingIntegerSupplier.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/DecreasingIntegerSupplier.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.index.codec.tsdb.internal; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/EncodeBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/EncodeBenchmark.java index cba6041b7184c..9b0dba1cc6038 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/EncodeBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/EncodeBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.index.codec.tsdb.internal; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/IncreasingIntegerSupplier.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/IncreasingIntegerSupplier.java index 191d713f8bd19..ed3c433d548fb 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/IncreasingIntegerSupplier.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/IncreasingIntegerSupplier.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.index.codec.tsdb.internal; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/NonSortedIntegerSupplier.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/NonSortedIntegerSupplier.java index a330866e8489f..d8ffa07d4e252 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/NonSortedIntegerSupplier.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/NonSortedIntegerSupplier.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.index.codec.tsdb.internal; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/BeatsMapperBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/BeatsMapperBenchmark.java index bcedfeb9b693a..09ae18c34b052 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/BeatsMapperBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/BeatsMapperBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.index.mapper; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/DynamicMapperBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/DynamicMapperBenchmark.java index 2d042977cc4e7..c7c4557f19ff9 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/DynamicMapperBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/DynamicMapperBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.index.mapper; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/KeywordFieldMapperBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/KeywordFieldMapperBenchmark.java index 7a53b19ff2791..9026b473e0552 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/KeywordFieldMapperBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/KeywordFieldMapperBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.index.mapper; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java index 68b31481e17f3..74cea5d5f1549 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java @@ -1,15 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.index.mapper; import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.util.Accountable; import org.elasticsearch.TransportVersion; import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -27,7 +27,6 @@ import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ProvidedIdFieldMapper; -import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.script.Script; @@ -55,13 +54,7 @@ public static MapperService create(String mappings) { MapperRegistry mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry(); SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of()); - BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetFilterCache.Listener() { - @Override - public void onCache(ShardId shardId, Accountable accountable) {} - - @Override - public void onRemoval(ShardId shardId, Accountable accountable) {} - }); + BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, BitsetFilterCache.Listener.NOOP); MapperService mapperService = new MapperService( () -> TransportVersion.current(), indexSettings, diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/indices/breaker/MemoryStatsBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/indices/breaker/MemoryStatsBenchmark.java index 889d3107e2bb3..ae23d9fd8c120 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/indices/breaker/MemoryStatsBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/indices/breaker/MemoryStatsBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.indices.breaker; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/indices/common/RoundingBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/indices/common/RoundingBenchmark.java index 7ee8c33b2fe8e..6d16b1312883b 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/indices/common/RoundingBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/indices/common/RoundingBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.indices.common; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/indices/resolution/IndexNameExpressionResolverBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/indices/resolution/IndexNameExpressionResolverBenchmark.java new file mode 100644 index 0000000000000..13a222b1ed35e --- /dev/null +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/indices/resolution/IndexNameExpressionResolverBenchmark.java @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.benchmark.indices.resolution; + +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.indices.SystemIndices; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +@State(Scope.Benchmark) +@Fork(3) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@SuppressWarnings("unused") // invoked by benchmarking framework +public class IndexNameExpressionResolverBenchmark { + + private static final String DATA_STREAM_PREFIX = "my-ds-"; + private static final String INDEX_PREFIX = "my-index-"; + + @Param( + { + // # data streams | # indices + " 1000| 100", + " 5000| 500", + " 10000| 1000" } + ) + public String resourceMix = "100|10"; + + @Setup + public void setUp() { + final String[] params = resourceMix.split("\\|"); + + int numDataStreams = toInt(params[0]); + int numIndices = toInt(params[1]); + + Metadata.Builder mb = Metadata.builder(); + String[] indices = new String[numIndices + numDataStreams * (numIndices + 1)]; + int position = 0; + for (int i = 1; i <= numIndices; i++) { + String indexName = INDEX_PREFIX + i; + createIndexMetadata(indexName, mb); + indices[position++] = indexName; + } + + for (int i = 1; i <= numDataStreams; i++) { + String dataStreamName = DATA_STREAM_PREFIX + i; + List backingIndices = new ArrayList<>(); + for (int j = 1; j <= numIndices; j++) { + String backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, j); + backingIndices.add(createIndexMetadata(backingIndexName, mb).getIndex()); + indices[position++] = backingIndexName; + } + indices[position++] = dataStreamName; + mb.put(DataStream.builder(dataStreamName, backingIndices).build()); + } + int mid = indices.length / 2; + clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(mb).build(); + resolver = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY), new SystemIndices(List.of())); + indexListRequest = new Request(IndicesOptions.lenientExpandOpenHidden(), indices); + starRequest = new Request(IndicesOptions.lenientExpandOpenHidden(), "*"); + String[] mixed = indices.clone(); + mixed[mid] = "my-*"; + mixedRequest = new Request(IndicesOptions.lenientExpandOpenHidden(), mixed); + } + + private IndexMetadata createIndexMetadata(String indexName, Metadata.Builder mb) { + IndexMetadata indexMetadata = IndexMetadata.builder(indexName) + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + mb.put(indexMetadata, false); + return indexMetadata; + } + + private IndexNameExpressionResolver resolver; + private ClusterState clusterState; + private Request starRequest; + private Request indexListRequest; + private Request mixedRequest; + + @Benchmark + public String[] resolveResourcesListToConcreteIndices() { + return resolver.concreteIndexNames(clusterState, indexListRequest); + } + + @Benchmark + public String[] resolveAllStarToConcreteIndices() { + return resolver.concreteIndexNames(clusterState, starRequest); + } + + @Benchmark + public String[] resolveMixedConcreteIndices() { + return resolver.concreteIndexNames(clusterState, mixedRequest); + } + + private int toInt(String v) { + return Integer.parseInt(v.trim()); + } + + record Request(IndicesOptions indicesOptions, String... indices) implements IndicesRequest { + + } +} diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java index a9338d5660b28..70183b7728f45 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.routing.allocation; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java index 387d2355437fe..be1ba93f91c36 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.routing.allocation; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java index d7a72615f4b93..4f2914716b58f 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.routing.allocation; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/TestShardRoutingRoleStrategies.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/TestShardRoutingRoleStrategies.java index 2c62072855957..e3ca434824657 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/TestShardRoutingRoleStrategies.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/TestShardRoutingRoleStrategies.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.routing.allocation; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java index fe221ec980dc3..b44f04c3a26a4 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.script; @@ -33,6 +34,7 @@ import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.plugins.PluginsLoader; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.script.DocReader; @@ -75,8 +77,7 @@ public class ScriptScoreBenchmark { private final PluginsService pluginsService = new PluginsService( Settings.EMPTY, null, - null, - Path.of(System.getProperty("plugins.dir")) + PluginsLoader.createPluginsLoader(null, Path.of(System.getProperty("plugins.dir"))) ); private final ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, pluginsService.filterPlugins(ScriptPlugin.class).toList()); diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java index cff15d9c36d34..e4bfdfa1749e9 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.search; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java index 8e60a7435cbc7..671564fcd424e 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.search.aggregations; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/TermsReduceBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/TermsReduceBenchmark.java index 691874c775302..9fd319f9e9b1c 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/TermsReduceBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/TermsReduceBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.search.aggregations; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/bucket/terms/LongKeyedBucketOrdsBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/bucket/terms/LongKeyedBucketOrdsBenchmark.java index 38cebcd42e62d..f8faa1bc25d84 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/bucket/terms/LongKeyedBucketOrdsBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/bucket/terms/LongKeyedBucketOrdsBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.search.aggregations.bucket.terms; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/bucket/terms/StringTermsSerializationBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/bucket/terms/StringTermsSerializationBenchmark.java index a5786b75e05cf..6065dedc8de4f 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/bucket/terms/StringTermsSerializationBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/bucket/terms/StringTermsSerializationBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.search.aggregations.bucket.terms; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java index 848ee6e556dc1..55b8c18138f46 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java @@ -63,7 +63,7 @@ public void setup() throws IOException { ); includesSet = Set.of(fetchContext.includes()); excludesSet = Set.of(fetchContext.excludes()); - parserConfig = XContentParserConfiguration.EMPTY.withFiltering(includesSet, excludesSet, false); + parserConfig = XContentParserConfiguration.EMPTY.withFiltering(null, includesSet, excludesSet, false); } private BytesReference read300BytesExample() throws IOException { diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/SourceFilteringBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/SourceFilteringBenchmark.java index 2f8a9eb8836dc..7f569246c1614 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/SourceFilteringBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/SourceFilteringBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.search.fetch.subphase; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/spatial/GeometrySimplificationBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/spatial/GeometrySimplificationBenchmark.java index e744b34ed764d..7ba23dda9d71c 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/spatial/GeometrySimplificationBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/spatial/GeometrySimplificationBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.spatial; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/SortBench.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/SortBench.java index 862479571113d..4bec6a183fe94 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/SortBench.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/SortBench.java @@ -21,7 +21,12 @@ package org.elasticsearch.benchmark.tdigest; +import org.elasticsearch.common.breaker.NoopCircuitBreaker; +import org.elasticsearch.search.aggregations.metrics.MemoryTrackingTDigestArrays; import org.elasticsearch.tdigest.Sort; +import org.elasticsearch.tdigest.arrays.TDigestArrays; +import org.elasticsearch.tdigest.arrays.TDigestDoubleArray; +import org.elasticsearch.tdigest.arrays.TDigestIntArray; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -35,7 +40,6 @@ import org.openjdk.jmh.annotations.Threads; import org.openjdk.jmh.annotations.Warmup; -import java.util.Arrays; import java.util.Random; import java.util.concurrent.TimeUnit; @@ -49,7 +53,8 @@ @State(Scope.Thread) public class SortBench { private final int size = 100000; - private final double[] values = new double[size]; + private final TDigestArrays arrays = new MemoryTrackingTDigestArrays(new NoopCircuitBreaker("default-wrapper-tdigest-arrays")); + private final TDigestDoubleArray values = arrays.newDoubleArray(size); @Param({ "0", "1", "-1" }) public int sortDirection; @@ -58,22 +63,22 @@ public class SortBench { public void setup() { Random prng = new Random(999983); for (int i = 0; i < size; i++) { - values[i] = prng.nextDouble(); + values.set(i, prng.nextDouble()); } if (sortDirection > 0) { - Arrays.sort(values); + values.sort(); } else if (sortDirection < 0) { - Arrays.sort(values); - Sort.reverse(values, 0, values.length); + values.sort(); + Sort.reverse(values, 0, values.size()); } } @Benchmark - public void quicksort() { - int[] order = new int[size]; + public void stableSort() { + TDigestIntArray order = arrays.newIntArray(size); for (int i = 0; i < size; i++) { - order[i] = i; + order.set(i, i); } - Sort.sort(order, values, null, values.length); + Sort.stableSort(order, values, values.size()); } } diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/TDigestBench.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/TDigestBench.java index abf08395f90a6..7a5aaa0187533 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/TDigestBench.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/TDigestBench.java @@ -21,9 +21,10 @@ package org.elasticsearch.benchmark.tdigest; -import org.elasticsearch.tdigest.AVLTreeDigest; -import org.elasticsearch.tdigest.MergingDigest; +import org.elasticsearch.common.breaker.NoopCircuitBreaker; +import org.elasticsearch.search.aggregations.metrics.MemoryTrackingTDigestArrays; import org.elasticsearch.tdigest.TDigest; +import org.elasticsearch.tdigest.arrays.TDigestArrays; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -56,18 +57,25 @@ @Threads(1) @State(Scope.Thread) public class TDigestBench { + private static final TDigestArrays arrays = new MemoryTrackingTDigestArrays(new NoopCircuitBreaker("default-wrapper-tdigest-arrays")); public enum TDigestFactory { MERGE { @Override TDigest create(double compression) { - return new MergingDigest(compression, (int) (10 * compression)); + return TDigest.createMergingDigest(arrays, compression); } }, AVL_TREE { @Override TDigest create(double compression) { - return new AVLTreeDigest(compression); + return TDigest.createAvlTreeDigest(arrays, compression); + } + }, + HYBRID { + @Override + TDigest create(double compression) { + return TDigest.createHybridDigest(arrays, compression); } }; @@ -77,7 +85,7 @@ TDigest create(double compression) { @Param({ "100", "300" }) double compression; - @Param({ "MERGE", "AVL_TREE" }) + @Param({ "MERGE", "AVL_TREE", "HYBRID" }) TDigestFactory tdigestFactory; @Param({ "NORMAL", "GAUSSIAN" }) diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/time/DateFormatterFromBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/time/DateFormatterFromBenchmark.java index 0f180972c3117..12a9b577a5738 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/time/DateFormatterFromBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/time/DateFormatterFromBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.time; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/DistanceFunctionBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/DistanceFunctionBenchmark.java index 0a4c836e2a6cf..187d300979476 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/DistanceFunctionBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/DistanceFunctionBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.vector; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/VectorScorerBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/VectorScorerBenchmark.java index 89b512920cb09..569e8909e1e12 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/VectorScorerBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/VectorScorerBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.vector; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/FilterContentBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/FilterContentBenchmark.java index c869e5f677918..aa9236e9f314f 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/FilterContentBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/FilterContentBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.benchmark.xcontent; @@ -169,7 +170,7 @@ private XContentParserConfiguration buildParseConfig(boolean matchDotsInFieldNam includes = null; excludes = filters; } - return XContentParserConfiguration.EMPTY.withFiltering(includes, excludes, matchDotsInFieldNames); + return XContentParserConfiguration.EMPTY.withFiltering(null, includes, excludes, matchDotsInFieldNames); } private BytesReference filter(XContentParserConfiguration contentParserConfiguration) throws IOException { diff --git a/branches.json b/branches.json index 1d860501cbc87..95fbdb1efd655 100644 --- a/branches.json +++ b/branches.json @@ -5,7 +5,13 @@ "branch": "main" }, { - "branch": "8.15" + "branch": "8.16" + }, + { + "branch": "8.17" + }, + { + "branch": "8.x" }, { "branch": "7.17" diff --git a/build-conventions/build.gradle b/build-conventions/build.gradle index 94b0312d0d5d3..b0eda5a34065a 100644 --- a/build-conventions/build.gradle +++ b/build-conventions/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.gradle.plugins.ide.eclipse.model.SourceFolder @@ -11,9 +12,6 @@ import org.gradle.plugins.ide.eclipse.model.SourceFolder buildscript { repositories { - maven { - url 'https://jitpack.io' - } mavenCentral() } } @@ -69,10 +67,6 @@ gradlePlugin { } repositories { - maven { - url 'https://jitpack.io' - } - mavenCentral() gradlePluginPortal() } diff --git a/build-conventions/settings.gradle b/build-conventions/settings.gradle index ae3211a4c4543..3256de7d56286 100644 --- a/build-conventions/settings.gradle +++ b/build-conventions/settings.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ rootProject.name = 'build-conventions' @@ -13,4 +14,4 @@ dependencyResolutionManagement { from(files("../gradle/build.versions.toml")) } } -} \ No newline at end of file +} diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/checkstyle/AstUtils.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/checkstyle/AstUtils.java index 5b7af1902ebb5..ef571e32a502d 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/checkstyle/AstUtils.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/checkstyle/AstUtils.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.checkstyle; diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/checkstyle/SnippetLengthCheck.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/checkstyle/SnippetLengthCheck.java index 7053bd1626153..32f28c4b7f6e8 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/checkstyle/SnippetLengthCheck.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/checkstyle/SnippetLengthCheck.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.checkstyle; diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/checkstyle/StringFormattingCheck.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/checkstyle/StringFormattingCheck.java index 48fa3ad6ee485..ca46ffc5f846c 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/checkstyle/StringFormattingCheck.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/checkstyle/StringFormattingCheck.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.checkstyle; diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/checkstyle/SwitchBetweenCheck.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/checkstyle/SwitchBetweenCheck.java index 7957b65cc0e7e..4c263d7ed95b0 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/checkstyle/SwitchBetweenCheck.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/checkstyle/SwitchBetweenCheck.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.checkstyle; diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/BuildToolsConventionsPlugin.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/BuildToolsConventionsPlugin.java index afadede56f240..e873c7e93e280 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/BuildToolsConventionsPlugin.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/BuildToolsConventionsPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.conventions; diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/EclipseConventionPlugin.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/EclipseConventionPlugin.java index fec7cba313350..58b183fac3155 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/EclipseConventionPlugin.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/EclipseConventionPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.conventions; diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/GUtils.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/GUtils.java index fdfa6d942ab0b..0b04496866ca9 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/GUtils.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/GUtils.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.conventions; @@ -15,4 +16,12 @@ public abstract class GUtils { public static String capitalize(String s) { return s.substring(0, 1).toUpperCase(Locale.ROOT) + s.substring(1); } + + public static T elvis(T given, T fallback) { + if (given == null) { + return fallback; + } else { + return given; + } + } } diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/GitInfoPlugin.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/GitInfoPlugin.java index a0fb8ee0504a6..6f64edfc8c0dc 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/GitInfoPlugin.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/GitInfoPlugin.java @@ -1,55 +1,54 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.conventions; import org.elasticsearch.gradle.internal.conventions.info.GitInfo; +import org.elasticsearch.gradle.internal.conventions.info.GitInfoValueSource; import org.elasticsearch.gradle.internal.conventions.util.Util; import org.gradle.api.Plugin; import org.gradle.api.Project; -import org.gradle.api.model.ObjectFactory; import org.gradle.api.provider.Property; import org.gradle.api.provider.Provider; import org.gradle.api.provider.ProviderFactory; -import javax.inject.Inject; import java.io.File; -class GitInfoPlugin implements Plugin { +import javax.inject.Inject; - private ProviderFactory factory; - private ObjectFactory objectFactory; +public abstract class GitInfoPlugin implements Plugin { + private ProviderFactory factory; private Provider revision; - private Property gitInfo; @Inject - GitInfoPlugin(ProviderFactory factory, ObjectFactory objectFactory) { + public GitInfoPlugin(ProviderFactory factory) { this.factory = factory; - this.objectFactory = objectFactory; } @Override public void apply(Project project) { - File rootDir = Util.locateElasticsearchWorkspace(project.getGradle()); - gitInfo = objectFactory.property(GitInfo.class).value(factory.provider(() -> - GitInfo.gitInfo(rootDir) - )); - gitInfo.disallowChanges(); - gitInfo.finalizeValueOnRead(); - - revision = gitInfo.map(info -> info.getRevision() == null ? info.getRevision() : "master"); + File rootDir = getGitRootDir(project); + getGitInfo().convention(factory.of(GitInfoValueSource.class, spec -> { spec.getParameters().getPath().set(rootDir); })); + revision = getGitInfo().map(info -> info.getRevision() == null ? info.getRevision() : "main"); } - public Property getGitInfo() { - return gitInfo; + private static File getGitRootDir(Project project) { + File rootDir = project.getRootDir(); + if (new File(rootDir, ".git").exists()) { + return rootDir; + } + return Util.locateElasticsearchWorkspace(project.getGradle()); } + public abstract Property getGitInfo(); + public Provider getRevision() { return revision; } diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/LicensingPlugin.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/LicensingPlugin.java index d0c58fc8ed0cf..63514ae671bf3 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/LicensingPlugin.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/LicensingPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.conventions; @@ -14,12 +15,14 @@ import org.gradle.api.provider.Provider; import org.gradle.api.provider.ProviderFactory; -import javax.inject.Inject; import java.util.Map; +import javax.inject.Inject; + public class LicensingPlugin implements Plugin { static final String ELASTIC_LICENSE_URL_PREFIX = "https://raw.githubusercontent.com/elastic/elasticsearch/"; static final String ELASTIC_LICENSE_URL_POSTFIX = "/licenses/ELASTIC-LICENSE-2.0.txt"; + static final String AGPL_ELASTIC_LICENSE_URL_POSTFIX = "/licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt"; private ProviderFactory providerFactory; @@ -31,21 +34,33 @@ public LicensingPlugin(ProviderFactory providerFactory) { @Override public void apply(Project project) { Provider revision = project.getRootProject().getPlugins().apply(GitInfoPlugin.class).getRevision(); - Provider licenseCommitProvider = providerFactory.provider(() -> - isSnapshotVersion(project) ? revision.get() : "v" + project.getVersion() + Provider licenseCommitProvider = providerFactory.provider( + () -> isSnapshotVersion(project) ? revision.get() : "v" + project.getVersion() ); - Provider projectLicenseURL = licenseCommitProvider.map(licenseCommit -> ELASTIC_LICENSE_URL_PREFIX + - licenseCommit + ELASTIC_LICENSE_URL_POSTFIX); + Provider elasticLicenseURL = licenseCommitProvider.map( + licenseCommit -> ELASTIC_LICENSE_URL_PREFIX + licenseCommit + ELASTIC_LICENSE_URL_POSTFIX + ); + Provider agplLicenseURL = licenseCommitProvider.map( + licenseCommit -> ELASTIC_LICENSE_URL_PREFIX + licenseCommit + AGPL_ELASTIC_LICENSE_URL_POSTFIX + ); // But stick the Elastic license url in project.ext so we can get it if we need to switch to it - project.getExtensions().getExtraProperties().set("elasticLicenseUrl", projectLicenseURL); + project.getExtensions().getExtraProperties().set("elasticLicenseUrl", elasticLicenseURL); - MapProperty licensesProperty = project.getObjects().mapProperty(String.class, String.class).convention( - providerFactory.provider(() -> Map.of( - "Server Side Public License, v 1", "https://www.mongodb.com/licensing/server-side-public-license", - "Elastic License 2.0", projectLicenseURL.get()) + MapProperty> licensesProperty = project.getObjects() + .mapProperty(String.class, (Class>) (Class) Provider.class) + .convention( + providerFactory.provider( + () -> Map.of( + "Server Side Public License, v 1", + providerFactory.provider(() -> "https://www.mongodb.com/licensing/server-side-public-license"), + "Elastic License 2.0", + elasticLicenseURL, + "GNU Affero General Public License Version 3", + agplLicenseURL + ) ) - ); + ); // Default to the SSPL+Elastic dual license project.getExtensions().getExtraProperties().set("projectLicenses", licensesProperty); @@ -55,4 +70,4 @@ private boolean isSnapshotVersion(Project project) { return project.getVersion().toString().endsWith("-SNAPSHOT"); } -} \ No newline at end of file +} diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/PublishPlugin.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/PublishPlugin.java index f7ddc300c0f0e..22b0ab1918024 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/PublishPlugin.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/PublishPlugin.java @@ -1,19 +1,22 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.conventions; -import org.elasticsearch.gradle.internal.conventions.precommit.PomValidationPrecommitPlugin; +import groovy.util.Node; + import com.github.jengelman.gradle.plugins.shadow.ShadowExtension; import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin; -import groovy.util.Node; -import org.elasticsearch.gradle.internal.conventions.util.Util; + import org.elasticsearch.gradle.internal.conventions.info.GitInfo; +import org.elasticsearch.gradle.internal.conventions.precommit.PomValidationPrecommitPlugin; +import org.elasticsearch.gradle.internal.conventions.util.Util; import org.gradle.api.NamedDomainObjectSet; import org.gradle.api.Plugin; import org.gradle.api.Project; @@ -25,6 +28,7 @@ import org.gradle.api.plugins.JavaLibraryPlugin; import org.gradle.api.plugins.JavaPlugin; import org.gradle.api.provider.MapProperty; +import org.gradle.api.provider.Provider; import org.gradle.api.provider.ProviderFactory; import org.gradle.api.publish.PublishingExtension; import org.gradle.api.publish.maven.MavenPublication; @@ -34,12 +38,14 @@ import org.gradle.api.tasks.bundling.Jar; import org.gradle.initialization.layout.BuildLayout; import org.gradle.language.base.plugins.LifecycleBasePlugin; +import org.w3c.dom.Element; -import javax.inject.Inject; import java.io.File; import java.util.Map; import java.util.concurrent.Callable; +import javax.inject.Inject; + public class PublishPlugin implements Plugin { private ProjectLayout projectLayout; @@ -63,6 +69,7 @@ public void apply(Project project) { configureSourcesJar(project); configurePomGeneration(project); configurePublications(project); + formatGeneratedPom(project); } private void configurePublications(Project project) { @@ -76,7 +83,7 @@ private void configurePublications(Project project) { } }); @SuppressWarnings("unchecked") - var projectLicenses = (MapProperty) project.getExtensions().getExtraProperties().get("projectLicenses"); + var projectLicenses = (MapProperty>) project.getExtensions().getExtraProperties().get("projectLicenses"); publication.getPom().withXml(xml -> { var node = xml.asNode(); node.appendNode("inceptionYear", "2009"); @@ -84,7 +91,7 @@ private void configurePublications(Project project) { projectLicenses.get().entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(entry -> { Node license = licensesNode.appendNode("license"); license.appendNode("name", entry.getKey()); - license.appendNode("url", entry.getValue()); + license.appendNode("url", entry.getValue().get()); license.appendNode("distribution", "repo"); }); var developer = node.appendNode("developers").appendNode("developer"); @@ -112,29 +119,32 @@ private void configurePomGeneration(Project project) { var archivesBaseName = providerFactory.provider(() -> getArchivesBaseName(extensions)); var projectVersion = providerFactory.provider(() -> project.getVersion()); var generateMavenPoms = project.getTasks().withType(GenerateMavenPom.class); - generateMavenPoms.configureEach( - pomTask -> pomTask.setDestination( + generateMavenPoms.configureEach(pomTask -> { + pomTask.setDestination( (Callable) () -> String.format( "%s/distributions/%s-%s.pom", projectLayout.getBuildDirectory().get().getAsFile().getPath(), archivesBaseName.get(), projectVersion.get() ) - ) - ); + ); + }); + var publishing = extensions.getByType(PublishingExtension.class); final var mavenPublications = publishing.getPublications().withType(MavenPublication.class); - addNameAndDescriptiontoPom(project, mavenPublications); + addNameAndDescriptionToPom(project, mavenPublications); mavenPublications.configureEach(publication -> { - // Add git origin info to generated POM files for internal builds - publication.getPom().withXml(xml -> addScmInfo(xml, gitInfo.get())); + publication.getPom().withXml(xml -> { + // Add git origin info to generated POM files for internal builds + addScmInfo(xml, gitInfo.get()); + }); // have to defer this until archivesBaseName is set project.afterEvaluate(p -> publication.setArtifactId(archivesBaseName.get())); generatePomTask.configure(t -> t.dependsOn(generateMavenPoms)); }); } - private void addNameAndDescriptiontoPom(Project project, NamedDomainObjectSet mavenPublications) { + private void addNameAndDescriptionToPom(Project project, NamedDomainObjectSet mavenPublications) { var name = project.getName(); var description = providerFactory.provider(() -> project.getDescription() != null ? project.getDescription() : ""); mavenPublications.configureEach(p -> p.getPom().withXml(xml -> { @@ -185,4 +195,31 @@ static void configureSourcesJar(Project project) { project.getTasks().named(BasePlugin.ASSEMBLE_TASK_NAME).configure(t -> t.dependsOn(sourcesJarTask)); }); } + + /** + * Format the generated pom files to be in a sort of reproducible order. + */ + private void formatGeneratedPom(Project project) { + var publishing = project.getExtensions().getByType(PublishingExtension.class); + final var mavenPublications = publishing.getPublications().withType(MavenPublication.class); + mavenPublications.configureEach(publication -> { + publication.getPom().withXml(xml -> { + // Add some pom formatting + formatDependencies(xml); + }); + }); + } + + /** + * just ensure we put dependencies to the end. more a cosmetic thing than anything else + * */ + private void formatDependencies(XmlProvider xml) { + Element rootElement = xml.asElement(); + var dependencies = rootElement.getElementsByTagName("dependencies"); + if (dependencies.getLength() == 1 && dependencies.item(0) != null) { + org.w3c.dom.Node item = dependencies.item(0); + rootElement.removeChild(item); + rootElement.appendChild(item); + } + } } diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/VersionPropertiesBuildService.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/VersionPropertiesBuildService.java index adc25dec5e07f..5d7f53e828721 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/VersionPropertiesBuildService.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/VersionPropertiesBuildService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.conventions; diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/VersionPropertiesLoader.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/VersionPropertiesLoader.java index 17842461636c6..a5c00b9b81391 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/VersionPropertiesLoader.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/VersionPropertiesLoader.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.conventions; diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/VersionPropertiesPlugin.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/VersionPropertiesPlugin.java index 76395fafe5e0d..ab29204062318 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/VersionPropertiesPlugin.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/VersionPropertiesPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.conventions; diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/GitInfo.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/GitInfo.java index 17d6ef6917c94..e6a41093205cc 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/GitInfo.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/GitInfo.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.conventions.info; @@ -21,6 +22,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.Map; +import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -189,4 +191,15 @@ public String urlFromOrigin() { } } + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + GitInfo gitInfo = (GitInfo) o; + return Objects.equals(revision, gitInfo.revision) && Objects.equals(origin, gitInfo.origin); + } + + @Override + public int hashCode() { + return Objects.hash(revision, origin); + } } diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/GitInfoValueSource.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/GitInfoValueSource.java new file mode 100644 index 0000000000000..c422c2eb74e39 --- /dev/null +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/GitInfoValueSource.java @@ -0,0 +1,22 @@ +package org.elasticsearch.gradle.internal.conventions.info; + +import org.gradle.api.provider.Property; +import org.gradle.api.provider.ValueSource; +import org.gradle.api.provider.ValueSourceParameters; +import org.jetbrains.annotations.Nullable; + +import java.io.File; + +public abstract class GitInfoValueSource implements ValueSource { + + @Nullable + @Override + public GitInfo obtain() { + File path = getParameters().getPath().get(); + return GitInfo.gitInfo(path); + } + + public interface Parameters extends ValueSourceParameters { + Property getPath(); + } +} diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/ParallelDetector.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/ParallelDetector.java index 688f2f858ae16..fe5a70221aeb8 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/ParallelDetector.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/ParallelDetector.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.conventions.info; diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/FormattingPrecommitPlugin.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/FormattingPrecommitPlugin.java index c83ca4974497d..ea9009172c7e2 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/FormattingPrecommitPlugin.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/FormattingPrecommitPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.conventions.precommit; diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/LicenseHeadersPrecommitPlugin.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/LicenseHeadersPrecommitPlugin.java index 11dca2953b245..5883d2cc5d35f 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/LicenseHeadersPrecommitPlugin.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/LicenseHeadersPrecommitPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.conventions.precommit; diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/LicenseHeadersTask.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/LicenseHeadersTask.java index e49feb4289586..2874f62c5b451 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/LicenseHeadersTask.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/LicenseHeadersTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.conventions.precommit; @@ -66,15 +67,21 @@ public abstract class LicenseHeadersTask extends DefaultTask { private final RegularFileProperty reportFile; private static List conventionalLicenses = Arrays.asList( - // Dual SSPLv1 and Elastic - new License("DUAL", "SSPL+Elastic License", "the Elastic License 2.0 or the Server") + // Triple AGPL, SSPLv1 and Elastic + new License( + "TRIPLE", + "AGLP+SSPL+Elastic License", + "2.0\", the \"GNU Affero General Public License v3.0 only\", and the \"Server Side" + ) ); /** * Allowed license families for this project. */ @Input - private List approvedLicenses = new ArrayList(Arrays.asList("SSPL+Elastic License", "Generated", "Vendored", "Apache LZ4-Java")); + private List approvedLicenses = new ArrayList( + Arrays.asList("AGLP+SSPL+Elastic License", "Generated", "Vendored", "Apache LZ4-Java") + ); /** * Files that should be excluded from the license header check. Use with extreme care, only in situations where the license on the * source file is compatible with the codebase but we do not want to add the license to the list of approved headers (to avoid the @@ -88,9 +95,7 @@ public abstract class LicenseHeadersTask extends DefaultTask { @Inject public LicenseHeadersTask(ObjectFactory objectFactory, ProjectLayout projectLayout) { additionalLicenses = objectFactory.listProperty(License.class).convention(conventionalLicenses); - reportFile = objectFactory.fileProperty().convention( - projectLayout.getBuildDirectory().file("reports/licenseHeaders/rat.xml") - ); + reportFile = objectFactory.fileProperty().convention(projectLayout.getBuildDirectory().file("reports/licenseHeaders/rat.xml")); setDescription("Checks sources for missing, incorrect, or unacceptable license headers"); } diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PomValidationPrecommitPlugin.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PomValidationPrecommitPlugin.java index 8bcb7f46475f9..78afdb6c48ab7 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PomValidationPrecommitPlugin.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PomValidationPrecommitPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.conventions.precommit; diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PomValidationTask.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PomValidationTask.java index 395f7b26a1701..9d06e632ec928 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PomValidationTask.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PomValidationTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.conventions.precommit; diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PrecommitPlugin.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PrecommitPlugin.java index e59ef2bd2b931..d8807265f94fa 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PrecommitPlugin.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PrecommitPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.conventions.precommit; diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PrecommitTask.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PrecommitTask.java index 5cdc291895ffe..8dc951a4c15d6 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PrecommitTask.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PrecommitTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.conventions.precommit; diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PrecommitTaskPlugin.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PrecommitTaskPlugin.java index 49148330e02ed..fa8720f3dce28 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PrecommitTaskPlugin.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PrecommitTaskPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.conventions.precommit; diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/util/Util.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/util/Util.java index 39a4ec3f30e85..16e8af5832c56 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/util/Util.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/util/Util.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.conventions.util; @@ -13,18 +14,19 @@ import org.gradle.api.Project; import org.gradle.api.file.FileTree; import org.gradle.api.initialization.IncludedBuild; +import org.gradle.api.internal.GradleInternal; import org.gradle.api.invocation.Gradle; import org.gradle.api.plugins.JavaPluginExtension; import org.gradle.api.tasks.SourceSet; import org.gradle.api.tasks.SourceSetContainer; import org.gradle.api.tasks.util.PatternFilterable; -import javax.annotation.Nullable; import java.io.File; -import java.util.Collection; import java.util.Optional; import java.util.function.Supplier; +import javax.annotation.Nullable; + public class Util { public static boolean getBooleanProperty(String property, boolean defaultValue) { @@ -119,6 +121,14 @@ public static SourceSetContainer getJavaSourceSets(Project project) { return project.getExtensions().getByType(JavaPluginExtension.class).getSourceSets(); } + public static File getRootFolder(Gradle gradle) { + Gradle parent = gradle.getParent(); + if (parent == null) { + return gradle.getRootProject().getRootDir(); + } + return getRootFolder(parent); + } + public static File locateElasticsearchWorkspace(Gradle gradle) { if (gradle.getRootProject().getName().startsWith("build-tools")) { File buildToolsParent = gradle.getRootProject().getRootDir().getParentFile(); diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle index a8d1110ff4736..f2a02645f8c09 100644 --- a/build-tools-internal/build.gradle +++ b/build-tools-internal/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.apache.tools.ant.filters.ReplaceTokens @@ -257,9 +258,6 @@ tasks.named('licenseHeaders').configure { *****************************************************************************/ repositories { - maven { - url 'https://jitpack.io' - } mavenCentral() gradlePluginPortal() } @@ -383,6 +381,18 @@ tasks.named("jar") { exclude("classpath.index") } +spotless { + java { + + // workaround for https://github.com/diffplug/spotless/issues/2317 + //toggleOffOn() + target project.fileTree("src/main/java") { + include '**/*.java' + exclude '**/DockerBase.java' + } + } +} + def resolveMainWrapperVersion() { new URL("https://raw.githubusercontent.com/elastic/elasticsearch/main/build-tools-internal/src/main/resources/minimumGradleVersion").text.trim() } diff --git a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties index 9036682bf0f0c..e712035eabc7b 100644 --- a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties +++ b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=682b4df7fe5accdca84a4d1ef6a3a6ab096b3efd5edf7de2bd8c758d95a93703 -distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-all.zip +distributionSha256Sum=7ebdac923867a3cec0098302416d1e3c6c0c729fc4e2e05c10637a8af33a76c5 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.12-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/build-tools-internal/settings.gradle b/build-tools-internal/settings.gradle index 1b4fb1215a59d..8c88d36046768 100644 --- a/build-tools-internal/settings.gradle +++ b/build-tools-internal/settings.gradle @@ -1,8 +1,5 @@ pluginManagement { repositories { - maven { - url 'https://jitpack.io' - } mavenCentral() gradlePluginPortal() } diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGitAwareGradleFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGitAwareGradleFuncTest.groovy index 02bab0a0dae8b..01effd52dafef 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGitAwareGradleFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGitAwareGradleFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.fixtures diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleInternalPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleInternalPluginFuncTest.groovy index a89a26d2800d4..d93cc4b1873f2 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleInternalPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleInternalPluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.fixtures diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractRestResourcesFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractRestResourcesFuncTest.groovy index 21582b6823b81..1396272026abc 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractRestResourcesFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractRestResourcesFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.fixtures; diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/LocalRepositoryFixture.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/LocalRepositoryFixture.groovy index 1926cf9f03955..8d45142fdd48e 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/LocalRepositoryFixture.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/LocalRepositoryFixture.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.fixtures @@ -66,7 +67,7 @@ class LocalRepositoryFixture extends ExternalResource { artifact() } } - } + } """ } diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/BuildPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/BuildPluginFuncTest.groovy index 96e342e995a36..03b044583add0 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/BuildPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/BuildPluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal @@ -83,10 +84,11 @@ class BuildPluginFuncTest extends AbstractGradleFuncTest { file('src/main/java/org/elasticsearch/SampleClass.java') << """\ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch; diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavaModulePathPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavaModulePathPluginFuncTest.groovy index 6f5f7b2574969..7e294b2fc6cd1 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavaModulePathPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavaModulePathPluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavaPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavaPluginFuncTest.groovy index ff329e766bfe7..9fc6aa7276b2d 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavaPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavaPluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavadocPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavadocPluginFuncTest.groovy index c63eca8680179..34fa73ce502ac 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavadocPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavadocPluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal @@ -72,7 +73,7 @@ class ElasticsearchJavadocPluginFuncTest extends AbstractGradleFuncTest { buildFile << """ plugins { id 'elasticsearch.java-doc' - id 'com.github.johnrengelman.shadow' version '7.1.2' + id 'com.gradleup.shadow' id 'java' } group = 'org.acme.depending' diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchTestBasePluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchTestBasePluginFuncTest.groovy index 06f489a8bc1bf..733742653e4a3 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchTestBasePluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchTestBasePluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ExportElasticsearchBuildResourcesTaskFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ExportElasticsearchBuildResourcesTaskFuncTest.groovy index 371ceca952eb1..d407f763c7594 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ExportElasticsearchBuildResourcesTaskFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ExportElasticsearchBuildResourcesTaskFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalBwcGitPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalBwcGitPluginFuncTest.groovy index a3bb5e8327e9e..9f77ea4c2eb7a 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalBwcGitPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalBwcGitPluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionArchiveCheckPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionArchiveCheckPluginFuncTest.groovy index 860dc4e6f4d91..05d4853d1a778 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionArchiveCheckPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionArchiveCheckPluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPluginFuncTest.groovy index c1db414c80fd3..43edd15a5dcf6 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy index fc7ccd651d73b..bb100b6b23882 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy @@ -1,23 +1,19 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal -import org.elasticsearch.gradle.Architecture +import spock.lang.Unroll + import org.elasticsearch.gradle.fixtures.AbstractGitAwareGradleFuncTest import org.gradle.testkit.runner.TaskOutcome -import spock.lang.IgnoreIf -import spock.lang.Unroll -/* - * Test is ignored on ARM since this test case tests the ability to build certain older BWC branches that we don't support on ARM - */ -@IgnoreIf({ Architecture.current() == Architecture.AARCH64 }) class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleFuncTest { def setup() { @@ -27,9 +23,11 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF buildFile << """ apply plugin: 'elasticsearch.internal-distribution-bwc-setup' """ - execute("git branch origin/8.0", file("cloned")) + execute("git branch origin/8.x", file("cloned")) + execute("git branch origin/8.3", file("cloned")) + execute("git branch origin/8.2", file("cloned")) + execute("git branch origin/8.1", file("cloned")) execute("git branch origin/7.16", file("cloned")) - execute("git branch origin/7.15", file("cloned")) } def "builds distribution from branches via archives extractedAssemble"() { @@ -53,10 +51,11 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF assertOutputContains(result.output, "[$bwcDistVersion] > Task :distribution:archives:darwin-tar:${expectedAssembleTaskName}") where: - bwcDistVersion | bwcProject | expectedAssembleTaskName - "8.0.0" | "minor" | "extractedAssemble" - "7.16.0" | "staged" | "extractedAssemble" - "7.15.2" | "bugfix" | "extractedAssemble" + bwcDistVersion | bwcProject | expectedAssembleTaskName + "8.4.0" | "minor" | "extractedAssemble" + "8.3.0" | "staged" | "extractedAssemble" + "8.2.1" | "bugfix" | "extractedAssemble" + "8.1.3" | "bugfix2" | "extractedAssemble" } @Unroll @@ -75,8 +74,8 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF where: bwcDistVersion | platform - "8.0.0" | "darwin" - "8.0.0" | "linux" + "8.4.0" | "darwin" + "8.4.0" | "linux" } def "bwc expanded distribution folder can be resolved as bwc project artifact"() { @@ -112,11 +111,11 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF result.task(":resolveExpandedDistribution").outcome == TaskOutcome.SUCCESS result.task(":distribution:bwc:minor:buildBwcDarwinTar").outcome == TaskOutcome.SUCCESS and: "assemble task triggered" - result.output.contains("[8.0.0] > Task :distribution:archives:darwin-tar:extractedAssemble") - result.output.contains("expandedRootPath /distribution/bwc/minor/build/bwc/checkout-8.0/" + + result.output.contains("[8.4.0] > Task :distribution:archives:darwin-tar:extractedAssemble") + result.output.contains("expandedRootPath /distribution/bwc/minor/build/bwc/checkout-8.x/" + "distribution/archives/darwin-tar/build/install") - result.output.contains("nested folder /distribution/bwc/minor/build/bwc/checkout-8.0/" + - "distribution/archives/darwin-tar/build/install/elasticsearch-8.0.0-SNAPSHOT") + result.output.contains("nested folder /distribution/bwc/minor/build/bwc/checkout-8.x/" + + "distribution/archives/darwin-tar/build/install/elasticsearch-8.4.0-SNAPSHOT") } } diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy index d43dbec5ef6b6..fc5d432a9ef9a 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal @@ -56,7 +57,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest elasticsearch_distributions { test_distro { - version = "8.0.0" + version = "8.4.0" type = "archive" platform = "linux" architecture = Architecture.current(); @@ -86,7 +87,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest elasticsearch_distributions { test_distro { - version = "8.0.0" + version = "8.4.0" type = "archive" platform = "linux" architecture = Architecture.current(); diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy index 24131c633e9d7..94df02b280ca6 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy index 9d32eaadf7aec..b22fc11019b73 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal @@ -17,7 +18,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { def setup() { // required for JarHell to work - subProject(":libs:elasticsearch-core") << "apply plugin:'java'" + subProject(":libs:core") << "apply plugin:'java'" configurationCacheCompatible = false } @@ -44,7 +45,8 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { file("build/distributions/hello-world-1.0-javadoc.jar").exists() file("build/distributions/hello-world-1.0-sources.jar").exists() file("build/distributions/hello-world-1.0.pom").exists() - assertXmlEquals(file("build/distributions/hello-world-1.0.pom").text, """ + assertXmlEquals( + file("build/distributions/hello-world-1.0.pom").text, """ @@ -68,6 +70,11 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { https://raw.githubusercontent.com/elastic/elasticsearch/v1.0/licenses/ELASTIC-LICENSE-2.0.txt repo + + GNU Affero General Public License Version 3 + https://raw.githubusercontent.com/elastic/elasticsearch/v1.0/licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt + repo + Server Side Public License, v 1 https://www.mongodb.com/licensing/server-side-public-license @@ -90,7 +97,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { plugins { id 'elasticsearch.java' id 'elasticsearch.publish' - id 'com.github.johnrengelman.shadow' + id 'com.gradleup.shadow' } repositories { @@ -111,7 +118,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { } version = "1.0" group = 'org.acme' - description = 'some description' + description = 'shadowed project' """ when: @@ -124,14 +131,15 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { file("build/distributions/hello-world-1.0-javadoc.jar").exists() file("build/distributions/hello-world-1.0-sources.jar").exists() file("build/distributions/hello-world-1.0.pom").exists() - assertXmlEquals(file("build/distributions/hello-world-1.0.pom").text, """ + assertXmlEquals( + file("build/distributions/hello-world-1.0.pom").text, """ 4.0.0 org.acme hello-world 1.0 hello-world - some description + shadowed project unknown unknown @@ -143,6 +151,11 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { https://raw.githubusercontent.com/elastic/elasticsearch/v1.0/licenses/ELASTIC-LICENSE-2.0.txt repo + + GNU Affero General Public License Version 3 + https://raw.githubusercontent.com/elastic/elasticsearch/v1.0/licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt + repo + Server Side Public License, v 1 https://www.mongodb.com/licensing/server-side-public-license @@ -175,7 +188,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { plugins { id 'elasticsearch.java' id 'elasticsearch.publish' - id 'com.github.johnrengelman.shadow' + id 'com.gradleup.shadow' } dependencies { @@ -195,7 +208,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { group = 'org.acme' } - description = 'some description' + description = 'with shadowed dependencies' """ when: @@ -208,14 +221,15 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { file("build/distributions/hello-world-1.0-javadoc.jar").exists() file("build/distributions/hello-world-1.0-sources.jar").exists() file("build/distributions/hello-world-1.0.pom").exists() - assertXmlEquals(file("build/distributions/hello-world-1.0.pom").text, """ + assertXmlEquals( + file("build/distributions/hello-world-1.0.pom").text, """ 4.0.0 org.acme hello-world 1.0 hello-world - some description + with shadowed dependencies unknown unknown @@ -227,6 +241,11 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { https://raw.githubusercontent.com/elastic/elasticsearch/v1.0/licenses/ELASTIC-LICENSE-2.0.txt repo + + GNU Affero General Public License Version 3 + https://raw.githubusercontent.com/elastic/elasticsearch/v1.0/licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt + repo + Server Side Public License, v 1 https://www.mongodb.com/licensing/server-side-public-license @@ -261,13 +280,13 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { plugins { id 'elasticsearch.internal-es-plugin' id 'elasticsearch.publish' - id 'com.github.johnrengelman.shadow' + id 'com.gradleup.shadow' } esplugin { name = 'hello-world-plugin' - classname 'org.acme.HelloWorldPlugin' - description = "custom project description" + classname = 'org.acme.HelloWorldPlugin' + description = "shadowed es plugin" } publishing { @@ -296,7 +315,8 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { file("build/distributions/hello-world-plugin-1.0-javadoc.jar").exists() file("build/distributions/hello-world-plugin-1.0-sources.jar").exists() file("build/distributions/hello-world-plugin-1.0.pom").exists() - assertXmlEquals(file("build/distributions/hello-world-plugin-1.0.pom").text, """ + assertXmlEquals( + file("build/distributions/hello-world-plugin-1.0.pom").text, """ @@ -308,7 +328,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { hello-world-plugin 1.0 hello-world - custom project description + shadowed es plugin unknown unknown @@ -320,6 +340,11 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { https://raw.githubusercontent.com/elastic/elasticsearch/v1.0/licenses/ELASTIC-LICENSE-2.0.txt repo + + GNU Affero General Public License Version 3 + https://raw.githubusercontent.com/elastic/elasticsearch/v1.0/licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt + repo + Server Side Public License, v 1 https://www.mongodb.com/licensing/server-side-public-license @@ -332,7 +357,6 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { https://www.elastic.co - """ ) } @@ -351,7 +375,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { esplugin { name = 'hello-world-plugin' - classname 'org.acme.HelloWorldPlugin' + classname = 'org.acme.HelloWorldPlugin' description = "custom project description" } @@ -369,7 +393,8 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { then: result.task(":generatePom").outcome == TaskOutcome.SUCCESS file("build/distributions/hello-world-plugin-2.0.pom").exists() - assertXmlEquals(file("build/distributions/hello-world-plugin-2.0.pom").text, """ + assertXmlEquals( + file("build/distributions/hello-world-plugin-2.0.pom").text, """ @@ -393,6 +418,11 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { https://raw.githubusercontent.com/elastic/elasticsearch/v2.0/licenses/ELASTIC-LICENSE-2.0.txt repo + + GNU Affero General Public License Version 3 + https://raw.githubusercontent.com/elastic/elasticsearch/v2.0/licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt + repo + Server Side Public License, v 1 https://www.mongodb.com/licensing/server-side-public-license @@ -414,8 +444,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { // scm info only added for internal builds internalBuild() buildFile << """ - BuildParams.init { it.setGitOrigin("https://some-repo.com/repo.git") } - + buildParams.setGitOrigin(project.providers.provider(() -> "https://some-repo.com/repo.git")) apply plugin:'elasticsearch.java' apply plugin:'elasticsearch.publish' @@ -423,7 +452,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { group = 'org.acme' description = "just a test project" - ext.projectLicenses.set(['The Apache Software License, Version 2.0': 'http://www.apache.org/licenses/LICENSE-2.0']) + ext.projectLicenses.set(['The Apache Software License, Version 2.0': project.providers.provider(() -> 'http://www.apache.org/licenses/LICENSE-2.0')]) """ when: @@ -432,7 +461,8 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { then: result.task(":generatePom").outcome == TaskOutcome.SUCCESS file("build/distributions/hello-world-1.0.pom").exists() - assertXmlEquals(file("build/distributions/hello-world-1.0.pom").text, """ + assertXmlEquals( + file("build/distributions/hello-world-1.0.pom").text, """ @@ -469,15 +499,15 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { private boolean assertXmlEquals(String toTest, String expected) { def diff = DiffBuilder.compare(Input.fromString(expected)) - .ignoreWhitespace() - .ignoreComments() - .normalizeWhitespace() - .withTest(Input.fromString(toTest)) - .build() + .ignoreWhitespace() + .ignoreComments() + .normalizeWhitespace() + .withTest(Input.fromString(toTest)) + .build() diff.differences.each { difference -> println difference } - if(diff.differences.size() > 0) { + if (diff.differences.size() > 0) { println """ given: $toTest """ diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/StringTemplatePluginTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/StringTemplatePluginTest.groovy index 6c94bdd3e86c5..44fb3a0fde505 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/StringTemplatePluginTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/StringTemplatePluginTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTarFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTarFuncTest.groovy index 237aa99e4b824..57ba6a7773d50 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTarFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTarFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/distibution/ElasticsearchDistributionPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/distibution/ElasticsearchDistributionPluginFuncTest.groovy index 8686ad3df79ae..038ba8b4f1917 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/distibution/ElasticsearchDistributionPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/distibution/ElasticsearchDistributionPluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.distibution @@ -22,11 +23,11 @@ class ElasticsearchDistributionPluginFuncTest extends AbstractGradleFuncTest { buildFile << """plugins { id 'elasticsearch.distro' } - + def someCopy = tasks.register('someCopy', Sync) { into 'build/targetDir' } - + distro.copyModule(someCopy, project(":module")) """ when: @@ -44,13 +45,13 @@ class ElasticsearchDistributionPluginFuncTest extends AbstractGradleFuncTest { plugins { id 'elasticsearch.esplugin' } - + esplugin { name = 'some-test-module' classname = 'org.acme.never.used.TestPluginClass' description = 'some plugin description' } - + // for testing purposes only configurations.compileOnly.dependencies.clear() """ diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/doc/DocsTestPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/doc/DocsTestPluginFuncTest.groovy index 934ff5233ec13..de2078b7a0c79 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/doc/DocsTestPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/doc/DocsTestPluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.doc diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/dra/DraResolvePluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/dra/DraResolvePluginFuncTest.groovy index fb7c68b010ff3..13907446aab0f 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/dra/DraResolvePluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/dra/DraResolvePluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.dra @@ -28,7 +29,7 @@ class DraResolvePluginFuncTest extends AbstractGradleFuncTest { plugins { id 'elasticsearch.dra-artifacts' } - + repositories.all { // for supporting http testing repos here allowInsecureProtocol = true @@ -49,7 +50,7 @@ class DraResolvePluginFuncTest extends AbstractGradleFuncTest { artifact() } } - } + } } """ @@ -57,11 +58,11 @@ class DraResolvePluginFuncTest extends AbstractGradleFuncTest { configurations { someConfig } - + dependencies { someConfig "org.acme:ml-cpp:8.6.0-SNAPSHOT" } - + tasks.register('resolveArtifacts') { doLast { configurations.someConfig.files.each { println it } @@ -92,13 +93,13 @@ class DraResolvePluginFuncTest extends AbstractGradleFuncTest { configurations { someConfig } - + dependencies { someConfig "some.group:bar:1.0.0" someConfig "some.group:baz:1.0.0-SNAPSHOT" someConfig "org.acme:$draArtifact:$draVersion:deps@zip" } - + tasks.register('resolveArtifacts') { doLast { configurations.someConfig.files.each { println it } diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/CheckstylePrecommitPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/CheckstylePrecommitPluginFuncTest.groovy index 4d999f1b0ded4..fd6e7199d2c37 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/CheckstylePrecommitPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/CheckstylePrecommitPluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/LicenseHeadersPrecommitPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/LicenseHeadersPrecommitPluginFuncTest.groovy index fed667b014c23..ff20af406b2a9 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/LicenseHeadersPrecommitPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/LicenseHeadersPrecommitPluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit @@ -130,10 +131,11 @@ class LicenseHeadersPrecommitPluginFuncTest extends AbstractGradleInternalPlugin file("src/main/java/org/acme/DualLicensed.java") << """ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.acme; diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPluginFuncTest.groovy index f6cbef21f5655..2b3206f13230d 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTaskFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTaskFuncTest.groovy index c6a4572cb8a86..36da4efaa1c57 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTaskFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTaskFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy index 0e0c4cb25a759..725f117d17e64 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.snyk @@ -160,7 +161,7 @@ class SnykDependencyMonitoringGradlePluginFuncTest extends AbstractGradleInterna }, "target": { "remoteUrl": "http://acme.org", - "branch": "unknown" + "branch": "$version" }, "targetReference": "$version", "projectAttributes": { diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rerun/InternalTestRerunPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rerun/InternalTestRerunPluginFuncTest.groovy index 0dd982665808a..39b86a8155fb9 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rerun/InternalTestRerunPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rerun/InternalTestRerunPluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rerun @@ -25,11 +26,11 @@ class InternalTestRerunPluginFuncTest extends AbstractGradleFuncTest { repositories { mavenCentral() } - + dependencies { testImplementation 'junit:junit:4.13.1' } - + tasks.named("test").configure { maxParallelForks = 4 testLogging { @@ -37,7 +38,7 @@ class InternalTestRerunPluginFuncTest extends AbstractGradleFuncTest { exceptionFormat "short" } } - + """ createTest("SimpleTest") createTest("SimpleTest2") @@ -81,11 +82,11 @@ Test jvm system exit trace:""") == false repositories { mavenCentral() } - + dependencies { testImplementation 'junit:junit:4.13.1' } - + tasks.named("test").configure { maxParallelForks = 4 testLogging { @@ -94,7 +95,7 @@ Test jvm system exit trace:""") == false exceptionFormat "short" } } - + """ createTest("AnotherTest") createTest("AnotherTest2") @@ -137,11 +138,11 @@ Gradle Test Executor 1 > AnotherTest6 > someTest repositories { mavenCentral() } - + dependencies { testImplementation 'junit:junit:4.13.1' } - + tasks.named("test").configure { maxParallelForks = 5 testLogging { @@ -149,7 +150,7 @@ Gradle Test Executor 1 > AnotherTest6 > someTest exceptionFormat "short" } } - + """ createSystemExitTest("AnotherTest6") createFailedTest("SimpleTest1") @@ -178,11 +179,11 @@ Gradle Test Executor 1 > AnotherTest6 > someTest repositories { mavenCentral() } - + dependencies { testImplementation 'junit:junit:4.13.1' } - + tasks.named("test").configure { rerun { maxReruns = 4 @@ -247,24 +248,24 @@ Gradle Test Executor 1 > AnotherTest6 > someTest import java.nio.*; import java.nio.file.*; import java.io.IOException; - + public class $clazzName { Path executionLogPath = Paths.get("test-executions" + getClass().getSimpleName() +".log"); - - @Before + + @Before public void beforeTest() { logExecution(); } - - @After + + @After public void afterTest() { } - - @Test + + @Test public void someTest() { ${content} } - + int countExecutions() { try { return Files.readAllLines(executionLogPath).size(); @@ -273,7 +274,7 @@ Gradle Test Executor 1 > AnotherTest6 > someTest return 0; } } - + void logExecution() { try { Files.write(executionLogPath, "Test executed\\n".getBytes(), StandardOpenOption.CREATE, StandardOpenOption.APPEND); diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy index 737c448f23be6..8a02b34023a27 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest @@ -38,6 +39,8 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe def "yamlRestTestVxCompatTest does nothing when there are no tests"() { given: + internalBuild() + subProject(":distribution:bwc:maintenance") << """ configurations { checkout } artifacts { @@ -46,9 +49,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe """ buildFile << """ - plugins { - id 'elasticsearch.legacy-yaml-rest-compat-test' - } + apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' """ when: @@ -61,7 +62,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe result.task(transformTask).outcome == TaskOutcome.NO_SOURCE } - def "yamlRestTestVxCompatTest executes and copies api and transforms tests from :bwc:maintenance"() { + def "yamlRestCompatTest executes and copies api and transforms tests from :bwc:maintenance"() { given: internalBuild() @@ -143,6 +144,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe def "yamlRestTestVxCompatTest is wired into check and checkRestCompat"() { given: + internalBuild() withVersionCatalogue() subProject(":distribution:bwc:maintenance") << """ configurations { checkout } @@ -152,10 +154,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe """ buildFile << """ - plugins { - id 'elasticsearch.legacy-yaml-rest-compat-test' - } - + apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' """ when: diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy index 5e96fa524268a..ce5c1519fe11f 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy @@ -1,14 +1,16 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest import spock.lang.IgnoreIf +import spock.lang.IgnoreRest import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.fixtures.AbstractRestResourcesFuncTest @@ -19,16 +21,16 @@ import org.gradle.testkit.runner.TaskOutcome class LegacyYamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest { def setup() { + configurationCacheCompatible = true buildApiRestrictionsDisabled = true } def "yamlRestTest does nothing when there are no tests"() { given: + internalBuild() buildFile << """ - plugins { - id 'elasticsearch.legacy-yaml-rest-test' - } + apply plugin: 'elasticsearch.legacy-yaml-rest-test' """ when: @@ -135,7 +137,7 @@ class LegacyYamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest { """ when: - def result = gradleRunner("yamlRestTest", "--console", 'plain', '--stacktrace').buildAndFail() + def result = gradleRunner("yamlRestTest", "--console", 'plain').buildAndFail() then: result.task(":distribution:archives:integ-test-zip:buildExpanded").outcome == TaskOutcome.SUCCESS diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/RestResourcesPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/RestResourcesPluginFuncTest.groovy index e083dc6ff56d5..2ea775412fe5f 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/RestResourcesPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/RestResourcesPluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest diff --git a/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/build.gradle b/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/build.gradle index 6611f7a28da1c..c671c18cad030 100644 --- a/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/build.gradle +++ b/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ subprojects { diff --git a/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/darwin-aarch64-tar/build.gradle b/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/darwin-aarch64-tar/build.gradle index 5c2d5b68ae2e0..ad21076a9ca93 100644 --- a/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/darwin-aarch64-tar/build.gradle +++ b/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/darwin-aarch64-tar/build.gradle @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ diff --git a/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/darwin-tar/build.gradle b/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/darwin-tar/build.gradle index 5c2d5b68ae2e0..ad21076a9ca93 100644 --- a/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/darwin-tar/build.gradle +++ b/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/darwin-tar/build.gradle @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ diff --git a/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/linux-aarch64-tar/build.gradle b/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/linux-aarch64-tar/build.gradle index 5c2d5b68ae2e0..ad21076a9ca93 100644 --- a/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/linux-aarch64-tar/build.gradle +++ b/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/linux-aarch64-tar/build.gradle @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ diff --git a/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/oss-darwin-aarch64-tar/build.gradle b/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/oss-darwin-aarch64-tar/build.gradle index 5c2d5b68ae2e0..ad21076a9ca93 100644 --- a/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/oss-darwin-aarch64-tar/build.gradle +++ b/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/oss-darwin-aarch64-tar/build.gradle @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ diff --git a/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/oss-linux-aarch64-tar/build.gradle b/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/oss-linux-aarch64-tar/build.gradle index 5c2d5b68ae2e0..ad21076a9ca93 100644 --- a/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/oss-linux-aarch64-tar/build.gradle +++ b/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/oss-linux-aarch64-tar/build.gradle @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ diff --git a/modules/repository-azure/licenses/protobuf-java-NOTICE.txt b/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/bwc/bugfix2/build.gradle similarity index 100% rename from modules/repository-azure/licenses/protobuf-java-NOTICE.txt rename to build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/bwc/bugfix2/build.gradle diff --git a/modules/repository-azure/licenses/tink-NOTICE.txt b/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/bwc/maintenance/build.gradle similarity index 100% rename from modules/repository-azure/licenses/tink-NOTICE.txt rename to build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/bwc/maintenance/build.gradle diff --git a/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/gradle.properties b/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/gradle.properties index 6eb38bd574f3d..28e034cf03fd3 100644 --- a/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/gradle.properties +++ b/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/gradle.properties @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # forcing to use TLS1.2 to avoid failure in vault diff --git a/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/settings.gradle b/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/settings.gradle index 12a802078e879..e931537fcd6e9 100644 --- a/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/settings.gradle +++ b/build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/settings.gradle @@ -1,17 +1,20 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ rootProject.name = "root" include ":distribution:bwc:bugfix" +include ":distribution:bwc:bugfix2" include ":distribution:bwc:minor" include ":distribution:bwc:major" include ":distribution:bwc:staged" +include ":distribution:bwc:maintenance" include ":distribution:archives:darwin-tar" include ":distribution:archives:oss-darwin-tar" include ":distribution:archives:darwin-aarch64-tar" diff --git a/build-tools-internal/src/main/groovy/elasticsearch.authenticated-testclusters.gradle b/build-tools-internal/src/main/groovy/elasticsearch.authenticated-testclusters.gradle index 102a838235cb1..729cf3b8fc267 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.authenticated-testclusters.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.authenticated-testclusters.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask diff --git a/build-tools-internal/src/main/groovy/elasticsearch.base.gradle b/build-tools-internal/src/main/groovy/elasticsearch.base.gradle index b65b6fca43b2c..f7b7ff179aaff 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.base.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.base.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.VersionProperties @@ -11,4 +12,4 @@ import org.elasticsearch.gradle.VersionProperties project.setDescription("Elasticsearch subproject " + project.getPath()); // common maven publishing configuration project.setGroup("org.elasticsearch"); -project.setVersion(VersionProperties.getElasticsearch()) \ No newline at end of file +project.setVersion(VersionProperties.getElasticsearch()) diff --git a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle index d604973efcb4b..bc6d6f8d48d33 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import java.lang.management.ManagementFactory; @@ -11,7 +12,6 @@ import java.time.LocalDateTime; import org.elasticsearch.gradle.Architecture import org.elasticsearch.gradle.OS -import org.elasticsearch.gradle.internal.info.BuildParams import java.lang.management.ManagementFactory import java.time.LocalDateTime @@ -33,12 +33,15 @@ develocity { publishing.onlyIf { false } } + def fips = buildParams.inFipsJvm + def gitRevision = buildParams.gitRevision + background { tag OS.current().name() tag Architecture.current().name() // Tag if this build is run in FIPS mode - if (BuildParams.inFipsJvm) { + if (fips) { tag 'FIPS' } @@ -91,8 +94,8 @@ develocity { link 'Source', "${prBaseUrl}/tree/${System.getenv('BUILDKITE_COMMIT')}" link 'Pull Request', "https://github.com/${repository}/pull/${prId}" } else { - value 'Git Commit ID', BuildParams.gitRevision - link 'Source', "https://github.com/${repository}/tree/${BuildParams.gitRevision}" + value 'Git Commit ID', gitRevision.get() + link 'Source', "https://github.com/${repository}/tree/${gitRevision.get()}" } buildFinished { result -> diff --git a/build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle b/build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle index ff9b6fe7a526d..77e509ea97870 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle @@ -1,14 +1,14 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.internal.ElasticsearchTestBasePlugin -import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.internal.test.rest.InternalJavaRestTestPlugin import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask @@ -18,7 +18,7 @@ ext.bwcTaskName = { Version version -> def bwcTestSnapshots = tasks.register("bwcTestSnapshots") { if (project.bwc_tests_enabled) { - dependsOn tasks.matching { task -> BuildParams.bwcVersions.unreleased.any { version -> bwcTaskName(version) == task.name } } + dependsOn tasks.matching { task -> buildParams.bwcVersions.unreleased.any { version -> bwcTaskName(version) == task.name } } } } diff --git a/build-tools-internal/src/main/groovy/elasticsearch.fips.gradle b/build-tools-internal/src/main/groovy/elasticsearch.fips.gradle index f691d4bd996a7..3c9cf121813c9 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.fips.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.fips.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ @@ -14,21 +15,22 @@ import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask import org.elasticsearch.gradle.testclusters.TestClustersAware import org.elasticsearch.gradle.testclusters.TestDistribution -// Common config when running with a FIPS-140 runtime JVM -if (BuildParams.inFipsJvm) { +//apply plugin: org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin +// Common config when running with a FIPS-140 runtime JVM +if (buildParams.inFipsJvm) { allprojects { - String javaSecurityFilename = BuildParams.runtimeJavaDetails.toLowerCase().contains('oracle') ? 'fips_java_oracle.security' : 'fips_java.security' + String javaSecurityFilename = buildParams.runtimeJavaDetails.get().toLowerCase().contains('oracle') ? 'fips_java_oracle.security' : 'fips_java.security' File fipsResourcesDir = new File(project.buildDir, 'fips-resources') File fipsSecurity = new File(fipsResourcesDir, javaSecurityFilename) File fipsPolicy = new File(fipsResourcesDir, 'fips_java.policy') File fipsTrustStore = new File(fipsResourcesDir, 'cacerts.bcfks') - def bcFips = dependencies.create('org.bouncycastle:bc-fips:1.0.2.4') - def bcTlsFips = dependencies.create('org.bouncycastle:bctls-fips:1.0.17') + def bcFips = dependencies.create('org.bouncycastle:bc-fips:1.0.2.5') + def bcTlsFips = dependencies.create('org.bouncycastle:bctls-fips:1.0.19') def manualDebug = false; //change this to manually debug bouncy castle in an IDE if(manualDebug) { - bcFips = dependencies.create('org.bouncycastle:bc-fips-debug:1.0.2.4') - bcTlsFips = dependencies.create('org.bouncycastle:bctls-fips:1.0.17'){ + bcFips = dependencies.create('org.bouncycastle:bc-fips-debug:1.0.2.5') + bcTlsFips = dependencies.create('org.bouncycastle:bctls-fips:1.0.19'){ exclude group: 'org.bouncycastle', module: 'bc-fips' // to avoid jar hell } } diff --git a/build-tools-internal/src/main/groovy/elasticsearch.forbidden-dependencies.gradle b/build-tools-internal/src/main/groovy/elasticsearch.forbidden-dependencies.gradle index e67cb7846c791..b52692e192ebb 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.forbidden-dependencies.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.forbidden-dependencies.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ // we do not want any of these dependencies on the compilation classpath diff --git a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle index dd8b582adb92f..671c79d9ed966 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.util.Pair @@ -24,12 +25,17 @@ allprojects { } } +interface Injected { + @Inject FileSystemOperations getFs() +} + // Applying this stuff, particularly the idea-ext plugin, has a cost so avoid it unless we're running in the IDE if (providers.systemProperty('idea.active').getOrNull() == 'true') { project.apply(plugin: org.jetbrains.gradle.ext.IdeaExtPlugin) def elasticsearchProject = locateElasticsearchWorkspace(gradle) + def rootFolder = project.rootDir tasks.register('configureIdeCheckstyle') { group = 'ide' description = 'Generated a suitable checkstyle config for IDEs' @@ -39,10 +45,10 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') { String checkstyleConfig = "${resources}/checkstyle.xml" String checkstyleSuppressions = "${resources}/checkstyle_suppressions.xml" String checkstyleIdeFragment = "${resources}/checkstyle_ide_fragment.xml" - String checkstyleIdeConfig = "${rootDir}/checkstyle_ide.xml" + String checkstyleIdeConfig = "${rootFolder}/checkstyle_ide.xml" String checkstylePluginConfigTemplate = "${resources}/checkstyle-idea.xml" - String checkstylePluginConfig = "${rootDir}/.idea/checkstyle-idea.xml" + String checkstylePluginConfig = "${rootFolder}/.idea/checkstyle-idea.xml" inputs.files( file(checkstyleConfig), @@ -53,31 +59,33 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') { file(checkstyleIdeConfig), file(checkstylePluginConfig) ) + def injected = project.objects.newInstance(Injected) + def projectFolder = project.layout.projectDirectory.asFile doLast { // Configure the IntelliJ Checkstyle plugin by copying a standard file. We don't simply commit // the result to version control, because the plugin has a habit of modifying the file and // replacing the `$PROJECT_DIR$` placeholders, which developers must then revert. - project.copy { + injected.fs.copy { from(checkstylePluginConfigTemplate) - into("${rootDir}/.idea") + into("${rootFolder}/.idea") expand(jarLocation: buildConventionsJar, configLocation: checkstyleIdeConfig) } // Create an IDE-specific checkstyle config by first copying the standard config Files.copy( - Paths.get(file(checkstyleConfig).getPath()), - Paths.get(file(checkstyleIdeConfig).getPath()), + Paths.get(new File(checkstyleConfig).getPath()), + Paths.get(new File(checkstyleIdeConfig).getPath()), StandardCopyOption.REPLACE_EXISTING ) // There are some rules that we only want to enable in an IDE. These // are extracted to a separate file, and merged into the IDE-specific // Checkstyle config. - Node xmlFragment = parseXml(checkstyleIdeFragment) + Node xmlFragment = IdeaXmlUtil.parseXml(checkstyleIdeFragment) // Edit the copy so that IntelliJ can copy with it - modifyXml(checkstyleIdeConfig, { xml -> + IdeaXmlUtil.modifyXml(checkstyleIdeConfig, { xml -> // Add all the nodes from the fragment file Node treeWalker = xml.module.find { it.'@name' == 'TreeWalker' } xmlFragment.module.each { treeWalker.append(it) } @@ -103,7 +111,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') { description = 'Configures the appropriate JVM for Gradle' doLast { - modifyXml('.idea/gradle.xml') { xml -> + IdeaXmlUtil.modifyXml('.idea/gradle.xml') { xml -> def gradleSettings = xml.component.find { it.'@name' == 'GradleSettings' }.option[0].GradleProjectSettings // Remove configured JVM option to force IntelliJ to use the project JDK for Gradle gradleSettings.option.findAll { it.'@name' == 'gradleJvm' }.each { it.parent().remove(it) } @@ -124,13 +132,13 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') { tasks.register('buildDependencyArtifacts') { group = 'ide' description = 'Builds artifacts needed as dependency for IDE modules' - dependsOn([':plugins:repository-hdfs:hadoop-client-api:shadowJar', + dependsOn([':plugins:repository-hdfs:hadoop-client-api:jar', ':x-pack:plugin:esql:compute:ann:jar', ':x-pack:plugin:esql:compute:gen:jar', ':server:generateModulesList', ':server:generatePluginsList', ':generateProviderImpls', - ':libs:elasticsearch-native:elasticsearch-native-libraries:extractLibs', + ':libs:native:native-libraries:extractLibs', ':x-pack:libs:es-opensaml-security-api:shadowJar'].collect { elasticsearchProject.right()?.task(it) ?: it }) } @@ -140,7 +148,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') { idea { project { vcs = 'Git' - jdkName = BuildParams.minimumCompilerVersion.majorVersion + jdkName = buildParams.minimumCompilerVersion.majorVersion settings { delegateActions { @@ -167,7 +175,8 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') { vmParameters = [ '-ea', '-Djava.security.manager=allow', - '-Djava.locale.providers=SPI,CLDR', + '-Djava.locale.providers=CLDR', + '-Dtests.testfeatures.enabled=true', '-Des.nativelibs.path="' + testLibraryPath + '"', // TODO: only open these for mockito when it is modularized '--add-opens=java.base/java.security.cert=ALL-UNNAMED', @@ -187,13 +196,14 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') { scopes = ['x-pack': 'Elastic', 'llrc': 'Apache2'] profiles { Default { - keyword = 'the Elastic License 2.0 or the Server' + keyword = 'GNU Affero General Public License v3.0' notice = '''\ Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - or more contributor license agreements. Licensed under the Elastic License - 2.0 and the Server Side Public License, v 1; you may not use this file except - in compliance with, at your election, the Elastic License 2.0 or the Server - Side Public License, v 1.'''.stripIndent() + or more contributor license agreements. Licensed under the "Elastic License + 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + Public License v 1"; you may not use this file except in compliance with, at + your election, the "Elastic License 2.0", the "GNU Affero General Public + License v3.0 only", or the "Server Side Public License, v 1".'''.stripIndent() } Elastic { keyword = '2.0; you may not use this file except in compliance with the Elastic License' @@ -238,33 +248,37 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') { * @param preface optional front matter to add after the XML declaration * but before the XML document, e.g. a doctype or comment */ -void modifyXml(Object path, Action action, String preface = null) { - if (project.file(path).exists()) { - Node xml = parseXml(path) - action.execute(xml) - - File xmlFile = project.file(path) - xmlFile.withPrintWriter { writer -> - def printer = new XmlNodePrinter(writer) - printer.namespaceAware = true - printer.preserveWhitespace = true - writer.write("\n") - - if (preface != null) { - writer.write(preface) + +class IdeaXmlUtil { + static Node parseXml(Object xmlPath) { + File xmlFile = new File(xmlPath) + XmlParser xmlParser = new XmlParser(false, true, true) + xmlParser.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false) + Node xml = xmlParser.parse(xmlFile) + return xml + } + + static void modifyXml(Object xmlPath, Action action, String preface = null) { + File xmlFile = new File(xmlPath) + if (xmlFile.exists()) { + Node xml = parseXml(xmlPath) + action.execute(xml) + + xmlFile.withPrintWriter { writer -> + def printer = new XmlNodePrinter(writer) + printer.namespaceAware = true + printer.preserveWhitespace = true + writer.write("\n") + + if (preface != null) { + writer.write(preface) + } + printer.print(xml) } - printer.print(xml) } } } -Node parseXml(Object path) { - File xmlFile = project.file(path) - XmlParser xmlParser = new XmlParser(false, true, true) - xmlParser.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false) - Node xml = xmlParser.parse(xmlFile) - return xml -} Pair locateElasticsearchWorkspace(Gradle gradle) { if (gradle.parent == null) { diff --git a/build-tools-internal/src/main/groovy/elasticsearch.local-distribution.gradle b/build-tools-internal/src/main/groovy/elasticsearch.local-distribution.gradle index 87d4be3c11d18..363a53dfaa5a0 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.local-distribution.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.local-distribution.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/build-tools-internal/src/main/groovy/elasticsearch.run-ccs.gradle b/build-tools-internal/src/main/groovy/elasticsearch.run-ccs.gradle index a3c7300cea951..587c97d3476ea 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.run-ccs.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.run-ccs.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.testclusters.RunTask diff --git a/build-tools-internal/src/main/groovy/elasticsearch.run.gradle b/build-tools-internal/src/main/groovy/elasticsearch.run.gradle index 3a905c001d0cf..90185179ef6ad 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.run.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.run.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.VersionProperties diff --git a/build-tools-internal/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle b/build-tools-internal/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle index 3400be77a588d..224e6bd4c50d1 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle @@ -1,15 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.Architecture import org.elasticsearch.gradle.OS import org.elasticsearch.gradle.VersionProperties -import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.internal.precommit.ThirdPartyAuditPrecommitPlugin import org.elasticsearch.gradle.internal.precommit.ThirdPartyAuditTask import org.elasticsearch.gradle.internal.test.rest.RestTestBasePlugin @@ -26,8 +26,8 @@ configure(allprojects) { JvmVendorSpec.matching(VersionProperties.bundledJdkVendor) } project.tasks.withType(Test).configureEach { Test test -> - if (BuildParams.getIsRuntimeJavaHomeSet()) { - test.executable = "${BuildParams.runtimeJavaHome}/bin/java" + + if (buildParams.getIsRuntimeJavaHomeSet()) { + test.executable = "${buildParams.runtimeJavaHome.get()}/bin/java" + (OS.current() == OS.WINDOWS ? '.exe' : '') } else { test.javaLauncher = javaToolchains.launcherFor { @@ -40,7 +40,7 @@ configure(allprojects) { } project.plugins.withId("elasticsearch.testclusters") { testClustersPlugin -> project.plugins.withId("elasticsearch.internal-testclusters") { internalPlugin -> - if (BuildParams.getIsRuntimeJavaHomeSet() == false) { + if (buildParams.getIsRuntimeJavaHomeSet() == false) { // If no runtime java home is set, use the bundled JDK for test clusters testClustersPlugin.setRuntimeJava(launcher.map { it.metadata.installationPath.asFile }) } diff --git a/build-tools-internal/src/main/groovy/elasticsearch.stable-api.gradle b/build-tools-internal/src/main/groovy/elasticsearch.stable-api.gradle index 0148caf8983ef..27b490329f8cb 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.stable-api.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.stable-api.gradle @@ -17,11 +17,11 @@ dependencies { newJar project(":libs:${project.name}") } -BuildParams.bwcVersions.withIndexCompatible({ it.onOrAfter(Version.fromString(ext.stableApiSince)) +buildParams.bwcVersions.withIndexCompatible({ it.onOrAfter(Version.fromString(ext.stableApiSince)) && it != VersionProperties.elasticsearchVersion }) { bwcVersion, baseName -> - BwcVersions.UnreleasedVersionInfo unreleasedVersion = BuildParams.bwcVersions.unreleasedInfo(bwcVersion) + BwcVersions.UnreleasedVersionInfo unreleasedVersion = buildParams.bwcVersions.unreleasedInfo(bwcVersion) configurations { "oldJar${baseName}" { @@ -33,12 +33,12 @@ BuildParams.bwcVersions.withIndexCompatible({ it.onOrAfter(Version.fromString(ex if (unreleasedVersion) { // For unreleased snapshot versions, build them from source "oldJar${baseName}"(files(project(unreleasedVersion.gradleProjectPath).tasks.named(buildBwcTaskName(project.name)))) - } else if(bwcVersion.onOrAfter('8.7.0') && project.name.endsWith("elasticsearch-logging")==false) { + } else if(bwcVersion.onOrAfter('8.7.0') && project.name.endsWith("logging")==false) { //there was a package rename in 8.7.0, except for es-logging - "oldJar${baseName}"("org.elasticsearch.plugin:${project.name}:${bwcVersion}") + "oldJar${baseName}"("org.elasticsearch.plugin:elasticsearch-${project.name}:${bwcVersion}") } else { // For released versions, download it - "oldJar${baseName}"("org.elasticsearch:${project.name}:${bwcVersion}") + "oldJar${baseName}"("org.elasticsearch:elasticsearch-${project.name}:${bwcVersion}") } } diff --git a/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntFixtureStop.groovy b/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntFixtureStop.groovy index 658e2623cbbd7..6c87149095186 100644 --- a/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntFixtureStop.groovy +++ b/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntFixtureStop.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal @@ -14,16 +15,12 @@ import org.elasticsearch.gradle.internal.test.AntFixture import org.gradle.api.file.FileSystemOperations import org.gradle.api.file.ProjectLayout import org.gradle.api.provider.ProviderFactory -import org.gradle.api.tasks.Internal import org.gradle.process.ExecOperations import javax.inject.Inject abstract class AntFixtureStop extends LoggedExec implements FixtureStop { - @Internal - AntFixture fixture - @Inject AntFixtureStop(ProjectLayout projectLayout, ExecOperations execOperations, @@ -33,12 +30,12 @@ abstract class AntFixtureStop extends LoggedExec implements FixtureStop { } void setFixture(AntFixture fixture) { - assert this.fixture == null - this.fixture = fixture; - final Object pid = "${-> this.fixture.pid}" - onlyIf("pidFile exists") { fixture.pidFile.exists() } + def pidFile = fixture.pidFile + def fixtureName = fixture.name + final Object pid = "${-> Integer.parseInt(pidFile.getText('UTF-8').trim())}" + onlyIf("pidFile exists") { pidFile.exists() } doFirst { - logger.info("Shutting down ${fixture.name} with pid ${pid}") + logger.info("Shutting down ${fixtureName} with pid ${pid}") } if (OS.current() == OS.WINDOWS) { @@ -50,9 +47,8 @@ abstract class AntFixtureStop extends LoggedExec implements FixtureStop { } doLast { fileSystemOperations.delete { - it.delete(fixture.pidFile) + it.delete(pidFile) } } - this.fixture = fixture } } diff --git a/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntTask.groovy b/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntTask.groovy index 69331af5ca18f..01a3bdaee2337 100644 --- a/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntTask.groovy +++ b/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntTask.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal @@ -28,11 +29,6 @@ import java.nio.charset.Charset */ public abstract class AntTask extends DefaultTask { - /** - * A buffer that will contain the output of the ant code run, - * if the output was not already written directly to stdout. - */ - public final ByteArrayOutputStream outputBuffer = new ByteArrayOutputStream() @Inject protected FileSystemOperations getFileSystemOperations() { @@ -56,6 +52,11 @@ public abstract class AntTask extends DefaultTask { // otherwise groovy replaces System.out, and you have no chance to debug // ant.saveStreams = false + /** + * A buffer that will contain the output of the ant code run, + * if the output was not already written directly to stdout. + */ + ByteArrayOutputStream outputBuffer = new ByteArrayOutputStream() final int outputLevel = logger.isDebugEnabled() ? Project.MSG_DEBUG : Project.MSG_INFO final PrintStream stream = useStdout() ? System.out : new PrintStream(outputBuffer, true, Charset.defaultCharset().name()) diff --git a/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/test/AntFixture.groovy b/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/test/AntFixture.groovy index c339916a5c46a..88a68f1194858 100644 --- a/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/test/AntFixture.groovy +++ b/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/test/AntFixture.groovy @@ -1,30 +1,46 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test import org.elasticsearch.gradle.OS + import org.elasticsearch.gradle.internal.AntFixtureStop import org.elasticsearch.gradle.internal.AntTask +import org.elasticsearch.gradle.testclusters.TestClusterInfo +import org.elasticsearch.gradle.testclusters.TestClusterValueSource +import org.elasticsearch.gradle.testclusters.TestClustersRegistry import org.gradle.api.GradleException +import org.gradle.api.file.ProjectLayout +import org.gradle.api.provider.Property +import org.gradle.api.provider.Provider +import org.gradle.api.provider.ProviderFactory +import org.gradle.api.provider.ValueSource +import org.gradle.api.provider.ValueSourceParameters +import org.gradle.api.tasks.Input import org.gradle.api.tasks.Internal import org.gradle.api.tasks.TaskProvider +import javax.inject.Inject + /** * A fixture for integration tests which runs in a separate process launched by Ant. */ -class AntFixture extends AntTask implements Fixture { +class AntFixture extends AntTask { /** The path to the executable that starts the fixture. */ @Internal String executable private final List arguments = new ArrayList<>() + private ProjectLayout projectLayout + private final ProviderFactory providerFactory void args(Object... args) { arguments.addAll(args) @@ -68,19 +84,14 @@ class AntFixture extends AntTask implements Fixture { return tmpFile.exists() } - private final TaskProvider stopTask - - AntFixture() { - stopTask = createStopTask() + @Inject + AntFixture(ProjectLayout projectLayout, ProviderFactory providerFactory) { + this.providerFactory = providerFactory + this.projectLayout = projectLayout; + TaskProvider stopTask = createStopTask() finalizedBy(stopTask) } - @Override - @Internal - TaskProvider getStopTask() { - return stopTask - } - @Override protected void runAnt(AntBuilder ant) { // reset everything @@ -230,7 +241,7 @@ class AntFixture extends AntTask implements Fixture { */ @Internal protected File getBaseDir() { - return new File(project.buildDir, "fixtures/${name}") + return new File(projectLayout.getBuildDirectory().getAsFile().get(), "fixtures/${name}") } /** Returns the working directory for the process. Defaults to "cwd" inside baseDir. */ @@ -241,7 +252,7 @@ class AntFixture extends AntTask implements Fixture { /** Returns the file the process writes its pid to. Defaults to "pid" inside baseDir. */ @Internal - protected File getPidFile() { + File getPidFile() { return new File(baseDir, 'pid') } @@ -263,6 +274,12 @@ class AntFixture extends AntTask implements Fixture { return portsFile.readLines("UTF-8").get(0) } + @Internal + Provider getAddressAndPortProvider() { + File thePortFile = portsFile + return providerFactory.provider(() -> thePortFile.readLines("UTF-8").get(0)) + } + /** Returns a file that wraps around the actual command when {@code spawn == true}. */ @Internal protected File getWrapperScript() { @@ -280,4 +297,22 @@ class AntFixture extends AntTask implements Fixture { protected File getRunLog() { return new File(cwd, 'run.log') } + + @Internal + Provider getAddressAndPortSource() { + return providerFactory.of(AntFixtureValueSource.class, spec -> { + spec.getParameters().getPortFile().set(portsFile); + }); + } + + static abstract class AntFixtureValueSource implements ValueSource { + @Override + String obtain() { + return getParameters().getPortFile().map { it.readLines("UTF-8").get(0) }.get() + } + + interface Parameters extends ValueSourceParameters { + Property getPortFile(); + } + } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java index 63147040a289d..49887dac5b6fd 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; @@ -11,7 +12,7 @@ import groovy.lang.Closure; import org.elasticsearch.gradle.internal.conventions.util.Util; -import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.BuildParameterExtension; import org.elasticsearch.gradle.internal.precommit.JarHellPrecommitPlugin; import org.elasticsearch.gradle.internal.test.HistoricalFeaturesMetadataPlugin; import org.elasticsearch.gradle.plugin.PluginBuildPlugin; @@ -38,6 +39,7 @@ public void apply(Project project) { project.getPluginManager().apply(JarHellPrecommitPlugin.class); project.getPluginManager().apply(ElasticsearchJavaPlugin.class); project.getPluginManager().apply(HistoricalFeaturesMetadataPlugin.class); + boolean isCi = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class).isCi(); // Clear default dependencies added by public PluginBuildPlugin as we add our // own project dependencies for internal builds // TODO remove once we removed default dependencies from PluginBuildPlugin @@ -53,7 +55,7 @@ public void apply(Project project) { .set("addQaCheckDependencies", new Closure(BaseInternalPluginBuildPlugin.this, BaseInternalPluginBuildPlugin.this) { public void doCall(Project proj) { // This is only a convenience for local developers so make this a noop when running in CI - if (BuildParams.isCi() == false) { + if (isCi == false) { proj.afterEvaluate(project1 -> { // let check depend on check tasks of qa sub-projects final var checkTaskProvider = project1.getTasks().named("check"); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BuildPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BuildPlugin.java index 6c7bc6753531c..75984e1bc6998 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BuildPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BuildPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; @@ -32,7 +33,7 @@ */ public class BuildPlugin implements Plugin { - public static final String SSPL_LICENSE_PATH = "licenses/SSPL-1.0+ELASTIC-LICENSE-2.0.txt"; + public static final String LICENSE_PATH = "licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt"; private final BuildLayout buildLayout; private final ObjectFactory objectFactory; @@ -101,7 +102,7 @@ private void updateJarTasksMetaInf(Project project) { } private void configureLicenseDefaultConvention(RegularFileProperty licenseFileProperty) { - File licenseFileDefault = new File(buildLayout.getRootDirectory(), SSPL_LICENSE_PATH); + File licenseFileDefault = new File(buildLayout.getRootDirectory(), LICENSE_PATH); licenseFileProperty.convention(projectLayout.file(providerFactory.provider(() -> licenseFileDefault))); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcGitExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcGitExtension.java index 4f4b8f0dbfdeb..a7dc027400c09 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcGitExtension.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcGitExtension.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java index 4112d96c7296b..d7bf839817e12 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; @@ -12,7 +13,6 @@ import org.elasticsearch.gradle.LoggedExec; import org.elasticsearch.gradle.OS; import org.elasticsearch.gradle.Version; -import org.elasticsearch.gradle.internal.info.BuildParams; import org.gradle.api.Action; import org.gradle.api.GradleException; import org.gradle.api.Project; @@ -46,6 +46,7 @@ public class BwcSetupExtension { private final ProviderFactory providerFactory; private final JavaToolchainService toolChainService; private final Provider unreleasedVersionInfo; + private final Boolean isCi; private Provider checkoutDir; @@ -55,7 +56,8 @@ public BwcSetupExtension( ProviderFactory providerFactory, JavaToolchainService toolChainService, Provider unreleasedVersionInfo, - Provider checkoutDir + Provider checkoutDir, + Boolean isCi ) { this.project = project; this.objectFactory = objectFactory; @@ -63,6 +65,7 @@ public BwcSetupExtension( this.toolChainService = toolChainService; this.unreleasedVersionInfo = unreleasedVersionInfo; this.checkoutDir = checkoutDir; + this.isCi = isCi; } TaskProvider bwcTask(String name, Action configuration) { @@ -79,7 +82,8 @@ TaskProvider bwcTask(String name, Action configuration, toolChainService, name, configuration, - useUniqueUserHome + useUniqueUserHome, + isCi ); } @@ -92,7 +96,8 @@ private static TaskProvider createRunBwcGradleTask( JavaToolchainService toolChainService, String name, Action configAction, - boolean useUniqueUserHome + boolean useUniqueUserHome, + boolean isCi ) { return project.getTasks().register(name, LoggedExec.class, loggedExec -> { loggedExec.dependsOn("checkoutBwcBranch"); @@ -103,7 +108,7 @@ private static TaskProvider createRunBwcGradleTask( spec.getParameters().getCheckoutDir().set(checkoutDir); }).flatMap(s -> getJavaHome(objectFactory, toolChainService, Integer.parseInt(s)))); - if (BuildParams.isCi() && OS.current() != OS.WINDOWS) { + if (isCi && OS.current() != OS.WINDOWS) { // TODO: Disabled for now until we can figure out why files are getting corrupted // loggedExec.getEnvironment().put("GRADLE_RO_DEP_CACHE", System.getProperty("user.home") + "/gradle_ro_cache"); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcVersions.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcVersions.java index 41bfddb01e665..cb93d2498327e 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcVersions.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcVersions.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; @@ -12,6 +13,7 @@ import org.elasticsearch.gradle.Version; import org.elasticsearch.gradle.VersionProperties; +import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -21,7 +23,6 @@ import java.util.Optional; import java.util.Set; import java.util.TreeMap; -import java.util.TreeSet; import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Predicate; @@ -29,7 +30,9 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; +import static java.util.Collections.reverseOrder; import static java.util.Collections.unmodifiableList; +import static java.util.Comparator.comparing; /** * A container for elasticsearch supported version information used in BWC testing. @@ -62,23 +65,23 @@ * We are then able to map the unreleased version to branches in git and Gradle projects that are capable of checking * out and building them, so we can include these in the testing plan as well. */ -public class BwcVersions { + +public class BwcVersions implements Serializable { private static final Pattern LINE_PATTERN = Pattern.compile( "\\W+public static final Version V_(\\d+)_(\\d+)_(\\d+)(_alpha\\d+|_beta\\d+|_rc\\d+)?.*\\);" ); - private static final Version MINIMUM_WIRE_COMPATIBLE_VERSION = Version.fromString("7.17.0"); private static final String GLIBC_VERSION_ENV_VAR = "GLIBC_VERSION"; private final Version currentVersion; - private final List versions; + private final transient List versions; private final Map unreleased; - public BwcVersions(List versionLines) { - this(versionLines, Version.fromString(VersionProperties.getElasticsearch())); + public BwcVersions(List versionLines, List developmentBranches) { + this(versionLines, Version.fromString(VersionProperties.getElasticsearch()), developmentBranches); } - public BwcVersions(Version currentVersionProperty, List allVersions) { + public BwcVersions(Version currentVersionProperty, List allVersions, List developmentBranches) { if (allVersions.isEmpty()) { throw new IllegalArgumentException("Could not parse any versions"); } @@ -87,12 +90,12 @@ public BwcVersions(Version currentVersionProperty, List allVersions) { this.currentVersion = allVersions.get(allVersions.size() - 1); assertCurrentVersionMatchesParsed(currentVersionProperty); - this.unreleased = computeUnreleased(); + this.unreleased = computeUnreleased(developmentBranches); } // Visible for testing - BwcVersions(List versionLines, Version currentVersionProperty) { - this(currentVersionProperty, parseVersionLines(versionLines)); + BwcVersions(List versionLines, Version currentVersionProperty, List developmentBranches) { + this(currentVersionProperty, parseVersionLines(versionLines), developmentBranches); } private static List parseVersionLines(List versionLines) { @@ -129,51 +132,77 @@ public void forPreviousUnreleased(Consumer consumer) { ).stream().map(unreleased::get).forEach(consumer); } - private String getBranchFor(Version version) { - if (version.equals(currentVersion)) { - // Just assume the current branch is 'main'. It's actually not important, we never check out the current branch. - return "main"; - } else { + private String getBranchFor(Version version, List developmentBranches) { + // If the current version matches a specific feature freeze branch, use that + if (developmentBranches.contains(version.getMajor() + "." + version.getMinor())) { return version.getMajor() + "." + version.getMinor(); + } else if (developmentBranches.contains(version.getMajor() + ".x")) { // Otherwise if an n.x branch exists and we are that major + return version.getMajor() + ".x"; + } else { // otherwise we're the main branch + return "main"; } } - private Map computeUnreleased() { - Set unreleased = new TreeSet<>(); - // The current version is being worked, is always unreleased - unreleased.add(currentVersion); - // Recurse for all unreleased versions starting from the current version - addUnreleased(unreleased, currentVersion, 0); + private Map computeUnreleased(List developmentBranches) { + Map result = new TreeMap<>(); - // Grab the latest version from the previous major if necessary as well, this is going to be a maintenance release - Version maintenance = versions.stream() - .filter(v -> v.getMajor() == currentVersion.getMajor() - 1) - .max(Comparator.naturalOrder()) - .orElseThrow(); - // This is considered the maintenance release only if we haven't yet encountered it - boolean hasMaintenanceRelease = unreleased.add(maintenance); + // The current version is always in development + String currentBranch = getBranchFor(currentVersion, developmentBranches); + result.put(currentVersion, new UnreleasedVersionInfo(currentVersion, currentBranch, ":distribution")); + + // Check for an n.x branch as well + if (currentBranch.equals("main") && developmentBranches.stream().anyMatch(s -> s.endsWith(".x"))) { + // This should correspond to the latest new minor + Version version = versions.stream() + .sorted(Comparator.reverseOrder()) + .filter(v -> v.getMajor() == (currentVersion.getMajor() - 1) && v.getRevision() == 0) + .findFirst() + .orElseThrow(() -> new IllegalStateException("Unable to determine development version for branch")); + String branch = getBranchFor(version, developmentBranches); + assert branch.equals(currentVersion.getMajor() - 1 + ".x") : "Expected branch does not match development branch"; + + result.put(version, new UnreleasedVersionInfo(version, branch, ":distribution:bwc:minor")); + } - List unreleasedList = unreleased.stream().sorted(Comparator.reverseOrder()).toList(); - Map result = new TreeMap<>(); - for (int i = 0; i < unreleasedList.size(); i++) { - Version esVersion = unreleasedList.get(i); - // This is either a new minor or staged release - if (currentVersion.equals(esVersion)) { - result.put(esVersion, new UnreleasedVersionInfo(esVersion, getBranchFor(esVersion), ":distribution")); - } else if (esVersion.getRevision() == 0) { - // If there are two upcoming unreleased minors then this one is the new minor - if (unreleasedList.get(i + 1).getRevision() == 0) { - result.put(esVersion, new UnreleasedVersionInfo(esVersion, getBranchFor(esVersion), ":distribution:bwc:minor")); - } else { - result.put(esVersion, new UnreleasedVersionInfo(esVersion, getBranchFor(esVersion), ":distribution:bwc:staged")); - } - } else { - // If this is the oldest unreleased version and we have a maintenance release - if (i == unreleasedList.size() - 1 && hasMaintenanceRelease) { - result.put(esVersion, new UnreleasedVersionInfo(esVersion, getBranchFor(esVersion), ":distribution:bwc:maintenance")); - } else { - result.put(esVersion, new UnreleasedVersionInfo(esVersion, getBranchFor(esVersion), ":distribution:bwc:bugfix")); - } + // Now handle all the feature freeze branches + List featureFreezeBranches = developmentBranches.stream() + .filter(b -> Pattern.matches("[0-9]+\\.[0-9]+", b)) + .sorted(reverseOrder(comparing(s -> Version.fromString(s, Version.Mode.RELAXED)))) + .toList(); + + boolean existingBugfix = false; + for (int i = 0; i < featureFreezeBranches.size(); i++) { + String branch = featureFreezeBranches.get(i); + Version version = versions.stream() + .sorted(Comparator.reverseOrder()) + .filter(v -> v.toString().startsWith(branch)) + .findFirst() + .orElse(null); + + // If we don't know about this version we can ignore it + if (version == null) { + continue; + } + + // If this is the current version we can ignore as we've already handled it + if (version.equals(currentVersion)) { + continue; + } + + // We only maintain compatibility back one major so ignore anything older + if (currentVersion.getMajor() - version.getMajor() > 1) { + continue; + } + + // This is the maintenance version + if (i == featureFreezeBranches.size() - 1) { + result.put(version, new UnreleasedVersionInfo(version, branch, ":distribution:bwc:maintenance")); + } else if (version.getRevision() == 0) { // This is the next staged minor + result.put(version, new UnreleasedVersionInfo(version, branch, ":distribution:bwc:staged")); + } else { // This is a bugfix + String project = existingBugfix ? "bugfix2" : "bugfix"; + result.put(version, new UnreleasedVersionInfo(version, branch, ":distribution:bwc:" + project)); + existingBugfix = true; } } @@ -222,7 +251,10 @@ public void compareToAuthoritative(List authoritativeReleasedVersions) } private List getReleased() { - return versions.stream().filter(v -> unreleased.containsKey(v) == false).toList(); + return versions.stream() + .filter(v -> v.getMajor() >= currentVersion.getMajor() - 1) + .filter(v -> unreleased.containsKey(v) == false) + .toList(); } /** @@ -248,7 +280,7 @@ public void withIndexCompatible(Predicate filter, BiConsumer getWireCompatible() { - return filterSupportedVersions(versions.stream().filter(v -> v.compareTo(MINIMUM_WIRE_COMPATIBLE_VERSION) >= 0).toList()); + return filterSupportedVersions(versions.stream().filter(v -> v.compareTo(getMinimumWireCompatibleVersion()) >= 0).toList()); } public void withWireCompatible(BiConsumer versionAction) { @@ -286,7 +318,17 @@ public List getUnreleasedWireCompatible() { } public Version getMinimumWireCompatibleVersion() { - return MINIMUM_WIRE_COMPATIBLE_VERSION; + // Determine minimum wire compatible version from list of known versions. + // Current BWC policy states the minimum wire compatible version is the last minor release or the previous major version. + return versions.stream() + .filter(v -> v.getRevision() == 0) + .filter(v -> v.getMajor() == currentVersion.getMajor() - 1) + .max(Comparator.naturalOrder()) + .orElseThrow(() -> new IllegalStateException("Unable to determine minimum wire compatible version.")); + } + + public Version getCurrentVersion() { + return currentVersion; } public record UnreleasedVersionInfo(Version version, String branch, String gradleProjectPath) {} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ConcatFilesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ConcatFilesTask.java index 55634c15f0524..c84b52c612256 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ConcatFilesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ConcatFilesTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoPlugin.java index 2c20b6e97d5f6..9bff87497c573 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoTask.java index 4941140dc6ce7..3ccae89139a12 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DistributionArchive.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DistributionArchive.java index 1c7f8f9259a62..fb51aaae8abad 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DistributionArchive.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DistributionArchive.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DistributionArchiveCheckExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DistributionArchiveCheckExtension.java index b1208740c2608..ebaac6bc984e7 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DistributionArchiveCheckExtension.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DistributionArchiveCheckExtension.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java index 3af59f00299b8..ae9decf668f04 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; @@ -20,19 +21,18 @@ public enum DockerBase { // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum"), - // Base image with extras for Cloud - CLOUD("ubuntu:20.04", "-cloud", "apt-get"), - - // Based on CLOUD above, with more extras. We don't set a base image because - // we programmatically extend from the Cloud image. - CLOUD_ESS(null, "-cloud-ess", "apt-get"), - // Chainguard based wolfi image with latest jdk - WOLFI( - "docker.elastic.co/wolfi/chainguard-base:latest@sha256:c16d3ad6cebf387e8dd2ad769f54320c4819fbbaa21e729fad087c7ae223b4d0", + // This is usually updated via renovatebot + // spotless:off + WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:bfdeddb33330a281950c2a54adef991dbbe6a42832bc505d13b11beaf50ae73f", "-wolfi", "apk" - ); + ), + // spotless:on + + // Based on WOLFI above, with more extras. We don't set a base image because + // we programmatically extend from the wolfi image. + CLOUD_ESS(null, "-cloud-ess", "apk"); private final String image; private final String suffix; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java index b27f480df4e63..b1207a2f5161d 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; @@ -14,6 +15,7 @@ import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; import org.apache.commons.io.IOUtils; +import org.elasticsearch.gradle.OS; import org.elasticsearch.gradle.util.GradleUtils; import org.gradle.api.Plugin; import org.gradle.api.Project; @@ -27,6 +29,8 @@ import org.gradle.api.provider.Property; import org.gradle.api.tasks.Input; import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; @@ -45,6 +49,8 @@ public abstract class ElasticsearchBuildCompletePlugin implements Plugin { + private static final Logger log = LoggerFactory.getLogger(ElasticsearchBuildCompletePlugin.class); + @Inject protected abstract FlowScope getFlowScope(); @@ -60,7 +66,7 @@ public void apply(Project target) { ? System.getenv("BUILD_NUMBER") : System.getenv("BUILDKITE_BUILD_NUMBER"); String performanceTest = System.getenv("BUILD_PERFORMANCE_TEST"); - if (buildNumber != null && performanceTest == null && GradleUtils.isIncludedBuild(target) == false) { + if (buildNumber != null && performanceTest == null && GradleUtils.isIncludedBuild(target) == false && OS.current() != OS.WINDOWS) { File targetFile = calculateTargetFile(target, buildNumber); File projectDir = target.getProjectDir(); File gradleWorkersDir = new File(target.getGradle().getGradleUserHomeDir(), "workers/"); @@ -145,12 +151,17 @@ interface Parameters extends FlowParameters { @SuppressWarnings("checkstyle:DescendantToken") @Override public void execute(BuildFinishedFlowAction.Parameters parameters) throws FileNotFoundException { + List filesToArchive = parameters.getFilteredFiles().get(); + if (filesToArchive.isEmpty()) { + return; + } File uploadFile = parameters.getUploadFile().get(); if (uploadFile.exists()) { getFileSystemOperations().delete(spec -> spec.delete(uploadFile)); } uploadFile.getParentFile().mkdirs(); - createBuildArchiveTar(parameters.getFilteredFiles().get(), parameters.getProjectDir().get(), uploadFile); + + createBuildArchiveTar(filesToArchive, parameters.getProjectDir().get(), uploadFile); if (uploadFile.exists() && "true".equals(System.getenv("BUILDKITE"))) { String uploadFilePath = uploadFile.getName(); File uploadFileDir = uploadFile.getParentFile(); @@ -162,7 +173,13 @@ public void execute(BuildFinishedFlowAction.Parameters parameters) throws FileNo // So, if you change this such that the artifact will have a slash/directory in it, you'll need to update the logic // below as well pb.directory(uploadFileDir); - pb.start().waitFor(); + try { + // we are very generious here, as the upload can take + // a long time depending on its size + pb.start().waitFor(30, java.util.concurrent.TimeUnit.MINUTES); + } catch (InterruptedException e) { + System.out.println("Failed to upload buildkite artifact " + e.getMessage()); + } System.out.println("Generating buildscan link for artifact..."); @@ -228,8 +245,11 @@ private static void createBuildArchiveTar(List files, File projectDir, Fil tOut.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU); tOut.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_STAR); for (Path path : files.stream().map(File::toPath).toList()) { - if (!Files.isRegularFile(path)) { - throw new IOException("Support only file!"); + if (Files.exists(path) == false) { + log.warn("File disappeared before it could be added to CI archive: " + path); + continue; + } else if (!Files.isRegularFile(path)) { + throw new IOException("Support only file!: " + path); } long entrySize = Files.size(path); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java index a3b1dd9731591..ee0eb3f6eb2bf 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java @@ -1,16 +1,17 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; import org.elasticsearch.gradle.VersionProperties; import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitTaskPlugin; -import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.BuildParameterExtension; import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.elasticsearch.gradle.internal.test.MutedTestPlugin; import org.elasticsearch.gradle.internal.test.TestUtil; @@ -48,6 +49,7 @@ public class ElasticsearchJavaBasePlugin implements Plugin { private final JavaToolchainService javaToolchains; + private BuildParameterExtension buildParams; @Inject ElasticsearchJavaBasePlugin(JavaToolchainService javaToolchains) { @@ -56,8 +58,10 @@ public class ElasticsearchJavaBasePlugin implements Plugin { @Override public void apply(Project project) { + project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class); // make sure the global build info plugin is applied to the root project project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class); + buildParams = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class); project.getPluginManager().apply(JavaBasePlugin.class); // common repositories setup project.getPluginManager().apply(RepositoriesSetupPlugin.class); @@ -128,14 +132,14 @@ private static void disableTransitiveDependenciesForSourceSet(Project project, S public void configureCompile(Project project) { project.getExtensions().getExtraProperties().set("compactProfile", "full"); JavaPluginExtension java = project.getExtensions().getByType(JavaPluginExtension.class); - if (BuildParams.getJavaToolChainSpec().isPresent()) { - java.toolchain(BuildParams.getJavaToolChainSpec().get()); + if (buildParams.getJavaToolChainSpec().getOrNull() != null) { + java.toolchain(buildParams.getJavaToolChainSpec().get()); } - java.setSourceCompatibility(BuildParams.getMinimumRuntimeVersion()); - java.setTargetCompatibility(BuildParams.getMinimumRuntimeVersion()); + java.setSourceCompatibility(buildParams.getMinimumRuntimeVersion()); + java.setTargetCompatibility(buildParams.getMinimumRuntimeVersion()); project.getTasks().withType(JavaCompile.class).configureEach(compileTask -> { compileTask.getJavaCompiler().set(javaToolchains.compilerFor(spec -> { - spec.getLanguageVersion().set(JavaLanguageVersion.of(BuildParams.getMinimumRuntimeVersion().getMajorVersion())); + spec.getLanguageVersion().set(JavaLanguageVersion.of(buildParams.getMinimumRuntimeVersion().getMajorVersion())); })); CompileOptions compileOptions = compileTask.getOptions(); @@ -158,7 +162,7 @@ public void configureCompile(Project project) { compileTask.getConventionMapping().map("sourceCompatibility", () -> java.getSourceCompatibility().toString()); compileTask.getConventionMapping().map("targetCompatibility", () -> java.getTargetCompatibility().toString()); compileOptions.getRelease().set(releaseVersionProviderFromCompileTask(project, compileTask)); - compileOptions.setIncremental(BuildParams.isCi() == false); + compileOptions.setIncremental(buildParams.isCi() == false); }); // also apply release flag to groovy, which is used in build-tools project.getTasks().withType(GroovyCompile.class).configureEach(compileTask -> { @@ -176,7 +180,7 @@ public static void configureInputNormalization(Project project) { } private static void configureNativeLibraryPath(Project project) { - String nativeProject = ":libs:elasticsearch-native:elasticsearch-native-libraries"; + String nativeProject = ":libs:native:native-libraries"; Configuration nativeConfig = project.getConfigurations().create("nativeLibs"); nativeConfig.defaultDependencies(deps -> { deps.add(project.getDependencies().project(Map.of("path", nativeProject, "configuration", "default"))); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaModulePathPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaModulePathPlugin.java index 662e1845850a7..90a053b2ec3d6 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaModulePathPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaModulePathPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaPlugin.java index c4437871e98bb..9c9e160a81364 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; @@ -14,8 +15,10 @@ import org.elasticsearch.gradle.VersionProperties; import org.elasticsearch.gradle.internal.conventions.util.Util; -import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.BuildParameterExtension; +import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.gradle.api.Action; +import org.gradle.api.JavaVersion; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.Task; @@ -23,6 +26,8 @@ import org.gradle.api.plugins.BasePlugin; import org.gradle.api.plugins.JavaLibraryPlugin; import org.gradle.api.plugins.JavaPlugin; +import org.gradle.api.provider.Property; +import org.gradle.api.provider.Provider; import org.gradle.api.tasks.TaskProvider; import org.gradle.api.tasks.bundling.Jar; import org.gradle.api.tasks.javadoc.Javadoc; @@ -33,6 +38,7 @@ import java.util.Map; import static org.elasticsearch.gradle.internal.conventions.util.Util.toStringable; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; /** * A wrapper around Gradle's Java plugin that applies our @@ -41,13 +47,15 @@ public class ElasticsearchJavaPlugin implements Plugin { @Override public void apply(Project project) { + project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class); + Property buildParams = loadBuildParams(project); project.getPluginManager().apply(ElasticsearchJavaBasePlugin.class); project.getPluginManager().apply(JavaLibraryPlugin.class); project.getPluginManager().apply(ElasticsearchJavaModulePathPlugin.class); // configureConfigurations(project); - configureJars(project); - configureJarManifest(project); + configureJars(project, buildParams.get()); + configureJarManifest(project, buildParams.get()); configureJavadoc(project); testCompileOnlyDeps(project); } @@ -62,7 +70,9 @@ private static void testCompileOnlyDeps(Project project) { /** * Adds additional manifest info to jars */ - static void configureJars(Project project) { + static void configureJars(Project project, BuildParameterExtension buildParams) { + String buildDate = buildParams.getBuildDate().toString(); + JavaVersion gradleJavaVersion = buildParams.getGradleJavaVersion(); project.getTasks().withType(Jar.class).configureEach(jarTask -> { // we put all our distributable files under distributions jarTask.getDestinationDirectory().set(new File(project.getBuildDir(), "distributions")); @@ -74,14 +84,11 @@ static void configureJars(Project project) { public void execute(Task task) { // this doFirst is added before the info plugin, therefore it will run // after the doFirst added by the info plugin, and we can override attributes - jarTask.getManifest() - .attributes( - Map.of("Build-Date", BuildParams.getBuildDate(), "Build-Java-Version", BuildParams.getGradleJavaVersion()) - ); + jarTask.getManifest().attributes(Map.of("Build-Date", buildDate, "Build-Java-Version", gradleJavaVersion)); } }); }); - project.getPluginManager().withPlugin("com.github.johnrengelman.shadow", p -> { + project.getPluginManager().withPlugin("com.gradleup.shadow", p -> { project.getTasks().withType(ShadowJar.class).configureEach(shadowJar -> { /* * Replace the default "-all" classifier with null @@ -101,10 +108,13 @@ public void execute(Task task) { }); } - private static void configureJarManifest(Project project) { + private static void configureJarManifest(Project project, BuildParameterExtension buildParams) { + Provider gitOrigin = buildParams.getGitOrigin(); + Provider gitRevision = buildParams.getGitRevision(); + project.getPlugins().withType(InfoBrokerPlugin.class).whenPluginAdded(manifestPlugin -> { - manifestPlugin.add("Module-Origin", toStringable(BuildParams::getGitOrigin)); - manifestPlugin.add("Change", toStringable(BuildParams::getGitRevision)); + manifestPlugin.add("Module-Origin", toStringable(() -> gitOrigin.get())); + manifestPlugin.add("Change", toStringable(() -> gitRevision.get())); manifestPlugin.add("X-Compile-Elasticsearch-Version", toStringable(VersionProperties::getElasticsearch)); manifestPlugin.add("X-Compile-Lucene-Version", toStringable(VersionProperties::getLucene)); manifestPlugin.add( diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavadocPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavadocPlugin.java index 6651fca81067a..aca310cbf1e47 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavadocPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavadocPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; @@ -81,12 +82,15 @@ private void configureJavadocForConfiguration(Project project, boolean shadow, C .sorted(Comparator.comparing(Dependency::getGroup)) .filter(d -> d instanceof ProjectDependency) .map(d -> (ProjectDependency) d) - .filter(p -> p.getDependencyProject() != null) .forEach(projectDependency -> configureDependency(project, shadow, projectDependency)); } private void configureDependency(Project project, boolean shadowed, ProjectDependency dep) { - var upstreamProject = dep.getDependencyProject(); + // we should use variant aware dependency management to resolve artifacts required for javadoc here + Project upstreamProject = project.project(dep.getPath()); + if (upstreamProject == null) { + return; + } if (shadowed) { /* * Include the source of shadowed upstream projects so we don't diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java index 2d6964c041fe2..703fc82cab819 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; @@ -12,7 +13,6 @@ import org.elasticsearch.gradle.OS; import org.elasticsearch.gradle.internal.conventions.util.Util; -import org.elasticsearch.gradle.internal.info.BuildParams; import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.elasticsearch.gradle.internal.test.ErrorReportingTestListener; import org.elasticsearch.gradle.internal.test.SimpleCommandLineArgumentProvider; @@ -36,6 +36,7 @@ import javax.inject.Inject; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; import static org.elasticsearch.gradle.util.FileUtils.mkdirs; import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure; @@ -51,6 +52,8 @@ public abstract class ElasticsearchTestBasePlugin implements Plugin { @Override public void apply(Project project) { + project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class); + var buildParams = loadBuildParams(project); project.getPluginManager().apply(GradleTestPolicySetupPlugin.class); // for fips mode check project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class); @@ -92,14 +95,14 @@ public void execute(Task t) { mkdirs(test.getWorkingDir().toPath().resolve("temp").toFile()); // TODO remove once jvm.options are added to test system properties - test.systemProperty("java.locale.providers", "SPI,CLDR"); + test.systemProperty("java.locale.providers", "CLDR"); } }); test.getJvmArgumentProviders().add(nonInputProperties); test.getExtensions().add("nonInputProperties", nonInputProperties); test.setWorkingDir(project.file(project.getBuildDir() + "/testrun/" + test.getName().replace("#", "_"))); - test.setMaxParallelForks(Integer.parseInt(System.getProperty("tests.jvms", BuildParams.getDefaultParallel().toString()))); + test.setMaxParallelForks(Integer.parseInt(System.getProperty("tests.jvms", buildParams.get().getDefaultParallel().toString()))); test.exclude("**/*$*.class"); @@ -107,6 +110,7 @@ public void execute(Task t) { "-Xmx" + System.getProperty("tests.heap.size", "512m"), "-Xms" + System.getProperty("tests.heap.size", "512m"), "-Djava.security.manager=allow", + "-Dtests.testfeatures.enabled=true", "--add-opens=java.base/java.util=ALL-UNNAMED", // TODO: only open these for mockito when it is modularized "--add-opens=java.base/java.security.cert=ALL-UNNAMED", @@ -144,9 +148,9 @@ public void execute(Task t) { // ignore changing test seed when build is passed -Dignore.tests.seed for cacheability experimentation if (System.getProperty("ignore.tests.seed") != null) { - nonInputProperties.systemProperty("tests.seed", BuildParams.getTestSeed()); + nonInputProperties.systemProperty("tests.seed", buildParams.get().getTestSeed()); } else { - test.systemProperty("tests.seed", BuildParams.getTestSeed()); + test.systemProperty("tests.seed", buildParams.get().getTestSeed()); } // don't track these as inputs since they contain absolute paths and break cache relocatability @@ -183,15 +187,15 @@ public void execute(Task t) { }); if (OS.current().equals(OS.WINDOWS) && System.getProperty("tests.timeoutSuite") == null) { - // override the suite timeout to 30 mins for windows, because it has the most inefficient filesystem known to man - test.systemProperty("tests.timeoutSuite", "2400000!"); + // override the suite timeout to 60 mins for windows, because it has the most inefficient filesystem known to man + test.systemProperty("tests.timeoutSuite", "3600000!"); } /* * If this project builds a shadow JAR than any unit tests should test against that artifact instead of * compiled class output and dependency jars. This better emulates the runtime environment of consumers. */ - project.getPluginManager().withPlugin("com.github.johnrengelman.shadow", p -> { + project.getPluginManager().withPlugin("com.gradleup.shadow", p -> { if (test.getName().equals(JavaPlugin.TEST_TASK_NAME)) { // Remove output class files and any other dependencies from the test classpath, since the shadow JAR includes these SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmbeddedProviderExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmbeddedProviderExtension.java index 03b8f19d10b13..bc2f4bd6b31b1 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmbeddedProviderExtension.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmbeddedProviderExtension.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmbeddedProviderPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmbeddedProviderPlugin.java index 99186c6221443..2515afdc0f484 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmbeddedProviderPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmbeddedProviderPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmptyDirTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmptyDirTask.java index 15a224b0ff206..13cac20b34306 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmptyDirTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmptyDirTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ExportElasticsearchBuildResourcesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ExportElasticsearchBuildResourcesTask.java index 64b9203f8dc4b..dcf34a1e8af56 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ExportElasticsearchBuildResourcesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ExportElasticsearchBuildResourcesTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/FixtureStop.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/FixtureStop.java index cc5a3bc2bcf2c..9dc34e06a49e2 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/FixtureStop.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/FixtureStop.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/GenerateProviderManifest.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/GenerateProviderManifest.java index 621210cd935b2..854c60d983124 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/GenerateProviderManifest.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/GenerateProviderManifest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalAvailableTcpPortProviderPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalAvailableTcpPortProviderPlugin.java index 85e56f8e2d946..faa715085d745 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalAvailableTcpPortProviderPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalAvailableTcpPortProviderPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java index 7add1e615f577..c5a87b7d84344 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveCheckPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveCheckPlugin.java index 94b1c70f29650..805a91b03ee4e 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveCheckPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveCheckPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; @@ -154,7 +155,7 @@ private TaskProvider registerCheckLicenseTask(Project project, TaskProvide public void execute(Task task) { String licenseFilename = null; if (project.getName().contains("oss-") || project.getName().equals("integ-test-zip")) { - licenseFilename = "SSPL-1.0+ELASTIC-LICENSE-2.0.txt"; + licenseFilename = "AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt"; } else { licenseFilename = "ELASTIC-LICENSE-2.0.txt"; } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java index d10cecf7fa50e..c2547b72e21fa 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; @@ -75,12 +76,14 @@ private void registerAndConfigureDistributionArchivesExtension(Project project) sub.getArtifacts().add(DEFAULT_CONFIGURATION_NAME, distributionArchive.getArchiveTask()); var extractedConfiguration = sub.getConfigurations().create(EXTRACTED_CONFIGURATION_NAME); extractedConfiguration.setCanBeResolved(false); + extractedConfiguration.setCanBeConsumed(true); extractedConfiguration.getAttributes() .attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE); sub.getArtifacts().add(EXTRACTED_CONFIGURATION_NAME, distributionArchive.getExpandedDistTask()); // The "composite" configuration is specifically used for resolving transformed artifacts in an included build var compositeConfiguration = sub.getConfigurations().create(COMPOSITE_CONFIGURATION_NAME); compositeConfiguration.setCanBeResolved(false); + compositeConfiguration.setCanBeConsumed(true); compositeConfiguration.getAttributes() .attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE); compositeConfiguration.getAttributes().attribute(Attribute.of("composite", Boolean.class), true); @@ -131,14 +134,14 @@ private void registerEmptyDirectoryTasks(Project project) { }); File pluginsDir = new File(project.getBuildDir(), "plugins-hack/plugins"); - project.getExtensions().add("pluginsDir", pluginsDir); + project.getExtensions().getExtraProperties().set("pluginsDir", pluginsDir); project.getTasks().register("createPluginsDir", EmptyDirTask.class, t -> { t.setDir(pluginsDir); t.setDirMode(0755); }); File jvmOptionsDir = new File(project.getBuildDir(), "jvm-options-hack/jvm.options.d"); - project.getExtensions().add("jvmOptionsDir", jvmOptionsDir); + project.getExtensions().getExtraProperties().set("jvmOptionsDir", jvmOptionsDir); project.getTasks().register("createJvmOptionsDir", EmptyDirTask.class, t -> { t.setDir(jvmOptionsDir); t.setDirMode(0750); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java index a2247adcf7b9e..28ce3df6c7815 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java @@ -1,27 +1,28 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; import org.elasticsearch.gradle.Version; -import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.BuildParameterExtension; import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.gradle.api.Action; import org.gradle.api.InvalidUserDataException; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.Task; +import org.gradle.api.file.FileSystemOperations; import org.gradle.api.file.ProjectLayout; import org.gradle.api.model.ObjectFactory; import org.gradle.api.plugins.JvmToolchainsPlugin; import org.gradle.api.provider.Provider; import org.gradle.api.provider.ProviderFactory; -import org.gradle.api.tasks.Copy; import org.gradle.api.tasks.TaskProvider; import org.gradle.jvm.toolchain.JavaToolchainService; import org.gradle.language.base.plugins.LifecycleBasePlugin; @@ -38,6 +39,7 @@ import static java.util.Arrays.asList; import static java.util.Arrays.stream; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; /** * We want to be able to do BWC tests for unreleased versions without relying on and waiting for snapshots. @@ -51,11 +53,17 @@ public class InternalDistributionBwcSetupPlugin implements Plugin { private final ObjectFactory objectFactory; private ProviderFactory providerFactory; private JavaToolchainService toolChainService; + private FileSystemOperations fileSystemOperations; @Inject - public InternalDistributionBwcSetupPlugin(ObjectFactory objectFactory, ProviderFactory providerFactory) { + public InternalDistributionBwcSetupPlugin( + ObjectFactory objectFactory, + ProviderFactory providerFactory, + FileSystemOperations fileSystemOperations + ) { this.objectFactory = objectFactory; this.providerFactory = providerFactory; + this.fileSystemOperations = fileSystemOperations; } @Override @@ -63,23 +71,31 @@ public void apply(Project project) { project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class); project.getPlugins().apply(JvmToolchainsPlugin.class); toolChainService = project.getExtensions().getByType(JavaToolchainService.class); - BuildParams.getBwcVersions().forPreviousUnreleased((BwcVersions.UnreleasedVersionInfo unreleasedVersion) -> { + var buildParams = loadBuildParams(project).get(); + Boolean isCi = buildParams.isCi(); + buildParams.getBwcVersions().forPreviousUnreleased((BwcVersions.UnreleasedVersionInfo unreleasedVersion) -> { configureBwcProject( project.project(unreleasedVersion.gradleProjectPath()), + buildParams, unreleasedVersion, providerFactory, objectFactory, - toolChainService + toolChainService, + isCi, + fileSystemOperations ); }); } private static void configureBwcProject( Project project, + BuildParameterExtension buildParams, BwcVersions.UnreleasedVersionInfo versionInfo, ProviderFactory providerFactory, ObjectFactory objectFactory, - JavaToolchainService toolChainService + JavaToolchainService toolChainService, + Boolean isCi, + FileSystemOperations fileSystemOperations ) { ProjectLayout layout = project.getLayout(); Provider versionInfoProvider = providerFactory.provider(() -> versionInfo); @@ -95,7 +111,8 @@ private static void configureBwcProject( providerFactory, toolChainService, versionInfoProvider, - checkoutDir + checkoutDir, + isCi ); BwcGitExtension gitExtension = project.getPlugins().apply(InternalBwcGitPlugin.class).getGitExtension(); Provider bwcVersion = versionInfoProvider.map(info -> info.version()); @@ -110,17 +127,27 @@ private static void configureBwcProject( List distributionProjects = resolveArchiveProjects(checkoutDir.get(), bwcVersion.get()); // Setup gradle user home directory - project.getTasks().register("setupGradleUserHome", Copy.class, copy -> { - copy.into(project.getGradle().getGradleUserHomeDir().getAbsolutePath() + "-" + project.getName()); - copy.from(project.getGradle().getGradleUserHomeDir().getAbsolutePath(), copySpec -> { - copySpec.include("gradle.properties"); - copySpec.include("init.d/*"); + // We don't use a normal `Copy` task here as snapshotting the entire gradle user home is very expensive. This task is cheap, so + // up-to-date checking doesn't buy us much + project.getTasks().register("setupGradleUserHome", task -> { + File gradleUserHome = project.getGradle().getGradleUserHomeDir(); + String projectName = project.getName(); + task.doLast(t -> { + fileSystemOperations.copy(copy -> { + String absoluteGradleUserHomePath = gradleUserHome.getAbsolutePath(); + copy.into(absoluteGradleUserHomePath + "-" + projectName); + copy.from(absoluteGradleUserHomePath, copySpec -> { + copySpec.include("gradle.properties"); + copySpec.include("init.d/*"); + }); + }); }); }); for (DistributionProject distributionProject : distributionProjects) { createBuildBwcTask( bwcSetupExtension, + buildParams, project, bwcVersion, distributionProject.name, @@ -143,6 +170,7 @@ private static void configureBwcProject( createBuildBwcTask( bwcSetupExtension, + buildParams, project, bwcVersion, "jdbc", @@ -164,13 +192,19 @@ private static void configureBwcProject( DistributionProjectArtifact stableAnalysisPluginProjectArtifact = new DistributionProjectArtifact( new File( checkoutDir.get(), - relativeDir + "/build/distributions/" + stableApiProject.getName() + "-" + bwcVersion.get() + "-SNAPSHOT.jar" + relativeDir + + "/build/distributions/elasticsearch-" + + stableApiProject.getName() + + "-" + + bwcVersion.get() + + "-SNAPSHOT.jar" ), null ); createBuildBwcTask( bwcSetupExtension, + buildParams, project, bwcVersion, stableApiProject.getName(), @@ -274,7 +308,7 @@ private static List resolveArchiveProjects(File checkoutDir } private static List resolveStableProjects(Project project) { - Set stableProjectNames = Set.of("elasticsearch-logging", "elasticsearch-plugin-api", "elasticsearch-plugin-analysis-api"); + Set stableProjectNames = Set.of("logging", "plugin-api", "plugin-analysis-api"); return project.findProject(":libs") .getSubprojects() .stream() @@ -290,6 +324,7 @@ public static String buildBwcTaskName(String projectName) { static void createBuildBwcTask( BwcSetupExtension bwcSetupExtension, + BuildParameterExtension buildParams, Project project, Provider bwcVersion, String projectName, @@ -310,8 +345,10 @@ static void createBuildBwcTask( } else { c.getOutputs().files(expectedOutputFile); } - c.getOutputs().doNotCacheIf("BWC distribution caching is disabled for local builds", task -> BuildParams.isCi() == false); - c.getArgs().add(projectPath.replace('/', ':') + ":" + assembleTaskName); + c.getOutputs().doNotCacheIf("BWC distribution caching is disabled for local builds", task -> buildParams.isCi() == false); + c.getArgs().add("-p"); + c.getArgs().add(projectPath); + c.getArgs().add(assembleTaskName); if (project.getGradle().getStartParameter().isBuildCacheEnabled()) { c.getArgs().add("--build-cache"); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java index eeb4306ce6fb9..78b9b48a99fcb 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; @@ -19,7 +20,6 @@ import org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes; import org.elasticsearch.gradle.internal.docker.DockerSupportPlugin; import org.elasticsearch.gradle.internal.docker.DockerSupportService; -import org.elasticsearch.gradle.internal.info.BuildParams; import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.elasticsearch.gradle.util.GradleUtils; import org.gradle.api.GradleException; @@ -34,6 +34,8 @@ import java.util.Map; import java.util.function.Function; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; + /** * An internal elasticsearch build plugin that registers additional * distribution resolution strategies to the 'elasticsearch.download-distribution' plugin @@ -46,6 +48,8 @@ public void apply(Project project) { // this is needed for isInternal project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class); project.getRootProject().getPluginManager().apply(DockerSupportPlugin.class); + var buildParams = loadBuildParams(project).get(); + DistributionDownloadPlugin distributionDownloadPlugin = project.getPlugins().apply(DistributionDownloadPlugin.class); Provider dockerSupport = GradleUtils.getBuildService( project.getGradle().getSharedServices(), @@ -54,7 +58,10 @@ public void apply(Project project) { distributionDownloadPlugin.setDockerAvailability( dockerSupport.map(dockerSupportService -> dockerSupportService.getDockerAvailability().isAvailable()) ); - registerInternalDistributionResolutions(DistributionDownloadPlugin.getRegistrationsContainer(project)); + registerInternalDistributionResolutions( + DistributionDownloadPlugin.getRegistrationsContainer(project), + buildParams.getBwcVersionsProvider() + ); } /** @@ -65,7 +72,7 @@ public void apply(Project project) { *

* BWC versions are resolved as project to projects under `:distribution:bwc`. */ - private void registerInternalDistributionResolutions(List resolutions) { + private void registerInternalDistributionResolutions(List resolutions, Provider bwcVersions) { resolutions.add(new DistributionResolution("local-build", (project, distribution) -> { if (isCurrentVersion(distribution)) { // non-external project, so depend on local build @@ -77,7 +84,7 @@ private void registerInternalDistributionResolutions(List { - BwcVersions.UnreleasedVersionInfo unreleasedInfo = BuildParams.getBwcVersions() + BwcVersions.UnreleasedVersionInfo unreleasedInfo = bwcVersions.get() .unreleasedInfo(Version.fromString(distribution.getVersion())); if (unreleasedInfo != null) { if (distribution.getBundledJdk() == false) { @@ -171,9 +178,6 @@ private static String distributionProjectName(ElasticsearchDistribution distribu if (distribution.getType() == InternalElasticsearchDistributionTypes.DOCKER_IRONBANK) { return projectName + "ironbank-docker" + archString + "-export"; } - if (distribution.getType() == InternalElasticsearchDistributionTypes.DOCKER_CLOUD) { - return projectName + "cloud-docker" + archString + "-export"; - } if (distribution.getType() == InternalElasticsearchDistributionTypes.DOCKER_CLOUD_ESS) { return projectName + "cloud-ess-docker" + archString + "-export"; } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java index a4412cd3db247..92a8db6b5b913 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; @@ -47,10 +48,11 @@ public class InternalDistributionModuleCheckTaskProvider { /** ES jars in the lib directory that are not modularized. For now, es-log4j is the only one. */ private static final List ES_JAR_EXCLUDES = List.of("elasticsearch-log4j"); - /** List of the current Elasticsearch Java Modules, by name. */ + /** List of the current Elasticsearch Java Modules, alphabetically by name. */ private static final List EXPECTED_ES_SERVER_MODULES = List.of( "org.elasticsearch.base", "org.elasticsearch.cli", + "org.elasticsearch.entitlement", "org.elasticsearch.geo", "org.elasticsearch.grok", "org.elasticsearch.logging", @@ -58,7 +60,6 @@ public class InternalDistributionModuleCheckTaskProvider { "org.elasticsearch.nativeaccess", "org.elasticsearch.plugin", "org.elasticsearch.plugin.analysis", - "org.elasticsearch.pluginclassloader", "org.elasticsearch.securesm", "org.elasticsearch.server", "org.elasticsearch.simdvec", diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalPluginBuildPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalPluginBuildPlugin.java index 065a255a28e6e..e22ee1430f3a3 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalPluginBuildPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalPluginBuildPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalReaperPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalReaperPlugin.java index 9ed0afff9807b..9cf520a5f9e45 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalReaperPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalReaperPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestArtifactBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestArtifactBasePlugin.java index afaa240c0bc6d..81a3570c20524 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestArtifactBasePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestArtifactBasePlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestArtifactExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestArtifactExtension.java index 63e139a647777..69c26ce2d05d5 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestArtifactExtension.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestArtifactExtension.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestArtifactPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestArtifactPlugin.java index 53cd6000e82c1..674effecd8fbd 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestArtifactPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestArtifactPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestClustersPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestClustersPlugin.java index 196835ccdd06d..c618fe6c2e1bf 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestClustersPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestClustersPlugin.java @@ -1,42 +1,37 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; import org.elasticsearch.gradle.VersionProperties; -import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.elasticsearch.gradle.testclusters.ElasticsearchCluster; import org.elasticsearch.gradle.testclusters.TestClustersPlugin; import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Plugin; import org.gradle.api.Project; -import org.gradle.api.provider.ProviderFactory; -import javax.inject.Inject; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; public class InternalTestClustersPlugin implements Plugin { - private ProviderFactory providerFactory; - - @Inject - public InternalTestClustersPlugin(ProviderFactory providerFactory) { - this.providerFactory = providerFactory; - } - @Override public void apply(Project project) { project.getPlugins().apply(InternalDistributionDownloadPlugin.class); + project.getRootProject().getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class); + var buildParams = loadBuildParams(project).get(); project.getRootProject().getPluginManager().apply(InternalReaperPlugin.class); TestClustersPlugin testClustersPlugin = project.getPlugins().apply(TestClustersPlugin.class); - testClustersPlugin.setRuntimeJava(providerFactory.provider(() -> BuildParams.getRuntimeJavaHome())); + testClustersPlugin.setRuntimeJava(buildParams.getRuntimeJavaHome()); testClustersPlugin.setIsReleasedVersion( - version -> (version.equals(VersionProperties.getElasticsearchVersion()) && BuildParams.isSnapshotBuild() == false) - || BuildParams.getBwcVersions().unreleasedInfo(version) == null + version -> (version.equals(VersionProperties.getElasticsearchVersion()) && buildParams.isSnapshotBuild() == false) + || buildParams.getBwcVersions().unreleasedInfo(version) == null ); if (shouldConfigureTestClustersWithOneProcessor()) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/JarApiComparisonTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/JarApiComparisonTask.java index 729188616d6d5..9a85f44953b6d 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/JarApiComparisonTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/JarApiComparisonTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/JavaClassPublicifier.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/JavaClassPublicifier.java index 27488ba01cf5b..82aacd935e5f4 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/JavaClassPublicifier.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/JavaClassPublicifier.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/Jdk.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/Jdk.java index f66bcb51ea527..c70d421939d6d 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/Jdk.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/Jdk.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/JdkDownloadPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/JdkDownloadPlugin.java index 5aa72c57e4be4..5b195cad3388f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/JdkDownloadPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/JdkDownloadPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/LoggingOutputStream.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/LoggingOutputStream.java index 3572964a82ed5..a373c1054881e 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/LoggingOutputStream.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/LoggingOutputStream.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java index 756d1ea48610b..0d58f80363330 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java @@ -1,14 +1,16 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; -import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.BuildParameterExtension; +import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.elasticsearch.gradle.internal.precommit.CheckForbiddenApisTask; import org.elasticsearch.gradle.util.GradleUtils; import org.gradle.api.JavaVersion; @@ -19,9 +21,12 @@ import org.gradle.api.plugins.JavaPluginExtension; import org.gradle.api.tasks.SourceSet; import org.gradle.api.tasks.SourceSetContainer; +import org.gradle.api.tasks.TaskProvider; import org.gradle.api.tasks.compile.CompileOptions; import org.gradle.api.tasks.compile.JavaCompile; +import org.gradle.api.tasks.javadoc.Javadoc; import org.gradle.api.tasks.testing.Test; +import org.gradle.external.javadoc.CoreJavadocOptions; import org.gradle.jvm.tasks.Jar; import org.gradle.jvm.toolchain.JavaLanguageVersion; import org.gradle.jvm.toolchain.JavaToolchainService; @@ -44,6 +49,7 @@ import javax.inject.Inject; import static de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin.FORBIDDEN_APIS_TASK_NAME; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; import static org.objectweb.asm.Opcodes.V_PREVIEW; public class MrjarPlugin implements Plugin { @@ -61,6 +67,8 @@ public class MrjarPlugin implements Plugin { @Override public void apply(Project project) { project.getPluginManager().apply(ElasticsearchJavaBasePlugin.class); + project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class); + var buildParams = loadBuildParams(project).get(); var javaExtension = project.getExtensions().getByType(JavaPluginExtension.class); var isIdeaSync = System.getProperty("idea.sync.active", "false").equals("true"); var ideaSourceSetsEnabled = project.hasProperty(MRJAR_IDEA_ENABLED) && project.property(MRJAR_IDEA_ENABLED).equals("true"); @@ -78,13 +86,14 @@ public void apply(Project project) { String mainSourceSetName = SourceSet.MAIN_SOURCE_SET_NAME + javaVersion; SourceSet mainSourceSet = addSourceSet(project, javaExtension, mainSourceSetName, mainSourceSets, javaVersion); configureSourceSetInJar(project, mainSourceSet, javaVersion); + addJar(project, mainSourceSet, javaVersion); mainSourceSets.add(mainSourceSetName); testSourceSets.add(mainSourceSetName); String testSourceSetName = SourceSet.TEST_SOURCE_SET_NAME + javaVersion; SourceSet testSourceSet = addSourceSet(project, javaExtension, testSourceSetName, testSourceSets, javaVersion); testSourceSets.add(testSourceSetName); - createTestTask(project, testSourceSet, javaVersion, mainSourceSets); + createTestTask(project, buildParams, testSourceSet, javaVersion, mainSourceSets); } } @@ -141,12 +150,41 @@ private SourceSet addSourceSet( return sourceSet; } + private void addJar(Project project, SourceSet sourceSet, int javaVersion) { + project.getConfigurations().register("java" + javaVersion); + TaskProvider jarTask = project.getTasks().register("java" + javaVersion + "Jar", Jar.class, task -> { + task.from(sourceSet.getOutput()); + }); + project.getArtifacts().add("java" + javaVersion, jarTask); + } + + private void configurePreviewFeatures(Project project, SourceSet sourceSet, int javaVersion) { + project.getTasks().withType(JavaCompile.class).named(sourceSet.getCompileJavaTaskName()).configure(compileTask -> { + CompileOptions compileOptions = compileTask.getOptions(); + compileOptions.getCompilerArgs().add("--enable-preview"); + compileOptions.getCompilerArgs().add("-Xlint:-preview"); + + compileTask.doLast(t -> { stripPreviewFromFiles(compileTask.getDestinationDirectory().getAsFile().get().toPath()); }); + }); + project.getTasks().withType(Javadoc.class).named(name -> name.equals(sourceSet.getJavadocTaskName())).configureEach(javadocTask -> { + CoreJavadocOptions options = (CoreJavadocOptions) javadocTask.getOptions(); + options.addBooleanOption("-enable-preview", true); + options.addStringOption("-release", String.valueOf(javaVersion)); + }); + } + private void configureSourceSetInJar(Project project, SourceSet sourceSet, int javaVersion) { var jarTask = project.getTasks().withType(Jar.class).named(JavaPlugin.JAR_TASK_NAME); jarTask.configure(task -> task.into("META-INF/versions/" + javaVersion, copySpec -> copySpec.from(sourceSet.getOutput()))); } - private void createTestTask(Project project, SourceSet sourceSet, int javaVersion, List mainSourceSets) { + private void createTestTask( + Project project, + BuildParameterExtension buildParams, + SourceSet sourceSet, + int javaVersion, + List mainSourceSets + ) { var jarTask = project.getTasks().withType(Jar.class).named(JavaPlugin.JAR_TASK_NAME); var testTaskProvider = project.getTasks().register(JavaPlugin.TEST_TASK_NAME + javaVersion, Test.class); testTaskProvider.configure(testTask -> { @@ -163,9 +201,9 @@ private void createTestTask(Project project, SourceSet sourceSet, int javaVersio // only set the jdk if runtime java isn't set because setting the toolchain is incompatible with // runtime java setting the executable directly - if (BuildParams.getIsRuntimeJavaHomeSet()) { + if (buildParams.getIsRuntimeJavaHomeSet()) { testTask.onlyIf("runtime java must support java " + javaVersion, t -> { - JavaVersion runtimeJavaVersion = BuildParams.getRuntimeJavaVersion(); + JavaVersion runtimeJavaVersion = buildParams.getRuntimeJavaVersion().get(); return runtimeJavaVersion.isCompatibleWith(JavaVersion.toVersion(javaVersion)); }); } else { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/NoticeTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/NoticeTask.java index 751ac92512dad..c9d0c34ee01d7 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/NoticeTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/NoticeTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RepositoriesSetupPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RepositoriesSetupPlugin.java index 098b7923bcbde..d559d955e63db 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RepositoriesSetupPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RepositoriesSetupPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ResolveAllDependencies.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ResolveAllDependencies.java index e9a3da6a4c8d3..694282adac051 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ResolveAllDependencies.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ResolveAllDependencies.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; @@ -11,8 +12,11 @@ import org.elasticsearch.gradle.VersionProperties; import org.gradle.api.DefaultTask; import org.gradle.api.artifacts.Configuration; +import org.gradle.api.artifacts.FileCollectionDependency; +import org.gradle.api.artifacts.component.ModuleComponentIdentifier; import org.gradle.api.file.FileCollection; import org.gradle.api.model.ObjectFactory; +import org.gradle.api.provider.ProviderFactory; import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.Internal; import org.gradle.api.tasks.TaskAction; @@ -25,9 +29,6 @@ import javax.inject.Inject; -import static org.elasticsearch.gradle.DistributionDownloadPlugin.DISTRO_EXTRACTED_CONFIG_PREFIX; -import static org.elasticsearch.gradle.internal.test.rest.compat.compat.LegacyYamlRestCompatTestPlugin.BWC_MINOR_CONFIG_NAME; - public abstract class ResolveAllDependencies extends DefaultTask { private boolean resolveJavaToolChain = false; @@ -36,18 +37,28 @@ public abstract class ResolveAllDependencies extends DefaultTask { protected abstract JavaToolchainService getJavaToolchainService(); private final ObjectFactory objectFactory; + private final ProviderFactory providerFactory; private Collection configs; @Inject - public ResolveAllDependencies(ObjectFactory objectFactory) { + public ResolveAllDependencies(ObjectFactory objectFactory, ProviderFactory providerFactory) { this.objectFactory = objectFactory; + this.providerFactory = providerFactory; } @InputFiles public FileCollection getResolvedArtifacts() { - return objectFactory.fileCollection() - .from(configs.stream().filter(ResolveAllDependencies::canBeResolved).collect(Collectors.toList())); + return objectFactory.fileCollection().from(configs.stream().filter(ResolveAllDependencies::canBeResolved).map(c -> { + // Make a copy of the configuration, omitting file collection dependencies to avoid building project artifacts + Configuration copy = c.copyRecursive(d -> d instanceof FileCollectionDependency == false); + copy.setCanBeConsumed(false); + return copy; + }) + // Include only module dependencies, ignoring things like project dependencies so we don't unnecessarily build stuff + .map(c -> c.getIncoming().artifactView(v -> v.lenient(true).componentFilter(i -> i instanceof ModuleComponentIdentifier))) + .map(artifactView -> providerFactory.provider(artifactView::getFiles)) + .collect(Collectors.toList())); } @TaskAction @@ -94,8 +105,8 @@ private static boolean canBeResolved(Configuration configuration) { return false; } } - return configuration.getName().startsWith(DISTRO_EXTRACTED_CONFIG_PREFIX) == false - && configuration.getName().equals(BWC_MINOR_CONFIG_NAME) == false; + + return true; } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java index 9cc98e79183ce..7e3e8bd458c92 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; @@ -52,8 +53,6 @@ private static ListMultimap, String> createLegacyRestTestBasePluginUsag map.put(LegacyRestTestBasePlugin.class, ":plugins:repository-hdfs"); map.put(LegacyRestTestBasePlugin.class, ":plugins:store-smb"); map.put(LegacyRestTestBasePlugin.class, ":qa:ccs-rolling-upgrade-remote-cluster"); - map.put(LegacyRestTestBasePlugin.class, ":qa:ccs-unavailable-clusters"); - map.put(LegacyRestTestBasePlugin.class, ":qa:logging-config"); map.put(LegacyRestTestBasePlugin.class, ":qa:mixed-cluster"); map.put(LegacyRestTestBasePlugin.class, ":qa:multi-cluster-search"); map.put(LegacyRestTestBasePlugin.class, ":qa:remote-clusters"); @@ -64,7 +63,6 @@ private static ListMultimap, String> createLegacyRestTestBasePluginUsag map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-ingest-with-all-dependencies"); map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-plugins"); map.put(LegacyRestTestBasePlugin.class, ":qa:system-indices"); - map.put(LegacyRestTestBasePlugin.class, ":qa:unconfigured-node-name"); map.put(LegacyRestTestBasePlugin.class, ":qa:verify-version-constants"); map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-apm-integration"); map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-delayed-aggs"); @@ -72,7 +70,6 @@ private static ListMultimap, String> createLegacyRestTestBasePluginUsag map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-error-query"); map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-latency-simulating-directory"); map.put(LegacyRestTestBasePlugin.class, ":test:yaml-rest-runner"); - map.put(LegacyRestTestBasePlugin.class, ":distribution:archives:integ-test-zip"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:core"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ent-search"); map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:fleet"); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/StringTemplatePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/StringTemplatePlugin.java index 019216bfe1d2d..fdf5bdd703c15 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/StringTemplatePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/StringTemplatePlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/StringTemplateTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/StringTemplateTask.java index 6a887141e79db..d292c5d2e1686 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/StringTemplateTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/StringTemplateTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTar.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTar.java index 52000e8c8fd71..ec4b1fea9e962 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTar.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTar.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DebElasticsearchDistributionType.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DebElasticsearchDistributionType.java index 7febdca5a806d..09dfdb62e22ee 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DebElasticsearchDistributionType.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DebElasticsearchDistributionType.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.distribution; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerCloudElasticsearchDistributionType.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerCloudElasticsearchDistributionType.java deleted file mode 100644 index c95db0e1aaa91..0000000000000 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerCloudElasticsearchDistributionType.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.gradle.internal.distribution; - -import org.elasticsearch.gradle.ElasticsearchDistributionType; - -public class DockerCloudElasticsearchDistributionType implements ElasticsearchDistributionType { - - DockerCloudElasticsearchDistributionType() {} - - @Override - public String getName() { - return "dockerCloud"; - } - - @Override - public boolean isDocker() { - return true; - } -} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerCloudEssElasticsearchDistributionType.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerCloudEssElasticsearchDistributionType.java index 65a07331e412d..28c9bf114e657 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerCloudEssElasticsearchDistributionType.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerCloudEssElasticsearchDistributionType.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.distribution; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerElasticsearchDistributionType.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerElasticsearchDistributionType.java index 439ec23481f75..f4a26a4da515b 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerElasticsearchDistributionType.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerElasticsearchDistributionType.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.distribution; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerIronBankElasticsearchDistributionType.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerIronBankElasticsearchDistributionType.java index e261cf3b5925a..ba857ee72b2b8 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerIronBankElasticsearchDistributionType.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerIronBankElasticsearchDistributionType.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.distribution; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerUbiElasticsearchDistributionType.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerUbiElasticsearchDistributionType.java index 65f4e3e9e4858..aa19bf6d60c53 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerUbiElasticsearchDistributionType.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerUbiElasticsearchDistributionType.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.distribution; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerWolfiElasticsearchDistributionType.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerWolfiElasticsearchDistributionType.java index d055337436a88..aa28d6b62bc85 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerWolfiElasticsearchDistributionType.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerWolfiElasticsearchDistributionType.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.distribution; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/ElasticsearchDistributionExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/ElasticsearchDistributionExtension.java index 83932fdceb47e..f0502a674e246 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/ElasticsearchDistributionExtension.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/ElasticsearchDistributionExtension.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.distribution; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/ElasticsearchDistributionPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/ElasticsearchDistributionPlugin.java index 148114c3ea2f9..70668dddc350a 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/ElasticsearchDistributionPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/ElasticsearchDistributionPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.distribution; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java index 5f8ef58e44a68..8f0951da86b88 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.distribution; @@ -18,7 +19,6 @@ public class InternalElasticsearchDistributionTypes { public static ElasticsearchDistributionType DOCKER = new DockerElasticsearchDistributionType(); public static ElasticsearchDistributionType DOCKER_UBI = new DockerUbiElasticsearchDistributionType(); public static ElasticsearchDistributionType DOCKER_IRONBANK = new DockerIronBankElasticsearchDistributionType(); - public static ElasticsearchDistributionType DOCKER_CLOUD = new DockerCloudElasticsearchDistributionType(); public static ElasticsearchDistributionType DOCKER_CLOUD_ESS = new DockerCloudEssElasticsearchDistributionType(); public static ElasticsearchDistributionType DOCKER_WOLFI = new DockerWolfiElasticsearchDistributionType(); @@ -28,7 +28,6 @@ public class InternalElasticsearchDistributionTypes { DOCKER, DOCKER_UBI, DOCKER_IRONBANK, - DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI ); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/RpmElasticsearchDistributionType.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/RpmElasticsearchDistributionType.java index a23cac67a7031..e9a65d8c6e6c5 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/RpmElasticsearchDistributionType.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/RpmElasticsearchDistributionType.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.distribution; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/AsciidocSnippetParser.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/AsciidocSnippetParser.java index f291566d526ff..d50689a6d62cd 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/AsciidocSnippetParser.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/AsciidocSnippetParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.doc; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/DocSnippetTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/DocSnippetTask.java index 07e3bc93bb6a1..60ddbd0cdcfc6 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/DocSnippetTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/DocSnippetTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.doc; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/DocsTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/DocsTestPlugin.java index 2504ea1e74a36..7d3ff5102f680 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/DocsTestPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/DocsTestPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.doc; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/MdxSnippetParser.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/MdxSnippetParser.java index 0a0bb6328491e..81604e82ef946 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/MdxSnippetParser.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/MdxSnippetParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.doc; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/ParsingUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/ParsingUtils.java index 53009e1ce5978..49359dc1f7f13 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/ParsingUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/ParsingUtils.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.doc; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/RestTestsFromDocSnippetTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/RestTestsFromDocSnippetTask.java index 28733ff052777..cc15d4e9e4e90 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/RestTestsFromDocSnippetTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/RestTestsFromDocSnippetTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.doc; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/Snippet.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/Snippet.java index 227ecbcbfd386..9b4e2f781e1c1 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/Snippet.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/Snippet.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.doc; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/SnippetBuilder.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/SnippetBuilder.java index 36d15b9eb33ca..6d578eb1b124a 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/SnippetBuilder.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/SnippetBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.doc; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/SnippetParser.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/SnippetParser.java index c4ae0b90127a9..12fbb5227793b 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/SnippetParser.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/SnippetParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.doc; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/SnippetParserException.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/SnippetParserException.java index 79563a97de119..8ae4e7b1cc4d2 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/SnippetParserException.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/SnippetParserException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.doc; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/Source.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/Source.java index b7f2f01aa7987..2dff211a7f3c9 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/Source.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/doc/Source.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.doc; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java index b84a90259787e..9b28401994ee2 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.docker; @@ -29,6 +30,7 @@ import org.gradle.api.tasks.PathSensitivity; import org.gradle.api.tasks.TaskAction; import org.gradle.process.ExecOperations; +import org.gradle.process.ExecSpec; import org.gradle.workers.WorkAction; import org.gradle.workers.WorkParameters; import org.gradle.workers.WorkerExecutor; @@ -165,6 +167,7 @@ private void pullBaseImage(String baseImage) { for (int attempt = 1; attempt <= maxAttempts; attempt++) { try { LoggedExec.exec(execOperations, spec -> { + maybeConfigureDockerConfig(spec); spec.executable("docker"); spec.args("pull"); spec.args(baseImage); @@ -180,6 +183,13 @@ private void pullBaseImage(String baseImage) { throw new GradleException("Failed to pull Docker base image [" + baseImage + "], all attempts failed"); } + private void maybeConfigureDockerConfig(ExecSpec spec) { + String dockerConfig = System.getenv("DOCKER_CONFIG"); + if (dockerConfig != null) { + spec.environment("DOCKER_CONFIG", dockerConfig); + } + } + @Override public void execute() { final Parameters parameters = getParameters(); @@ -192,6 +202,8 @@ public void execute() { final boolean isCrossPlatform = isCrossPlatform(); LoggedExec.exec(execOperations, spec -> { + maybeConfigureDockerConfig(spec); + spec.executable("docker"); if (isCrossPlatform) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerResult.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerResult.java index d101c0046f926..4a18e9786b219 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerResult.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerResult.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.docker; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportPlugin.java index 4c035827bb7aa..7348181c4199c 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportPlugin.java @@ -1,12 +1,14 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.docker; +import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.Task; @@ -16,6 +18,8 @@ import java.util.List; import java.util.stream.Collectors; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; + /** * Plugin providing {@link DockerSupportService} for detecting Docker installations and determining requirements for Docker-based * Elasticsearch build tasks. @@ -29,11 +33,14 @@ public void apply(Project project) { if (project != project.getRootProject()) { throw new IllegalStateException(this.getClass().getName() + " can only be applied to the root project."); } + project.getPlugins().apply(GlobalBuildInfoPlugin.class); + var buildParams = loadBuildParams(project).get(); Provider dockerSupportServiceProvider = project.getGradle() .getSharedServices() .registerIfAbsent(DOCKER_SUPPORT_SERVICE_NAME, DockerSupportService.class, spec -> spec.parameters(params -> { params.setExclusionsFile(new File(project.getRootDir(), DOCKER_ON_LINUX_EXCLUSIONS_FILE)); + params.getIsCI().set(buildParams.isCi()); })); // Ensure that if we are trying to run any DockerBuildTask tasks, we assert an available Docker installation exists diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportService.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportService.java index fd75df6c06b84..f40f5d932b701 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportService.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.docker; @@ -12,10 +13,10 @@ import org.elasticsearch.gradle.Architecture; import org.elasticsearch.gradle.OS; import org.elasticsearch.gradle.Version; -import org.elasticsearch.gradle.internal.info.BuildParams; import org.gradle.api.GradleException; import org.gradle.api.logging.Logger; import org.gradle.api.logging.Logging; +import org.gradle.api.provider.Property; import org.gradle.api.provider.ProviderFactory; import org.gradle.api.services.BuildService; import org.gradle.api.services.BuildServiceParameters; @@ -58,7 +59,6 @@ public abstract class DockerSupportService implements BuildService serviceInfos; private Map> tcpPorts; private Map> udpPorts; @@ -227,7 +227,7 @@ private boolean isExcludedOs() { // We don't attempt to check the current flavor and version of Linux unless we're // running in CI, because we don't want to stop people running the Docker tests in // their own environments if they really want to. - if (BuildParams.isCi() == false) { + if (getParameters().getIsCI().get().booleanValue() == false) { return false; } @@ -355,10 +355,6 @@ public Map> getUdpPorts() { return udpPorts; } - public void setServiceInfos(Map serviceInfos) { - this.serviceInfos = serviceInfos; - } - /** * An immutable class that represents the results of a Docker search from {@link #getDockerAvailability()}}. */ @@ -401,5 +397,7 @@ interface Parameters extends BuildServiceParameters { File getExclusionsFile(); void setExclusionsFile(File exclusionsFile); + + Property getIsCI(); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerValueSource.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerValueSource.java index d71208b624d7a..c36c388b8decd 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerValueSource.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerValueSource.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.docker; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/ShellRetry.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/ShellRetry.java index e3193f7aea5d0..ca5b1d6fac38e 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/ShellRetry.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/ShellRetry.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.docker; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/TransformLog4jConfigFilter.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/TransformLog4jConfigFilter.java index 4be9380388a74..65fb6ad8ea626 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/TransformLog4jConfigFilter.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/TransformLog4jConfigFilter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.docker; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dra/DraResolvePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dra/DraResolvePlugin.java index 77b0c1b699dac..6ca7fd579d733 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dra/DraResolvePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dra/DraResolvePlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.dra; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParameterExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParameterExtension.java new file mode 100644 index 0000000000000..ef9055b3728d3 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParameterExtension.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.info; + +import org.elasticsearch.gradle.internal.BwcVersions; +import org.gradle.api.Action; +import org.gradle.api.JavaVersion; +import org.gradle.api.Task; +import org.gradle.api.provider.Provider; +import org.gradle.jvm.toolchain.JavaToolchainSpec; + +import java.io.File; +import java.time.ZonedDateTime; +import java.util.List; +import java.util.Random; + +public interface BuildParameterExtension { + String EXTENSION_NAME = "buildParams"; + + boolean getInFipsJvm(); + + Provider getRuntimeJavaHome(); + + void withFipsEnabledOnly(Task task); + + Boolean getIsRuntimeJavaHomeSet(); + + List getJavaVersions(); + + JavaVersion getMinimumCompilerVersion(); + + JavaVersion getMinimumRuntimeVersion(); + + JavaVersion getGradleJavaVersion(); + + Provider getRuntimeJavaVersion(); + + Provider> getJavaToolChainSpec(); + + Provider getRuntimeJavaDetails(); + + Provider getGitRevision(); + + Provider getGitOrigin(); + + ZonedDateTime getBuildDate(); + + String getTestSeed(); + + Boolean isCi(); + + Integer getDefaultParallel(); + + Boolean isSnapshotBuild(); + + BwcVersions getBwcVersions(); + + Provider getBwcVersionsProvider(); + + Random getRandom(); + + Boolean isGraalVmRuntime(); +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParameterService.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParameterService.java new file mode 100644 index 0000000000000..ec1bc4aec1324 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParameterService.java @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.info; + +import org.gradle.api.provider.Property; +import org.gradle.api.services.BuildService; +import org.gradle.api.services.BuildServiceParameters; + +public abstract class BuildParameterService implements BuildService, AutoCloseable { + public interface Params extends BuildServiceParameters { + Property getBuildParams(); + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParams.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParams.java index 964784643936b..ea8aeda8fc099 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParams.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParams.java @@ -1,49 +1,20 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.info; -import org.elasticsearch.gradle.internal.BwcVersions; -import org.gradle.api.Action; -import org.gradle.api.JavaVersion; -import org.gradle.api.Task; -import org.gradle.api.provider.Provider; -import org.gradle.jvm.toolchain.JavaToolchainSpec; - -import java.io.File; -import java.io.IOException; import java.lang.reflect.Modifier; -import java.time.ZonedDateTime; import java.util.Arrays; -import java.util.List; -import java.util.Random; import java.util.function.Consumer; -import static java.util.Objects.requireNonNull; - +@Deprecated public class BuildParams { - private static Provider runtimeJavaHome; - private static Boolean isRuntimeJavaHomeSet; - private static List javaVersions; - private static JavaVersion minimumCompilerVersion; - private static JavaVersion minimumRuntimeVersion; - private static JavaVersion gradleJavaVersion; - private static Provider runtimeJavaVersion; - private static Provider> javaToolChainSpec; - private static Provider runtimeJavaDetails; - private static Boolean inFipsJvm; - private static String gitRevision; - private static String gitOrigin; - private static ZonedDateTime buildDate; - private static String testSeed; private static Boolean isCi; - private static Integer defaultParallel; - private static Boolean isSnapshotBuild; - private static Provider bwcVersions; /** * Initialize global build parameters. This method accepts and a initialization function which in turn accepts a @@ -57,90 +28,10 @@ public static void init(Consumer initializer) { initializer.accept(MutableBuildParams.INSTANCE); } - public static File getRuntimeJavaHome() { - return value(runtimeJavaHome).get(); - } - - public static Boolean getIsRuntimeJavaHomeSet() { - return value(isRuntimeJavaHomeSet); - } - - public static List getJavaVersions() { - return value(javaVersions); - } - - public static JavaVersion getMinimumCompilerVersion() { - return value(minimumCompilerVersion); - } - - public static JavaVersion getMinimumRuntimeVersion() { - return value(minimumRuntimeVersion); - } - - public static JavaVersion getGradleJavaVersion() { - return value(gradleJavaVersion); - } - - public static JavaVersion getRuntimeJavaVersion() { - return value(runtimeJavaVersion.get()); - } - - public static String getRuntimeJavaDetails() { - return value(runtimeJavaDetails.get()); - } - - public static Boolean isInFipsJvm() { - return value(inFipsJvm); - } - - public static void withFipsEnabledOnly(Task task) { - task.onlyIf("FIPS mode disabled", task1 -> isInFipsJvm() == false); - } - - public static String getGitRevision() { - return value(gitRevision); - } - - public static String getGitOrigin() { - return value(gitOrigin); - } - - public static ZonedDateTime getBuildDate() { - return value(buildDate); - } - - public static BwcVersions getBwcVersions() { - return value(bwcVersions).get(); - } - - public static String getTestSeed() { - return value(testSeed); - } - - public static Random getRandom() { - return new Random(Long.parseUnsignedLong(testSeed.split(":")[0], 16)); - } - public static Boolean isCi() { return value(isCi); } - public static Boolean isGraalVmRuntime() { - return value(runtimeJavaDetails.get().toLowerCase().contains("graalvm")); - } - - public static Integer getDefaultParallel() { - return value(defaultParallel); - } - - public static boolean isSnapshotBuild() { - return value(BuildParams.isSnapshotBuild); - } - - public static Provider> getJavaToolChainSpec() { - return javaToolChainSpec; - } - private static T value(T object) { if (object == null) { String callingMethod = Thread.currentThread().getStackTrace()[2].getMethodName(); @@ -182,82 +73,8 @@ public void reset() { }); } - public void setRuntimeJavaHome(Provider runtimeJavaHome) { - BuildParams.runtimeJavaHome = runtimeJavaHome.map(javaHome -> { - try { - return javaHome.getCanonicalFile(); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - } - - public void setIsRuntimeJavaHomeSet(boolean isRuntimeJavaHomeSet) { - BuildParams.isRuntimeJavaHomeSet = isRuntimeJavaHomeSet; - } - - public void setJavaVersions(List javaVersions) { - BuildParams.javaVersions = requireNonNull(javaVersions); - } - - public void setMinimumCompilerVersion(JavaVersion minimumCompilerVersion) { - BuildParams.minimumCompilerVersion = requireNonNull(minimumCompilerVersion); - } - - public void setMinimumRuntimeVersion(JavaVersion minimumRuntimeVersion) { - BuildParams.minimumRuntimeVersion = requireNonNull(minimumRuntimeVersion); - } - - public void setGradleJavaVersion(JavaVersion gradleJavaVersion) { - BuildParams.gradleJavaVersion = requireNonNull(gradleJavaVersion); - } - - public void setRuntimeJavaVersion(Provider runtimeJavaVersion) { - BuildParams.runtimeJavaVersion = requireNonNull(runtimeJavaVersion); - } - - public void setRuntimeJavaDetails(Provider runtimeJavaDetails) { - BuildParams.runtimeJavaDetails = runtimeJavaDetails; - } - - public void setInFipsJvm(boolean inFipsJvm) { - BuildParams.inFipsJvm = inFipsJvm; - } - - public void setGitRevision(String gitRevision) { - BuildParams.gitRevision = requireNonNull(gitRevision); - } - - public void setGitOrigin(String gitOrigin) { - BuildParams.gitOrigin = requireNonNull(gitOrigin); - } - - public void setBuildDate(ZonedDateTime buildDate) { - BuildParams.buildDate = requireNonNull(buildDate); - } - - public void setTestSeed(String testSeed) { - BuildParams.testSeed = requireNonNull(testSeed); - } - public void setIsCi(boolean isCi) { BuildParams.isCi = isCi; } - - public void setDefaultParallel(int defaultParallel) { - BuildParams.defaultParallel = defaultParallel; - } - - public void setIsSnapshotBuild(final boolean isSnapshotBuild) { - BuildParams.isSnapshotBuild = isSnapshotBuild; - } - - public void setBwcVersions(Provider bwcVersions) { - BuildParams.bwcVersions = requireNonNull(bwcVersions); - } - - public void setJavaToolChainSpec(Provider> javaToolChain) { - BuildParams.javaToolChainSpec = javaToolChain; - } } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/DefaultBuildParameterExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/DefaultBuildParameterExtension.java new file mode 100644 index 0000000000000..283c02428e4e6 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/DefaultBuildParameterExtension.java @@ -0,0 +1,245 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.info; + +import org.elasticsearch.gradle.internal.BwcVersions; +import org.gradle.api.Action; +import org.gradle.api.JavaVersion; +import org.gradle.api.Task; +import org.gradle.api.provider.Provider; +import org.gradle.api.provider.ProviderFactory; +import org.gradle.jvm.toolchain.JavaToolchainSpec; + +import java.io.File; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.List; +import java.util.Random; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; + +public abstract class DefaultBuildParameterExtension implements BuildParameterExtension { + private final Provider inFipsJvm; + private final Provider runtimeJavaHome; + private final Boolean isRuntimeJavaHomeSet; + private final List javaVersions; + private final JavaVersion minimumCompilerVersion; + private final JavaVersion minimumRuntimeVersion; + private final JavaVersion gradleJavaVersion; + private final Provider runtimeJavaVersion; + private final Provider> javaToolChainSpec; + private final Provider runtimeJavaDetails; + private final Provider gitRevision; + + private transient AtomicReference buildDate = new AtomicReference<>(); + private final String testSeed; + private final Boolean isCi; + private final Integer defaultParallel; + private final Boolean isSnapshotBuild; + + // not final for testing + private Provider bwcVersions; + private Provider gitOrigin; + + public DefaultBuildParameterExtension( + ProviderFactory providers, + Provider runtimeJavaHome, + Provider> javaToolChainSpec, + Provider runtimeJavaVersion, + boolean isRuntimeJavaHomeSet, + Provider runtimeJavaDetails, + List javaVersions, + JavaVersion minimumCompilerVersion, + JavaVersion minimumRuntimeVersion, + JavaVersion gradleJavaVersion, + Provider gitRevision, + Provider gitOrigin, + String testSeed, + boolean isCi, + int defaultParallel, + final boolean isSnapshotBuild, + Provider bwcVersions + ) { + this.inFipsJvm = providers.systemProperty("tests.fips.enabled").map(DefaultBuildParameterExtension::parseBoolean); + this.runtimeJavaHome = cache(providers, runtimeJavaHome); + this.javaToolChainSpec = cache(providers, javaToolChainSpec); + this.runtimeJavaVersion = cache(providers, runtimeJavaVersion); + this.isRuntimeJavaHomeSet = isRuntimeJavaHomeSet; + this.runtimeJavaDetails = cache(providers, runtimeJavaDetails); + this.javaVersions = javaVersions; + this.minimumCompilerVersion = minimumCompilerVersion; + this.minimumRuntimeVersion = minimumRuntimeVersion; + this.gradleJavaVersion = gradleJavaVersion; + this.gitRevision = gitRevision; + this.testSeed = testSeed; + this.isCi = isCi; + this.defaultParallel = defaultParallel; + this.isSnapshotBuild = isSnapshotBuild; + this.bwcVersions = cache(providers, bwcVersions); + this.gitOrigin = gitOrigin; + } + + // This is a workaround for https://github.com/gradle/gradle/issues/25550 + private Provider cache(ProviderFactory providerFactory, Provider incomingProvider) { + SingleObjectCache cache = new SingleObjectCache<>(); + return providerFactory.provider(() -> cache.computeIfAbsent(() -> incomingProvider.getOrNull())); + } + + private static boolean parseBoolean(String s) { + if (s == null) { + return false; + } + return Boolean.parseBoolean(s); + } + + @Override + public boolean getInFipsJvm() { + return inFipsJvm.getOrElse(false); + } + + @Override + public Provider getRuntimeJavaHome() { + return runtimeJavaHome; + } + + @Override + public void withFipsEnabledOnly(Task task) { + task.onlyIf("FIPS mode disabled", task1 -> getInFipsJvm() == false); + } + + @Override + public Boolean getIsRuntimeJavaHomeSet() { + return isRuntimeJavaHomeSet; + } + + @Override + public List getJavaVersions() { + return javaVersions; + } + + @Override + public JavaVersion getMinimumCompilerVersion() { + return minimumCompilerVersion; + } + + @Override + public JavaVersion getMinimumRuntimeVersion() { + return minimumRuntimeVersion; + } + + @Override + public JavaVersion getGradleJavaVersion() { + return gradleJavaVersion; + } + + @Override + public Provider getRuntimeJavaVersion() { + return runtimeJavaVersion; + } + + @Override + public Provider> getJavaToolChainSpec() { + return javaToolChainSpec; + } + + @Override + public Provider getRuntimeJavaDetails() { + return runtimeJavaDetails; + } + + @Override + public Provider getGitRevision() { + return gitRevision; + } + + @Override + public Provider getGitOrigin() { + return gitOrigin; + } + + @Override + public ZonedDateTime getBuildDate() { + ZonedDateTime value = buildDate.get(); + if (value == null) { + value = ZonedDateTime.now(ZoneOffset.UTC); + if (buildDate.compareAndSet(null, value) == false) { + // If another thread initialized it first, return the initialized value + value = buildDate.get(); + } + } + return value; + } + + @Override + public String getTestSeed() { + return testSeed; + } + + @Override + public Boolean isCi() { + return isCi; + } + + @Override + public Integer getDefaultParallel() { + return defaultParallel; + } + + @Override + public Boolean isSnapshotBuild() { + return isSnapshotBuild; + } + + @Override + public BwcVersions getBwcVersions() { + return bwcVersions.get(); + } + + @Override + public Random getRandom() { + return new Random(Long.parseUnsignedLong(testSeed.split(":")[0], 16)); + } + + @Override + public Boolean isGraalVmRuntime() { + return runtimeJavaDetails.get().toLowerCase().contains("graalvm"); + } + + private static class SingleObjectCache { + private T instance; + + public T computeIfAbsent(Supplier supplier) { + synchronized (this) { + if (instance == null) { + instance = supplier.get(); + } + return instance; + } + } + + public T get() { + return instance; + } + } + + public Provider getBwcVersionsProvider() { + return bwcVersions; + } + + // for testing; not part of public api + public void setBwcVersions(Provider bwcVersions) { + this.bwcVersions = bwcVersions; + } + + // for testing; not part of public api + public void setGitOrigin(Provider gitOrigin) { + this.gitOrigin = gitOrigin; + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java index b287815854098..c680d65955873 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java @@ -1,15 +1,20 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.info; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + import org.apache.commons.io.IOUtils; import org.elasticsearch.gradle.VersionProperties; import org.elasticsearch.gradle.internal.BwcVersions; +import org.elasticsearch.gradle.internal.conventions.GitInfoPlugin; import org.elasticsearch.gradle.internal.conventions.info.GitInfo; import org.elasticsearch.gradle.internal.conventions.info.ParallelDetector; import org.elasticsearch.gradle.internal.conventions.util.Util; @@ -43,11 +48,11 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.io.UncheckedIOException; import java.nio.file.Files; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; +import java.util.ArrayList; import java.util.List; import java.util.Locale; import java.util.Random; @@ -57,6 +62,8 @@ import javax.inject.Inject; +import static org.elasticsearch.gradle.internal.conventions.GUtils.elvis; + public class GlobalBuildInfoPlugin implements Plugin { private static final Logger LOGGER = Logging.getLogger(GlobalBuildInfoPlugin.class); private static final String DEFAULT_VERSION_JAVA_FILE_PATH = "server/src/main/java/org/elasticsearch/Version.java"; @@ -65,7 +72,9 @@ public class GlobalBuildInfoPlugin implements Plugin { private final JavaInstallationRegistry javaInstallationRegistry; private final JvmMetadataDetector metadataDetector; private final ProviderFactory providers; + private final ObjectMapper objectMapper; private JavaToolchainService toolChainService; + private Project project; @Inject public GlobalBuildInfoPlugin( @@ -78,7 +87,7 @@ public GlobalBuildInfoPlugin( this.javaInstallationRegistry = javaInstallationRegistry; this.metadataDetector = new ErrorTraceMetadataDetector(metadataDetector); this.providers = providers; - + this.objectMapper = new ObjectMapper(); } @Override @@ -86,7 +95,10 @@ public void apply(Project project) { if (project != project.getRootProject()) { throw new IllegalStateException(this.getClass().getName() + " can only be applied to the root project."); } + this.project = project; project.getPlugins().apply(JvmToolchainsPlugin.class); + Provider gitInfo = project.getPlugins().apply(GitInfoPlugin.class).getGitInfo(); + toolChainService = project.getExtensions().getByType(JavaToolchainService.class); GradleVersion minimumGradleVersion = GradleVersion.version(getResourceContents("/minimumGradleVersion")); if (GradleVersion.current().compareTo(minimumGradleVersion) < 0) { @@ -97,55 +109,60 @@ public void apply(Project project) { JavaVersion minimumRuntimeVersion = JavaVersion.toVersion(getResourceContents("/minimumRuntimeVersion")); Provider explicitRuntimeJavaHome = findRuntimeJavaHome(); - boolean isExplicitRuntimeJavaHomeSet = explicitRuntimeJavaHome.isPresent(); - Provider actualRuntimeJavaHome = isExplicitRuntimeJavaHomeSet + boolean isRuntimeJavaHomeExplicitlySet = explicitRuntimeJavaHome.isPresent(); + Provider actualRuntimeJavaHome = isRuntimeJavaHomeExplicitlySet ? explicitRuntimeJavaHome : resolveJavaHomeFromToolChainService(VersionProperties.getBundledJdkMajorVersion()); - GitInfo gitInfo = GitInfo.gitInfo(project.getRootDir()); - - BuildParams.init(params -> { - params.reset(); - params.setRuntimeJavaHome(actualRuntimeJavaHome); - params.setJavaToolChainSpec(resolveToolchainSpecFromEnv()); - Provider runtimeJdkMetaData = actualRuntimeJavaHome.map( - runtimeJavaHome -> metadataDetector.getMetadata(getJavaInstallation(runtimeJavaHome)) - ); - params.setRuntimeJavaVersion( + Provider runtimeJdkMetaData = actualRuntimeJavaHome.map( + runtimeJavaHome -> metadataDetector.getMetadata(getJavaInstallation(runtimeJavaHome)) + ); + AtomicReference cache = new AtomicReference<>(); + Provider bwcVersionsProvider = providers.provider( + () -> cache.updateAndGet(val -> val == null ? resolveBwcVersions() : val) + ); + BuildParameterExtension buildParams = project.getExtensions() + .create( + BuildParameterExtension.class, + BuildParameterExtension.EXTENSION_NAME, + DefaultBuildParameterExtension.class, + providers, + actualRuntimeJavaHome, + resolveToolchainSpecFromEnv(), actualRuntimeJavaHome.map( javaHome -> determineJavaVersion( "runtime java.home", javaHome, - isExplicitRuntimeJavaHomeSet + isRuntimeJavaHomeExplicitlySet ? minimumRuntimeVersion : JavaVersion.toVersion(VersionProperties.getBundledJdkMajorVersion()) ) - ) + ), + isRuntimeJavaHomeExplicitlySet, + runtimeJdkMetaData.map(m -> formatJavaVendorDetails(m)), + getAvailableJavaVersions(), + minimumCompilerVersion, + minimumRuntimeVersion, + Jvm.current().getJavaVersion(), + gitInfo.map(g -> g.getRevision()), + gitInfo.map(g -> g.getOrigin()), + getTestSeed(), + System.getenv("JENKINS_URL") != null || System.getenv("BUILDKITE_BUILD_URL") != null || System.getProperty("isCI") != null, + ParallelDetector.findDefaultParallel(project), + Util.getBooleanProperty("build.snapshot", true), + bwcVersionsProvider ); - params.setIsRuntimeJavaHomeSet(isExplicitRuntimeJavaHomeSet); - params.setRuntimeJavaDetails(runtimeJdkMetaData.map(m -> formatJavaVendorDetails(m))); - params.setJavaVersions(getAvailableJavaVersions()); - params.setMinimumCompilerVersion(minimumCompilerVersion); - params.setMinimumRuntimeVersion(minimumRuntimeVersion); - params.setGradleJavaVersion(Jvm.current().getJavaVersion()); - params.setGitRevision(gitInfo.getRevision()); - params.setGitOrigin(gitInfo.getOrigin()); - params.setBuildDate(ZonedDateTime.now(ZoneOffset.UTC)); - params.setTestSeed(getTestSeed()); + + project.getGradle().getSharedServices().registerIfAbsent("buildParams", BuildParameterService.class, spec -> { + // Provide some parameters + spec.getParameters().getBuildParams().set(buildParams); + }); + + BuildParams.init(params -> { + params.reset(); params.setIsCi( System.getenv("JENKINS_URL") != null || System.getenv("BUILDKITE_BUILD_URL") != null || System.getProperty("isCI") != null ); - params.setDefaultParallel(ParallelDetector.findDefaultParallel(project)); - params.setInFipsJvm(Util.getBooleanProperty("tests.fips.enabled", false)); - params.setIsSnapshotBuild(Util.getBooleanProperty("build.snapshot", true)); - AtomicReference cache = new AtomicReference<>(); - params.setBwcVersions( - providers.provider( - () -> cache.updateAndGet( - val -> val == null ? resolveBwcVersions(Util.locateElasticsearchWorkspace(project.getGradle())) : val - ) - ) - ); }); // Enforce the minimum compiler version @@ -154,7 +171,7 @@ public void apply(Project project) { // Print global build info header just before task execution // Only do this if we are the root build of a composite if (GradleUtils.isIncludedBuild(project) == false) { - project.getGradle().getTaskGraph().whenReady(graph -> logGlobalBuildInfo()); + project.getGradle().getTaskGraph().whenReady(graph -> logGlobalBuildInfo(buildParams)); } } @@ -179,17 +196,35 @@ private String formatJavaVendorDetails(JvmInstallationMetadata runtimeJdkMetaDat /* Introspect all versions of ES that may be tested against for backwards * compatibility. It is *super* important that this logic is the same as the * logic in VersionUtils.java. */ - private static BwcVersions resolveBwcVersions(File root) { - File versionsFile = new File(root, DEFAULT_VERSION_JAVA_FILE_PATH); - try (var is = new FileInputStream(versionsFile)) { + private BwcVersions resolveBwcVersions() { + String versionsFilePath = elvis( + System.getProperty("BWC_VERSION_SOURCE"), + new File(Util.locateElasticsearchWorkspace(project.getGradle()), DEFAULT_VERSION_JAVA_FILE_PATH).getPath() + ); + try (var is = new FileInputStream(versionsFilePath)) { List versionLines = IOUtils.readLines(is, "UTF-8"); - return new BwcVersions(versionLines); + return new BwcVersions(versionLines, getDevelopmentBranches()); } catch (IOException e) { throw new IllegalStateException("Unable to resolve to resolve bwc versions from versionsFile.", e); } } - private void logGlobalBuildInfo() { + private List getDevelopmentBranches() { + List branches = new ArrayList<>(); + File branchesFile = new File(Util.locateElasticsearchWorkspace(project.getGradle()), "branches.json"); + try (InputStream is = new FileInputStream(branchesFile)) { + JsonNode json = objectMapper.readTree(is); + for (JsonNode node : json.get("branches")) { + branches.add(node.get("branch").asText()); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + return branches; + } + + private void logGlobalBuildInfo(BuildParameterExtension buildParams) { final String osName = System.getProperty("os.name"); final String osVersion = System.getProperty("os.version"); final String osArch = System.getProperty("os.arch"); @@ -201,14 +236,14 @@ private void logGlobalBuildInfo() { LOGGER.quiet("Elasticsearch Build Hamster says Hello!"); LOGGER.quiet(" Gradle Version : " + GradleVersion.current().getVersion()); LOGGER.quiet(" OS Info : " + osName + " " + osVersion + " (" + osArch + ")"); - if (BuildParams.getIsRuntimeJavaHomeSet()) { - JvmInstallationMetadata runtimeJvm = metadataDetector.getMetadata(getJavaInstallation(BuildParams.getRuntimeJavaHome())); + if (buildParams.getIsRuntimeJavaHomeSet()) { + JvmInstallationMetadata runtimeJvm = metadataDetector.getMetadata(getJavaInstallation(buildParams.getRuntimeJavaHome().get())); final String runtimeJvmVendorDetails = runtimeJvm.getVendor().getDisplayName(); final String runtimeJvmImplementationVersion = runtimeJvm.getJvmVersion(); final String runtimeVersion = runtimeJvm.getRuntimeVersion(); final String runtimeExtraDetails = runtimeJvmVendorDetails + ", " + runtimeVersion; LOGGER.quiet(" Runtime JDK Version : " + runtimeJvmImplementationVersion + " (" + runtimeExtraDetails + ")"); - LOGGER.quiet(" Runtime java.home : " + BuildParams.getRuntimeJavaHome()); + LOGGER.quiet(" Runtime java.home : " + buildParams.getRuntimeJavaHome().get()); LOGGER.quiet(" Gradle JDK Version : " + gradleJvmImplementationVersion + " (" + gradleJvmVendorDetails + ")"); LOGGER.quiet(" Gradle java.home : " + gradleJvm.getJavaHome()); } else { @@ -219,8 +254,8 @@ private void logGlobalBuildInfo() { if (javaToolchainHome != null) { LOGGER.quiet(" JAVA_TOOLCHAIN_HOME : " + javaToolchainHome); } - LOGGER.quiet(" Random Testing Seed : " + BuildParams.getTestSeed()); - LOGGER.quiet(" In FIPS 140 mode : " + BuildParams.isInFipsJvm()); + LOGGER.quiet(" Random Testing Seed : " + buildParams.getTestSeed()); + LOGGER.quiet(" In FIPS 140 mode : " + buildParams.getInFipsJvm()); LOGGER.quiet("======================================="); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/JavaHome.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/JavaHome.java index ad64528f88397..28a9c1d3c41f0 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/JavaHome.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/JavaHome.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.info; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/packer/CacheCacheableTestFixtures.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/packer/CacheCacheableTestFixtures.java index bfc52adcdecfd..2af72ad2c075b 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/packer/CacheCacheableTestFixtures.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/packer/CacheCacheableTestFixtures.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.packer; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/packer/CacheTestFixtureResourcesPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/packer/CacheTestFixtureResourcesPlugin.java index c8ce9d5ca2c71..c784a6ccd0130 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/packer/CacheTestFixtureResourcesPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/packer/CacheTestFixtureResourcesPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.packer; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckForbiddenApisTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckForbiddenApisTask.java index d03ec4ab09c95..2666e52b9630e 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckForbiddenApisTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckForbiddenApisTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckstylePrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckstylePrecommitPlugin.java index 978b20003223e..580624be81c94 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckstylePrecommitPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckstylePrecommitPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; @@ -40,8 +41,8 @@ public TaskProvider createTask(Project project) { File checkstyleDir = new File(project.getBuildDir(), "checkstyle"); File checkstyleSuppressions = new File(checkstyleDir, "checkstyle_suppressions.xml"); File checkstyleConf = new File(checkstyleDir, "checkstyle.xml"); - TaskProvider copyCheckstyleConf = project.getTasks().register("copyCheckstyleConf"); - + TaskProvider copyCheckstyleConf = project.getTasks() + .register("copyCheckstyleConf", CopyCheckStyleConfTask.class); // configure inputs and outputs so up to date works properly copyCheckstyleConf.configure(t -> t.getOutputs().files(checkstyleSuppressions, checkstyleConf)); if ("jar".equals(checkstyleConfUrl.getProtocol())) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CopyCheckStyleConfTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CopyCheckStyleConfTask.java new file mode 100644 index 0000000000000..9e0f9c24bcef8 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CopyCheckStyleConfTask.java @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.precommit; + +import org.gradle.api.DefaultTask; +import org.gradle.api.file.FileSystemOperations; + +import javax.inject.Inject; + +public abstract class CopyCheckStyleConfTask extends DefaultTask { + + @Inject + public abstract FileSystemOperations getFs(); +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesPrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesPrecommitPlugin.java index b1d9cbd1f01d1..816e5735ee674 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesPrecommitPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesPrecommitPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTask.java index 07817fdaed1fe..e9efee4fa157d 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/FilePermissionsPrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/FilePermissionsPrecommitPlugin.java index fd166c9c89b2a..9df3e9256ff54 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/FilePermissionsPrecommitPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/FilePermissionsPrecommitPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/FilePermissionsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/FilePermissionsTask.java index f505419122e6d..a198034c3c09b 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/FilePermissionsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/FilePermissionsTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenApisPrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenApisPrecommitPlugin.java index e737459ab0b16..f1ec236efe646 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenApisPrecommitPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenApisPrecommitPlugin.java @@ -1,16 +1,17 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; import org.elasticsearch.gradle.internal.ExportElasticsearchBuildResourcesTask; import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitPlugin; -import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.BuildParameterExtension; import org.gradle.api.Project; import org.gradle.api.Task; import org.gradle.api.plugins.JavaBasePlugin; @@ -29,7 +30,7 @@ public class ForbiddenApisPrecommitPlugin extends PrecommitPlugin { @Override public TaskProvider createTask(Project project) { project.getPluginManager().apply(JavaBasePlugin.class); - + var buildParams = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class); // Create a convenience task for all checks (this does not conflict with extension, as it has higher priority in DSL): var forbiddenTask = project.getTasks() .register(FORBIDDEN_APIS_TASK_NAME, task -> { task.setDescription("Runs forbidden-apis checks."); }); @@ -56,7 +57,7 @@ public TaskProvider createTask(Project project) { t.setClassesDirs(sourceSet.getOutput().getClassesDirs()); t.dependsOn(resourcesTask); t.setClasspath(sourceSet.getRuntimeClasspath().plus(sourceSet.getCompileClasspath())); - t.setTargetCompatibility(BuildParams.getMinimumRuntimeVersion().getMajorVersion()); + t.setTargetCompatibility(buildParams.getMinimumRuntimeVersion().getMajorVersion()); t.getBundledSignatures().set(BUNDLED_SIGNATURE_DEFAULTS); t.setSignaturesFiles( project.files( diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenPatternsPrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenPatternsPrecommitPlugin.java index 144551b0d77cd..b2c62ba9a4628 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenPatternsPrecommitPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenPatternsPrecommitPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenPatternsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenPatternsTask.java index b5582e4acf14a..d84a72b1a6d9d 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenPatternsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenPatternsTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/InternalPrecommitTasks.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/InternalPrecommitTasks.java index c3cc628a05742..762d0d19de859 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/InternalPrecommitTasks.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/InternalPrecommitTasks.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JarHellPrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JarHellPrecommitPlugin.java index 5982201eb691c..0a22a2b61c953 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JarHellPrecommitPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JarHellPrecommitPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; @@ -20,11 +21,11 @@ public class JarHellPrecommitPlugin extends PrecommitPlugin { public TaskProvider createTask(Project project) { project.getPluginManager().apply(JarHellPlugin.class); - if (project.getPath().equals(":libs:elasticsearch-core") == false) { + if (project.getPath().equals(":libs:core") == false) { // ideally we would configure this as a default dependency. But Default dependencies do not work correctly // with gradle project dependencies as they're resolved to late in the build and don't setup according task // dependencies properly - var elasticsearchCoreProject = project.findProject(":libs:elasticsearch-core"); + var elasticsearchCoreProject = project.findProject(":libs:core"); if (elasticsearchCoreProject != null) { project.getDependencies().add("jarHell", elasticsearchCoreProject); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JavaModulePrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JavaModulePrecommitPlugin.java index 2aaef5c1c70e7..8c4c72eaba923 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JavaModulePrecommitPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JavaModulePrecommitPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JavaModulePrecommitTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JavaModulePrecommitTask.java index d031113550295..e4528ba49da73 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JavaModulePrecommitTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/JavaModulePrecommitTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/LicenseAnalyzer.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/LicenseAnalyzer.java index 533066168c604..99aa825635d28 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/LicenseAnalyzer.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/LicenseAnalyzer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/LoggerUsagePrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/LoggerUsagePrecommitPlugin.java index 954ce1ad1f4ea..d3c803a100d4b 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/LoggerUsagePrecommitPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/LoggerUsagePrecommitPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/LoggerUsageTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/LoggerUsageTask.java index 559d7536c310a..36dcdfb531940 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/LoggerUsageTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/LoggerUsageTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/SplitPackagesAuditPrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/SplitPackagesAuditPrecommitPlugin.java index 90195db2ec296..23435aefb4a3c 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/SplitPackagesAuditPrecommitPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/SplitPackagesAuditPrecommitPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/SplitPackagesAuditTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/SplitPackagesAuditTask.java index f75adbe640297..17a87da3eeca3 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/SplitPackagesAuditTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/SplitPackagesAuditTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/TestingConventionsCheckTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/TestingConventionsCheckTask.java index 87f15351c1565..6f90c091e8356 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/TestingConventionsCheckTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/TestingConventionsCheckTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPlugin.java index 6adf422133db3..75e8a623ab0e2 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java index 9e40d96438e48..e45a1d3dd25b1 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; @@ -11,7 +12,6 @@ import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin; import org.elasticsearch.gradle.internal.ExportElasticsearchBuildResourcesTask; import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitPlugin; -import org.elasticsearch.gradle.internal.info.BuildParams; import org.gradle.api.Project; import org.gradle.api.Task; import org.gradle.api.artifacts.Configuration; @@ -22,18 +22,23 @@ import java.nio.file.Path; import static org.elasticsearch.gradle.internal.util.DependenciesUtils.createFileCollectionFromNonTransitiveArtifactsView; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin { public static final String JDK_JAR_HELL_CONFIG_NAME = "jdkJarHell"; - public static final String LIBS_ELASTICSEARCH_CORE_PROJECT_PATH = ":libs:elasticsearch-core"; + public static final String LIBS_ELASTICSEARCH_CORE_PROJECT_PATH = ":libs:core"; @Override public TaskProvider createTask(Project project) { + project.getRootProject().getPlugins().apply(CompileOnlyResolvePlugin.class); + var buildParams = loadBuildParams(project); + project.getPlugins().apply(CompileOnlyResolvePlugin.class); project.getConfigurations().create("forbiddenApisCliJar"); project.getDependencies().add("forbiddenApisCliJar", "de.thetaphi:forbiddenapis:3.6"); Configuration jdkJarHellConfig = project.getConfigurations().create(JDK_JAR_HELL_CONFIG_NAME); + if (project.getPath().equals(LIBS_ELASTICSEARCH_CORE_PROJECT_PATH) == false) { // Internal projects are not all plugins, so make sure the check is available // we are not doing this for this project itself to avoid jar hell with itself @@ -65,9 +70,12 @@ public TaskProvider createTask(Project project) { && ((ModuleComponentIdentifier) identifier).getGroup().startsWith("org.elasticsearch") == false ) ); + if (buildParams.get().getIsRuntimeJavaHomeSet()) { + t.getRuntimeJavaVersion().set(buildParams.get().getRuntimeJavaVersion()); + } t.dependsOn(resourcesTask); - t.getTargetCompatibility().set(project.provider(BuildParams::getRuntimeJavaVersion)); - t.getJavaHome().set(project.provider(BuildParams::getRuntimeJavaHome).map(File::getPath)); + t.getTargetCompatibility().set(buildParams.flatMap(params -> params.getRuntimeJavaVersion())); + t.getJavaHome().set(buildParams.flatMap(params -> params.getRuntimeJavaHome()).map(File::getPath)); t.setSignatureFile(resourcesDir.resolve("forbidden/third-party-audit.txt").toFile()); t.getJdkJarHellClasspath().from(jdkJarHellConfig); t.getForbiddenAPIsClasspath().from(project.getConfigurations().getByName("forbiddenApisCliJar").plus(compileOnly)); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java index 489cff65976b1..442797775de2f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; @@ -12,7 +13,6 @@ import org.apache.commons.io.output.NullOutputStream; import org.elasticsearch.gradle.OS; import org.elasticsearch.gradle.VersionProperties; -import org.elasticsearch.gradle.internal.info.BuildParams; import org.gradle.api.DefaultTask; import org.gradle.api.JavaVersion; import org.gradle.api.file.ArchiveOperations; @@ -193,6 +193,10 @@ public Set getMissingClassExcludes() { @SkipWhenEmpty public abstract ConfigurableFileCollection getJarsToScan(); + @Input + @Optional + public abstract Property getRuntimeJavaVersion(); + @Classpath public FileCollection getClasspath() { return classpath; @@ -370,14 +374,10 @@ private String runForbiddenAPIsCli() throws IOException { /** Returns true iff the build Java version is the same as the given version. */ private boolean isJavaVersion(JavaVersion version) { - if (BuildParams.getIsRuntimeJavaHomeSet()) { - if (version.equals(BuildParams.getRuntimeJavaVersion())) { - return true; - } - } else if (version.getMajorVersion().equals(VersionProperties.getBundledJdkMajorVersion())) { - return true; + if (getRuntimeJavaVersion().isPresent()) { + return getRuntimeJavaVersion().get().equals(version); } - return false; + return version.getMajorVersion().equals(VersionProperties.getBundledJdkMajorVersion()); } private Set runJdkJarHellCheck() throws IOException { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateJsonAgainstSchemaTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateJsonAgainstSchemaTask.java index d174da5bcdb11..2d26dd01064c8 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateJsonAgainstSchemaTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateJsonAgainstSchemaTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateJsonNoKeywordsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateJsonNoKeywordsTask.java index 95f7093bad0a0..738c150e275e0 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateJsonNoKeywordsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateJsonNoKeywordsTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateRestSpecPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateRestSpecPlugin.java index dcc2bf9a3330e..8d265a1eb26a1 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateRestSpecPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateRestSpecPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateYamlAgainstSchemaTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateYamlAgainstSchemaTask.java index f94aef4e55015..bf33858a4b7c2 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateYamlAgainstSchemaTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateYamlAgainstSchemaTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/AbstractVersionsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/AbstractVersionsTask.java index ad39faad1bc85..8903308634ef8 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/AbstractVersionsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/AbstractVersionsTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.release; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BreakingChangesGenerator.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BreakingChangesGenerator.java index 2ca67134b8922..849e05742f2b8 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BreakingChangesGenerator.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BreakingChangesGenerator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.release; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java index 3883bf63e9fc6..ff3ad2f944c52 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.release; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ExtractCurrentVersionsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ExtractCurrentVersionsTask.java index 53dd55041f6bd..76d6aeeca6db9 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ExtractCurrentVersionsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ExtractCurrentVersionsTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.release; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java index 37aa417504170..87aa03f0445c5 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.release; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GitWrapper.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GitWrapper.java index 9cfb649399a90..f50b3c0890220 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GitWrapper.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GitWrapper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.release; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/MigrationIndexGenerator.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/MigrationIndexGenerator.java index ca420294fede0..668da0da6d0d0 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/MigrationIndexGenerator.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/MigrationIndexGenerator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.release; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/MinorVersion.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/MinorVersion.java index 8c78c6b38b79d..4958f4bfefdbb 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/MinorVersion.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/MinorVersion.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.release; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/PruneChangelogsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/PruneChangelogsTask.java index b3c118ca17628..26236779a2390 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/PruneChangelogsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/PruneChangelogsTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.release; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/QualifiedVersion.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/QualifiedVersion.java index 356f943261fa2..2e0737ee4e006 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/QualifiedVersion.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/QualifiedVersion.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.release; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGenerator.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGenerator.java index a562f0f3583f1..25e4bec79c353 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGenerator.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGenerator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.release; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java index 58dc1f2c37299..df0e35c487f54 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.release; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGenerator.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGenerator.java index 81375c44acd3a..b8c195f47d824 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGenerator.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGenerator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.release; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java index ec79fe20492e1..22af46a4e5ccd 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.release; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/SetCompatibleVersionsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/SetCompatibleVersionsTask.java index 17761e5183b31..f2fa39cef860a 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/SetCompatibleVersionsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/SetCompatibleVersionsTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.release; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/TagVersionsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/TagVersionsTask.java index a7f67f87b602e..d030596e6735c 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/TagVersionsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/TagVersionsTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.release; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/TemplateUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/TemplateUtils.java index ef2915f847950..0db9a79f66175 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/TemplateUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/TemplateUtils.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.release; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/UpdateVersionsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/UpdateVersionsTask.java index b19e5c0beacf8..a6ead34b11079 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/UpdateVersionsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/UpdateVersionsTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.release; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/shadow/XmlClassRelocationTransformer.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/shadow/XmlClassRelocationTransformer.java index b365142282785..57afa7014240b 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/shadow/XmlClassRelocationTransformer.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/shadow/XmlClassRelocationTransformer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.shadow; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java index 9eb57eb636aa1..b19c1207d56fb 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java @@ -1,16 +1,16 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.snyk; import groovy.json.JsonOutput; -import org.elasticsearch.gradle.internal.info.BuildParams; import org.gradle.api.DefaultTask; import org.gradle.api.GradleException; import org.gradle.api.artifacts.Configuration; @@ -117,7 +117,7 @@ private Map> projectAttributesData() { } private Object buildTargetData() { - return Map.of("remoteUrl", remoteUrl.get(), "branch", BuildParams.getGitRevision()); + return Map.of("remoteUrl", remoteUrl.get(), "branch", getGitRevision().get()); } @InputFiles @@ -159,4 +159,9 @@ public Property getRemoteUrl() { public Property getTargetReference() { return targetReference; } + + @Input + public Property getGitRevision() { + return targetReference; + } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyGraph.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyGraph.java index fe62eb4e649db..a8fbb622d9b10 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyGraph.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyGraph.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.snyk; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyGraphBuilder.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyGraphBuilder.java index 4210c375921ec..df1c390063a57 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyGraphBuilder.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyGraphBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.snyk; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java index a22de5aae5a94..704394b4f01a9 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java @@ -1,14 +1,16 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.snyk; import org.elasticsearch.gradle.internal.conventions.info.GitInfo; +import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.artifacts.Configuration; @@ -20,6 +22,8 @@ import javax.inject.Inject; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; + public class SnykDependencyMonitoringGradlePlugin implements Plugin { public static final String UPLOAD_TASK_NAME = "uploadSnykDependencyGraph"; @@ -34,10 +38,14 @@ public SnykDependencyMonitoringGradlePlugin(ProjectLayout projectLayout, Provide @Override public void apply(Project project) { + project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class); + var buildParams = loadBuildParams(project); + var generateTaskProvider = project.getTasks() .register("generateSnykDependencyGraph", GenerateSnykDependencyGraph.class, generateSnykDependencyGraph -> { generateSnykDependencyGraph.getProjectPath().set(project.getPath()); generateSnykDependencyGraph.getProjectName().set(project.getName()); + generateSnykDependencyGraph.getGitRevision().set(buildParams.get().getGitRevision()); String projectVersion = project.getVersion().toString(); generateSnykDependencyGraph.getVersion().set(projectVersion); generateSnykDependencyGraph.getGradleVersion().set(project.getGradle().getGradleVersion()); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java index 445496f47eb7d..58b9c594a68bc 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.snyk; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java index 5b1044bbb29a3..d76753f32e344 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test; @@ -15,11 +16,11 @@ import org.elasticsearch.gradle.ElasticsearchDistributionType; import org.elasticsearch.gradle.Version; import org.elasticsearch.gradle.VersionProperties; +import org.elasticsearch.gradle.internal.BwcVersions; import org.elasticsearch.gradle.internal.InternalDistributionDownloadPlugin; import org.elasticsearch.gradle.internal.JdkDownloadPlugin; import org.elasticsearch.gradle.internal.docker.DockerSupportPlugin; import org.elasticsearch.gradle.internal.docker.DockerSupportService; -import org.elasticsearch.gradle.internal.info.BuildParams; import org.elasticsearch.gradle.test.SystemPropertyCommandLineArgumentProvider; import org.elasticsearch.gradle.util.GradleUtils; import org.gradle.api.Action; @@ -30,6 +31,7 @@ import org.gradle.api.artifacts.Configuration; import org.gradle.api.artifacts.dsl.DependencyHandler; import org.gradle.api.artifacts.type.ArtifactTypeDefinition; +import org.gradle.api.file.FileCollection; import org.gradle.api.plugins.JavaPluginExtension; import org.gradle.api.provider.Provider; import org.gradle.api.specs.Specs; @@ -48,12 +50,12 @@ import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.ALL_INTERNAL; import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DEB; import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER; -import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_CLOUD; import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_CLOUD_ESS; import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_IRONBANK; import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_UBI; import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_WOLFI; import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.RPM; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; /** * This class defines gradle tasks for testing our various distribution artifacts. @@ -72,6 +74,7 @@ public void apply(Project project) { project.getPlugins().apply(InternalDistributionDownloadPlugin.class); project.getPlugins().apply(JdkDownloadPlugin.class); project.getPluginManager().apply("elasticsearch.java"); + var buildParams = loadBuildParams(project).get(); Provider dockerSupport = GradleUtils.getBuildService( project.getGradle().getSharedServices(), @@ -84,11 +87,11 @@ public void apply(Project project) { List testDistributions = configureDistributions(project); Map> lifecycleTasks = lifecycleTasks(project, "destructiveDistroTest"); - Map> versionTasks = versionTasks(project, "destructiveDistroUpgradeTest"); + Map> versionTasks = versionTasks(project, "destructiveDistroUpgradeTest", buildParams.getBwcVersions()); TaskProvider destructiveDistroTest = project.getTasks().register("destructiveDistroTest"); - Configuration examplePlugin = configureExamplePlugin(project); - + Configuration examplePluginConfiguration = configureExamplePlugin(project); + FileCollection examplePluginFileCollection = examplePluginConfiguration; List> windowsTestTasks = new ArrayList<>(); Map>> linuxTestTasks = new HashMap<>(); @@ -101,9 +104,9 @@ public void apply(Project project) { t2 -> distribution.isDocker() == false || dockerSupport.get().getDockerAvailability().isAvailable() ); addDistributionSysprop(t, DISTRIBUTION_SYSPROP, distribution::getFilepath); - addDistributionSysprop(t, EXAMPLE_PLUGIN_SYSPROP, () -> examplePlugin.getSingleFile().toString()); + addDistributionSysprop(t, EXAMPLE_PLUGIN_SYSPROP, () -> examplePluginFileCollection.getSingleFile().toString()); t.exclude("**/PackageUpgradeTests.class"); - }, distribution, examplePlugin.getDependencies()); + }, distribution, examplePluginConfiguration.getDependencies()); if (distribution.getPlatform() == Platform.WINDOWS) { windowsTestTasks.add(destructiveTask); @@ -115,7 +118,7 @@ public void apply(Project project) { lifecycleTask.configure(t -> t.dependsOn(destructiveTask)); if ((type == DEB || type == RPM) && distribution.getBundledJdk()) { - for (Version version : BuildParams.getBwcVersions().getIndexCompatible()) { + for (Version version : buildParams.getBwcVersions().getIndexCompatible()) { final ElasticsearchDistribution bwcDistro; if (version.equals(Version.fromString(distribution.getVersion()))) { // this is the same as the distribution we are testing @@ -148,7 +151,6 @@ private static Map> lifecycleTask lifecyleTasks.put(DOCKER, project.getTasks().register(taskPrefix + ".docker")); lifecyleTasks.put(DOCKER_UBI, project.getTasks().register(taskPrefix + ".docker-ubi")); lifecyleTasks.put(DOCKER_IRONBANK, project.getTasks().register(taskPrefix + ".docker-ironbank")); - lifecyleTasks.put(DOCKER_CLOUD, project.getTasks().register(taskPrefix + ".docker-cloud")); lifecyleTasks.put(DOCKER_CLOUD_ESS, project.getTasks().register(taskPrefix + ".docker-cloud-ess")); lifecyleTasks.put(DOCKER_WOLFI, project.getTasks().register(taskPrefix + ".docker-wolfi")); lifecyleTasks.put(ARCHIVE, project.getTasks().register(taskPrefix + ".archives")); @@ -157,10 +159,10 @@ private static Map> lifecycleTask return lifecyleTasks; } - private static Map> versionTasks(Project project, String taskPrefix) { + private static Map> versionTasks(Project project, String taskPrefix, BwcVersions bwcVersions) { Map> versionTasks = new HashMap<>(); - for (Version version : BuildParams.getBwcVersions().getIndexCompatible()) { + for (Version version : bwcVersions.getIndexCompatible()) { versionTasks.put(version.toString(), project.getTasks().register(taskPrefix + ".v" + version)); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/ErrorReportingTestListener.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/ErrorReportingTestListener.java index 4361349392de3..80e64ad813230 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/ErrorReportingTestListener.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/ErrorReportingTestListener.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test; @@ -171,6 +172,11 @@ public Destination getDestination() { return Destination.StdErr; } + @Override + public long getLogTime() { + return result.getEndTime(); + } + @Override public String getMessage() { return message; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/Fixture.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/Fixture.java deleted file mode 100644 index 843c82f794066..0000000000000 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/Fixture.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.gradle.internal.test; - -/** - * Any object that can produce an accompanying stop task, meant to tear down - * a previously instantiated service. - */ -public interface Fixture { - - /** A task which will stop this fixture. This should be used as a finalizedBy for any tasks that use the fixture. */ - Object getStopTask(); - -} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/HistoricalFeaturesMetadataPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/HistoricalFeaturesMetadataPlugin.java index bd9df6d3903ca..be972f11d4586 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/HistoricalFeaturesMetadataPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/HistoricalFeaturesMetadataPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/HistoricalFeaturesMetadataTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/HistoricalFeaturesMetadataTask.java index 0891225d1e1ef..a2ea7af210dfd 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/HistoricalFeaturesMetadataTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/HistoricalFeaturesMetadataTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/InternalClusterTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/InternalClusterTestPlugin.java index d1f5fc1ab4a42..e13c2544ae9cf 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/InternalClusterTestPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/InternalClusterTestPlugin.java @@ -1,14 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test; -import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.elasticsearch.gradle.util.GradleUtils; import org.gradle.api.JavaVersion; import org.gradle.api.Plugin; @@ -17,16 +18,21 @@ import org.gradle.api.tasks.TaskProvider; import org.gradle.api.tasks.testing.Test; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; + public class InternalClusterTestPlugin implements Plugin { public static final String SOURCE_SET_NAME = "internalClusterTest"; @Override public void apply(Project project) { + project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class); + var buildParams = loadBuildParams(project).get(); + TaskProvider internalClusterTest = GradleUtils.addTestSourceSet(project, SOURCE_SET_NAME); internalClusterTest.configure(task -> { // Set GC options to mirror defaults in jvm.options - if (BuildParams.getRuntimeJavaVersion().compareTo(JavaVersion.VERSION_14) < 0) { + if (buildParams.getRuntimeJavaVersion().get().compareTo(JavaVersion.VERSION_14) < 0) { task.jvmArgs("-XX:+UseConcMarkSweepGC", "-XX:CMSInitiatingOccupancyFraction=75", "-XX:+UseCMSInitiatingOccupancyOnly"); } else { task.jvmArgs("-XX:+UseG1GC"); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/LegacyRestTestBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/LegacyRestTestBasePlugin.java index b29efbfab069f..62d91bf76b5d4 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/LegacyRestTestBasePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/LegacyRestTestBasePlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestPlugin.java index baa7704463a6d..c13a5f0e4d30d 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestPlugin.java @@ -1,14 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test; -import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.file.RegularFile; @@ -18,6 +19,8 @@ import java.util.Arrays; import java.util.List; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; + public class MutedTestPlugin implements Plugin { private static final String ADDITIONAL_FILES_PROPERTY = "org.elasticsearch.additional.muted.tests"; @@ -31,6 +34,9 @@ public void apply(Project project) { .map(p -> project.getRootProject().getLayout().getProjectDirectory().file(p)) .toList(); + project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class); + var buildParams = loadBuildParams(project).get(); + Provider mutedTestsProvider = project.getGradle() .getSharedServices() .registerIfAbsent("mutedTests", MutedTestsBuildService.class, spec -> { @@ -45,7 +51,7 @@ public void apply(Project project) { } // Don't fail when all tests are ignored when running in CI - filter.setFailOnNoMatchingTests(BuildParams.isCi() == false); + filter.setFailOnNoMatchingTests(buildParams.isCi() == false); }); }); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestsBuildService.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestsBuildService.java index 0fdb134c81649..df3d1c9b70a94 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestsBuildService.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestsBuildService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test; @@ -27,10 +28,12 @@ import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Set; public abstract class MutedTestsBuildService implements BuildService { - private final List excludePatterns = new ArrayList<>(); + private final Set excludePatterns = new LinkedHashSet<>(); private final ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory()); public MutedTestsBuildService() { @@ -42,23 +45,23 @@ public MutedTestsBuildService() { } } - public List getExcludePatterns() { + public Set getExcludePatterns() { return excludePatterns; } - private List buildExcludePatterns(File file) { + private Set buildExcludePatterns(File file) { List mutedTests; try (InputStream is = new BufferedInputStream(new FileInputStream(file))) { mutedTests = objectMapper.readValue(is, MutedTests.class).getTests(); if (mutedTests == null) { - return Collections.emptyList(); + return Collections.emptySet(); } } catch (IOException e) { throw new UncheckedIOException(e); } - List excludes = new ArrayList<>(); + Set excludes = new LinkedHashSet<>(); if (mutedTests.isEmpty() == false) { for (MutedTestsBuildService.MutedTest mutedTest : mutedTests) { if (mutedTest.getClassName() != null && mutedTest.getMethods().isEmpty() == false) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/RestIntegTestTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/RestIntegTestTask.java index 8000536bc2525..8702722054a5e 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/RestIntegTestTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/RestIntegTestTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/SimpleCommandLineArgumentProvider.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/SimpleCommandLineArgumentProvider.java index edd49f086ccdd..9350fb411a292 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/SimpleCommandLineArgumentProvider.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/SimpleCommandLineArgumentProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/StandaloneRestTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/StandaloneRestTestPlugin.java index 4e0ed0a78cf6b..c69cdf1d0ffb5 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/StandaloneRestTestPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/StandaloneRestTestPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/StandaloneTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/StandaloneTestPlugin.java index a6fc47bc454f8..c67cb6680da99 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/StandaloneTestPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/StandaloneTestPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestUtil.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestUtil.java index 965f3964c9a38..56ae19af259e4 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestUtil.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestUtil.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestWithDependenciesPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestWithDependenciesPlugin.java index fada17f0898dc..a2851bfa2ae55 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestWithDependenciesPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestWithDependenciesPlugin.java @@ -1,19 +1,23 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test; import org.apache.commons.lang.StringUtils; -import org.elasticsearch.gradle.plugin.PluginBuildPlugin; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.artifacts.Configuration; +import org.gradle.api.artifacts.Dependency; import org.gradle.api.artifacts.ProjectDependency; +import org.gradle.api.artifacts.dsl.DependencyHandler; +import org.gradle.api.attributes.Attribute; +import org.gradle.api.attributes.LibraryElements; import org.gradle.api.plugins.ExtraPropertiesExtension; import org.gradle.api.tasks.Copy; import org.gradle.api.tasks.SourceSetContainer; @@ -44,23 +48,31 @@ public void apply(final Project project) { Configuration testImplementationConfig = project.getConfigurations().getByName("testImplementation"); testImplementationConfig.getDependencies().all(dep -> { - if (dep instanceof ProjectDependency - && ((ProjectDependency) dep).getDependencyProject().getPlugins().hasPlugin(PluginBuildPlugin.class)) { - project.getGradle() - .projectsEvaluated(gradle -> addPluginResources(project, ((ProjectDependency) dep).getDependencyProject())); + if (dep instanceof ProjectDependency && dep.getGroup().contains("plugin")) { + addPluginResources(project, ((ProjectDependency) dep)); } }); } - private static void addPluginResources(final Project project, final Project pluginProject) { - final File outputDir = new File(project.getBuildDir(), "/generated-test-resources/" + pluginProject.getName()); - String camelProjectName = stream(pluginProject.getName().split("-")).map(t -> StringUtils.capitalize(t)) + private static void addPluginResources(final Project project, final ProjectDependency projectDependency) { + final File outputDir = new File(project.getBuildDir(), "/generated-test-resources/" + projectDependency.getName()); + String camelProjectName = stream(projectDependency.getName().split("-")).map(t -> StringUtils.capitalize(t)) .collect(Collectors.joining()); String taskName = "copy" + camelProjectName + "Metadata"; + String metadataConfiguration = "resolved" + camelProjectName + "Metadata"; + Configuration pluginMetadata = project.getConfigurations().maybeCreate(metadataConfiguration); + pluginMetadata.getAttributes().attribute(Attribute.of("pluginMetadata", Boolean.class), true); + pluginMetadata.getAttributes() + .attribute( + LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE, + project.getObjects().named(LibraryElements.class, LibraryElements.RESOURCES) + ); + DependencyHandler dependencyHandler = project.getDependencies(); + Dependency pluginMetadataDependency = dependencyHandler.project(Map.of("path", projectDependency.getPath())); + dependencyHandler.add(metadataConfiguration, pluginMetadataDependency); project.getTasks().register(taskName, Copy.class, copy -> { copy.into(outputDir); - copy.from(pluginProject.getTasks().named("pluginProperties")); - copy.from(pluginProject.file("src/main/plugin-metadata")); + copy.from(pluginMetadata); }); Map map = Map.of("builtBy", taskName); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestWithSslPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestWithSslPlugin.java index 833c7ad546a4a..94018d1501e0b 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestWithSslPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestWithSslPlugin.java @@ -1,16 +1,17 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test; import org.elasticsearch.gradle.internal.ExportElasticsearchBuildResourcesTask; import org.elasticsearch.gradle.internal.conventions.util.Util; -import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.BuildParameterExtension; import org.elasticsearch.gradle.internal.precommit.FilePermissionsPrecommitPlugin; import org.elasticsearch.gradle.internal.precommit.ForbiddenPatternsPrecommitPlugin; import org.elasticsearch.gradle.internal.precommit.ForbiddenPatternsTask; @@ -34,6 +35,7 @@ public class TestWithSslPlugin implements Plugin { @Override public void apply(Project project) { File keyStoreDir = new File(project.getBuildDir(), "keystore"); + var buildParams = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class); TaskProvider exportKeyStore = project.getTasks() .register("copyTestCertificates", ExportElasticsearchBuildResourcesTask.class, (t) -> { t.copy("test/ssl/test-client.crt"); @@ -86,7 +88,7 @@ public void apply(Project project) { .getExtensions() .getByName(TestClustersPlugin.EXTENSION_NAME); clusters.configureEach(c -> { - if (BuildParams.isInFipsJvm()) { + if (buildParams.getInFipsJvm()) { c.setting("xpack.security.transport.ssl.key", "test-node.key"); c.keystore("xpack.security.transport.ssl.secure_key_passphrase", "test-node-key-password"); c.setting("xpack.security.transport.ssl.certificate", "test-node.crt"); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestRerunPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestRerunPlugin.java index 1262f0e222e03..15911b7ed24fe 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestRerunPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestRerunPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rerun; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestRerunTaskExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestRerunTaskExtension.java index 5cf2577163013..f08d3dadc1bb0 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestRerunTaskExtension.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestRerunTaskExtension.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rerun; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestTaskConfigurer.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestTaskConfigurer.java index 9d116bbe0d8d8..49f4956a2d16b 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestTaskConfigurer.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestTaskConfigurer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rerun; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rerun/executer/RerunTestExecuter.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rerun/executer/RerunTestExecuter.java index cb85c83152605..450a0029b96d2 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rerun/executer/RerunTestExecuter.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rerun/executer/RerunTestExecuter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rerun.executer; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rerun/executer/RerunTestResultProcessor.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rerun/executer/RerunTestResultProcessor.java index 39b21df531332..2a508d52ce206 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rerun/executer/RerunTestResultProcessor.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rerun/executer/RerunTestResultProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rerun.executer; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestApiTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestApiTask.java index d4b680655a2e1..1617f317d52c9 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestApiTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestApiTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestTestsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestTestsTask.java index 94345ed80eec7..02309bb9c1811 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestTestsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestTestsTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/InternalJavaRestTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/InternalJavaRestTestPlugin.java index 1787ebcccf3a9..56c9a81a4a900 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/InternalJavaRestTestPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/InternalJavaRestTestPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/InternalYamlRestTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/InternalYamlRestTestPlugin.java index ba40998e2b02a..049ad7dec3057 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/InternalYamlRestTestPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/InternalYamlRestTestPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/LegacyJavaRestTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/LegacyJavaRestTestPlugin.java index a1b17c110b343..f821a43706234 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/LegacyJavaRestTestPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/LegacyJavaRestTestPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPlugin.java index 4977c0924efbc..07b8dd92a2d5f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestResourcesExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestResourcesExtension.java index a107cfc512262..2138191c76c94 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestResourcesExtension.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestResourcesExtension.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestResourcesPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestResourcesPlugin.java index 309480d9f600e..4724c576066f1 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestResourcesPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestResourcesPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java index 77af3445f530c..0a0c866e88113 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest; @@ -17,9 +18,8 @@ import org.elasticsearch.gradle.Version; import org.elasticsearch.gradle.VersionProperties; import org.elasticsearch.gradle.distribution.ElasticsearchDistributionTypes; -import org.elasticsearch.gradle.internal.ElasticsearchJavaPlugin; +import org.elasticsearch.gradle.internal.ElasticsearchJavaBasePlugin; import org.elasticsearch.gradle.internal.InternalDistributionDownloadPlugin; -import org.elasticsearch.gradle.internal.info.BuildParams; import org.elasticsearch.gradle.internal.test.ErrorReportingTestListener; import org.elasticsearch.gradle.internal.test.HistoricalFeaturesMetadataPlugin; import org.elasticsearch.gradle.plugin.BasePluginBuildPlugin; @@ -43,6 +43,7 @@ import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.file.FileCollection; import org.gradle.api.file.FileTree; +import org.gradle.api.internal.artifacts.dependencies.ProjectDependencyInternal; import org.gradle.api.provider.ProviderFactory; import org.gradle.api.tasks.ClasspathNormalizer; import org.gradle.api.tasks.PathSensitivity; @@ -57,6 +58,8 @@ import javax.inject.Inject; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; + /** * Base plugin used for wiring up build tasks to REST testing tasks using new JUnit rule-based test clusters framework. */ @@ -88,8 +91,9 @@ public RestTestBasePlugin(ProviderFactory providerFactory) { @Override public void apply(Project project) { - project.getPluginManager().apply(ElasticsearchJavaPlugin.class); + project.getPluginManager().apply(ElasticsearchJavaBasePlugin.class); project.getPluginManager().apply(InternalDistributionDownloadPlugin.class); + var bwcVersions = loadBuildParams(project).get().getBwcVersions(); // Register integ-test and default distributions ElasticsearchDistribution defaultDistro = createDistribution( @@ -218,7 +222,7 @@ public Void call(Object... args) { } Version version = (Version) args[0]; - boolean isReleased = BuildParams.getBwcVersions().unreleasedInfo(version) == null; + boolean isReleased = bwcVersions.unreleasedInfo(version) == null; String versionString = version.toString(); ElasticsearchDistribution bwcDistro = createDistribution(project, "bwc_" + versionString, versionString); @@ -230,9 +234,9 @@ public Void call(Object... args) { providerFactory.provider(() -> bwcDistro.getExtracted().getSingleFile().getPath()) ); - if (version.getMajor() > 0 && version.before(BuildParams.getBwcVersions().getMinimumWireCompatibleVersion())) { + if (version.getMajor() > 0 && version.before(bwcVersions.getMinimumWireCompatibleVersion())) { // If we are upgrade testing older versions we also need to upgrade to 7.last - this.call(BuildParams.getBwcVersions().getMinimumWireCompatibleVersion()); + this.call(bwcVersions.getMinimumWireCompatibleVersion()); } return null; } @@ -244,7 +248,7 @@ private void copyDependencies(Project project, DependencySet dependencies, Confi configuration.getDependencies() .stream() .filter(d -> d instanceof ProjectDependency) - .map(d -> project.getDependencies().project(Map.of("path", ((ProjectDependency) d).getDependencyProject().getPath()))) + .map(d -> project.getDependencies().project(Map.of("path", ((ProjectDependencyInternal) d).getPath()))) .forEach(dependencies::add); } @@ -321,8 +325,9 @@ private Configuration createPluginConfiguration(Project project, String name, bo Collection additionalDependencies = new LinkedHashSet<>(); for (Iterator iterator = dependencies.iterator(); iterator.hasNext();) { Dependency dependency = iterator.next(); + // this logic of relying on other projects metadata should probably live in a build service if (dependency instanceof ProjectDependency projectDependency) { - Project dependencyProject = projectDependency.getDependencyProject(); + Project dependencyProject = project.project(projectDependency.getPath()); List extendedPlugins = dependencyProject.getExtensions() .getByType(PluginPropertiesExtension.class) .getExtendedPlugins(); @@ -332,8 +337,8 @@ private Configuration createPluginConfiguration(Project project, String name, bo iterator.remove(); additionalDependencies.add( useExploded - ? getExplodedBundleDependency(project, dependencyProject.getPath()) - : getBundleZipTaskDependency(project, dependencyProject.getPath()) + ? getExplodedBundleDependency(project, projectDependency.getPath()) + : getBundleZipTaskDependency(project, projectDependency.getPath()) ); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestUtil.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestUtil.java index 99c25b9e25703..910abdeb91090 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestUtil.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestUtil.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java index e0581ebf67081..843f8854d3513 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.compat.compat; @@ -24,6 +25,7 @@ import org.gradle.api.artifacts.Configuration; import org.gradle.api.artifacts.Dependency; import org.gradle.api.file.Directory; +import org.gradle.api.file.FileCollection; import org.gradle.api.file.ProjectLayout; import org.gradle.api.file.RelativePath; import org.gradle.api.internal.file.FileOperations; @@ -73,7 +75,6 @@ public AbstractYamlRestCompatTestPlugin(ProjectLayout projectLayout, FileOperati @Override public void apply(Project project) { - final Path compatRestResourcesDir = Path.of("restResources").resolve("v" + COMPATIBLE_VERSION); final Path compatSpecsDir = compatRestResourcesDir.resolve("yamlSpecs"); final Path compatTestsDir = compatRestResourcesDir.resolve("yamlTests"); @@ -230,10 +231,11 @@ public void apply(Project project) { yamlRestCompatTestTask.configure(testTask -> { testTask.systemProperty("tests.restCompat", true); // Use test runner and classpath from "normal" yaml source set + FileCollection outputFileCollection = yamlCompatTestSourceSet.getOutput(); testTask.setTestClassesDirs( yamlTestSourceSet.getOutput().getClassesDirs().plus(yamlCompatTestSourceSet.getOutput().getClassesDirs()) ); - testTask.onlyIf("Compatibility tests are available", t -> yamlCompatTestSourceSet.getOutput().isEmpty() == false); + testTask.onlyIf("Compatibility tests are available", t -> outputFileCollection.isEmpty() == false); testTask.setClasspath( yamlCompatTestSourceSet.getRuntimeClasspath() // remove the "normal" api and tests diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/LegacyYamlRestCompatTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/LegacyYamlRestCompatTestPlugin.java index e84c84cc426a3..99947b89974d4 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/LegacyYamlRestCompatTestPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/LegacyYamlRestCompatTestPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.compat.compat; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/RestCompatTestTransformTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/RestCompatTestTransformTask.java index 9b1e8a67deec8..7fc16b8306a08 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/RestCompatTestTransformTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/RestCompatTestTransformTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.compat.compat; @@ -134,8 +135,8 @@ public void skipTest(String fullTestName, String reason) { // For example: indices.get_mapping/20_missing_type/Non-existent type returns 404 // However, the folder can be arbitrarily nest so, a == a1/a2/a3, and the test name can include forward slashes, so c == c1/c2/c3 // So we also need to support a1/a2/a3/b/c1/c2/c3 - - String[] testParts = fullTestName.split("/"); + boolean limitTo3Separators = fullTestName.equals("logsdb/20_source_mapping/include/exclude is supported with stored _source"); + String[] testParts = limitTo3Separators ? fullTestName.split("/", 3) : fullTestName.split("/"); if (testParts.length < 3) { throw new IllegalArgumentException( "To skip tests, all 3 parts [folder/file/test name] must be defined. found [" + fullTestName + "]" diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/YamlRestCompatTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/YamlRestCompatTestPlugin.java index 79588ca722ff1..87e8a83183e8b 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/YamlRestCompatTestPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/YamlRestCompatTestPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.compat.compat; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/ReplaceByKey.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/ReplaceByKey.java index 6bb0f8f6db0aa..17d50a1a95db1 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/ReplaceByKey.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/ReplaceByKey.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestContext.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestContext.java index 80b780e656062..83dc716bbb6e1 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestContext.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestContext.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransform.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransform.java index 5d99355ee9163..0704a34788f4f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransform.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransform.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformByParentArray.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformByParentArray.java index 3495040b08a70..b4ba1f1f357fe 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformByParentArray.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformByParentArray.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformByParentObject.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformByParentObject.java index 30fbd0de30340..fd75e92e65c28 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformByParentObject.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformByParentObject.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformGlobalSetup.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformGlobalSetup.java index 95c15b2890e5e..d09515c88ee51 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformGlobalSetup.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformGlobalSetup.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformGlobalTeardown.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformGlobalTeardown.java index 16eec09d07eb9..cb4a20c593758 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformGlobalTeardown.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformGlobalTeardown.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformer.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformer.java index 7237e3ce27ce8..56139d688a390 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformer.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/do_/ReplaceKeyInDo.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/do_/ReplaceKeyInDo.java index b2dd272a67bfe..a57a8126363ae 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/do_/ReplaceKeyInDo.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/do_/ReplaceKeyInDo.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.do_; // 'do' is a reserved word diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/feature/FeatureInjector.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/feature/FeatureInjector.java index ecd87c607947d..4d352b7d8b07f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/feature/FeatureInjector.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/feature/FeatureInjector.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.feature; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/headers/InjectHeaders.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/headers/InjectHeaders.java index f0a32a37cdffb..de2504965ab52 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/headers/InjectHeaders.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/headers/InjectHeaders.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.headers; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/length/ReplaceKeyInLength.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/length/ReplaceKeyInLength.java index 1e912d800dd78..83f8020ab8582 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/length/ReplaceKeyInLength.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/length/ReplaceKeyInLength.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.length; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/length/ReplaceValueInLength.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/length/ReplaceValueInLength.java index a70445d3e7832..936deea70703d 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/length/ReplaceValueInLength.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/length/ReplaceValueInLength.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.length; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/match/AddMatch.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/match/AddMatch.java index 40d2cf6d24adc..ff8ec820d60c4 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/match/AddMatch.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/match/AddMatch.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.match; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/match/RemoveMatch.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/match/RemoveMatch.java index 10c5152ba5376..0c801478f53d2 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/match/RemoveMatch.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/match/RemoveMatch.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.match; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/match/ReplaceKeyInMatch.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/match/ReplaceKeyInMatch.java index eb47ff469fe4f..0a0e2a14794b2 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/match/ReplaceKeyInMatch.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/match/ReplaceKeyInMatch.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.match; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/match/ReplaceValueInMatch.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/match/ReplaceValueInMatch.java index abadf9efdf85d..5e46934c8ba4a 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/match/ReplaceValueInMatch.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/match/ReplaceValueInMatch.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.match; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/Skip.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/Skip.java index c8217d8431316..5597b87d18aa5 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/Skip.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/Skip.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.skip; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceIsFalse.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceIsFalse.java index 859a8232fc833..53a35b09dd087 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceIsFalse.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceIsFalse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.text; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceIsTrue.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceIsTrue.java index 0f8c54caa2a20..4e8cfe1172768 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceIsTrue.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceIsTrue.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.text; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceTextual.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceTextual.java index a02da69026028..6397e938f091c 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceTextual.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceTextual.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.text; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectAllowedWarnings.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectAllowedWarnings.java index f8116d14f2bf5..594617c7076b2 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectAllowedWarnings.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectAllowedWarnings.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.warnings; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectWarnings.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectWarnings.java index 2cff61de0ac12..447147c3c3101 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectWarnings.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectWarnings.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.warnings; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/RemoveWarnings.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/RemoveWarnings.java index 3da7d0f531807..75de89f94d093 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/RemoveWarnings.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/RemoveWarnings.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.warnings; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/DockerComposeThrottle.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/DockerComposeThrottle.java index 2f56ebf8bfb74..57e091609e416 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/DockerComposeThrottle.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/DockerComposeThrottle.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.testfixtures; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixtureDeployment.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixtureDeployment.java index ee6f70db2b788..389aa6b217ab9 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixtureDeployment.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixtureDeployment.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.testfixtures; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixtureTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixtureTask.java index da7bcfa289808..2b86842aaa62f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixtureTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixtureTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.testfixtures; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesDeployPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesDeployPlugin.java index 362ced9c3234e..a934164d11af6 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesDeployPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesDeployPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.testfixtures; @@ -11,7 +12,7 @@ import org.apache.commons.lang.StringUtils; import org.elasticsearch.gradle.Architecture; import org.elasticsearch.gradle.internal.docker.DockerBuildTask; -import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Plugin; import org.gradle.api.Project; @@ -19,6 +20,8 @@ import java.util.Arrays; import java.util.List; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; + public class TestFixturesDeployPlugin implements Plugin { public static final String DEPLOY_FIXTURE_TASK_NAME = "deployFixtureDockerImages"; @@ -26,13 +29,19 @@ public class TestFixturesDeployPlugin implements Plugin { @Override public void apply(Project project) { + project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class); + var buildParams = loadBuildParams(project).get(); NamedDomainObjectContainer fixtures = project.container(TestFixtureDeployment.class); project.getExtensions().add("dockerFixtures", fixtures); - registerDeployTaskPerFixture(project, fixtures); + registerDeployTaskPerFixture(project, fixtures, buildParams.isCi()); project.getTasks().register(DEPLOY_FIXTURE_TASK_NAME, task -> task.dependsOn(project.getTasks().withType(DockerBuildTask.class))); } - private static void registerDeployTaskPerFixture(Project project, NamedDomainObjectContainer fixtures) { + private static void registerDeployTaskPerFixture( + Project project, + NamedDomainObjectContainer fixtures, + boolean isCi + ) { fixtures.all( fixture -> project.getTasks() .register("deploy" + StringUtils.capitalize(fixture.getName()) + "DockerImage", DockerBuildTask.class, task -> { @@ -41,12 +50,12 @@ private static void registerDeployTaskPerFixture(Project project, NamedDomainObj if (baseImages.isEmpty() == false) { task.setBaseImages(baseImages.toArray(new String[baseImages.size()])); } - task.setNoCache(BuildParams.isCi()); + task.setNoCache(isCi); task.setTags( new String[] { resolveTargetDockerRegistry(fixture) + "/" + fixture.getName() + "-fixture:" + fixture.getVersion().get() } ); - task.getPush().set(BuildParams.isCi()); + task.getPush().set(isCi); task.getPlatforms().addAll(Arrays.stream(Architecture.values()).map(a -> a.dockerPlatform).toList()); task.setGroup("Deploy TestFixtures"); task.setDescription("Deploys the " + fixture.getName() + " test fixture"); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java index a16057220ce89..ab28a66d93065 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.testfixtures; @@ -16,7 +17,7 @@ import org.elasticsearch.gradle.internal.docker.DockerSupportPlugin; import org.elasticsearch.gradle.internal.docker.DockerSupportService; -import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.elasticsearch.gradle.test.SystemPropertyCommandLineArgumentProvider; import org.elasticsearch.gradle.util.GradleUtils; import org.gradle.api.Action; @@ -46,6 +47,8 @@ import javax.inject.Inject; +import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; + public class TestFixturesPlugin implements Plugin { private static final Logger LOGGER = Logging.getLogger(TestFixturesPlugin.class); @@ -67,6 +70,8 @@ protected FileSystemOperations getFileSystemOperations() { @Override public void apply(Project project) { project.getRootProject().getPluginManager().apply(DockerSupportPlugin.class); + project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class); + var buildParams = loadBuildParams(project).get(); TaskContainer tasks = project.getTasks(); Provider dockerComposeThrottle = project.getGradle() @@ -126,7 +131,7 @@ public void apply(Project project) { tasks.withType(ComposeUp.class).named("composeUp").configure(t -> { // Avoid running docker-compose tasks in parallel in CI due to some issues on certain Linux distributions - if (BuildParams.isCi()) { + if (buildParams.isCi()) { t.usesService(dockerComposeThrottle); t.usesService(dockerSupport); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/AbstractCustomJavaToolchainResolver.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/AbstractCustomJavaToolchainResolver.java index 267af6e4f2b36..0c6a6bc26156b 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/AbstractCustomJavaToolchainResolver.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/AbstractCustomJavaToolchainResolver.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.toolchain; @@ -33,6 +34,7 @@ static String toArchString(Architecture architecture) { case X86_64 -> "x64"; case AARCH64 -> "aarch64"; case X86 -> "x86"; + default -> throw new UnsupportedOperationException("Architecture " + architecture); }; } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolver.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolver.java index 89a40711c9a19..ffbb9cc072919 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolver.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolver.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.toolchain; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/ArchivedOracleJdkToolchainResolver.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/ArchivedOracleJdkToolchainResolver.java index 913a15517f0af..b1f806b384374 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/ArchivedOracleJdkToolchainResolver.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/ArchivedOracleJdkToolchainResolver.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.toolchain; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/JavaToolChainResolverPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/JavaToolChainResolverPlugin.java index c24b23477811e..b89eb87325754 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/JavaToolChainResolverPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/JavaToolChainResolverPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.toolchain; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java index ec86798e653f1..bb26bfd16721d 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.toolchain; @@ -53,16 +54,16 @@ public String url(String os, String arch, String extension) { } } - record EarlyAccessJdkBuild(JavaLanguageVersion languageVersion, String version, String buildNumber) implements JdkBuild { - + record EarlyAccessJdkBuild(JavaLanguageVersion languageVersion) implements JdkBuild { @Override public String url(String os, String arch, String extension) { + String buildNumber = resolveBuildNumber(languageVersion.asInt()); return "https://download.java.net/java/early_access/jdk" - + version + + languageVersion.asInt() + "/" - + version + + buildNumber + "/GPL/openjdk-" - + version + + languageVersion.asInt() + "-ea+" + buildNumber + "_" @@ -72,6 +73,29 @@ public String url(String os, String arch, String extension) { + "_bin." + extension; } + + private static String resolveBuildNumber(int version) { + String buildNumber = System.getProperty("runtime.java." + version + ".build"); + if (buildNumber != null) { + System.out.println("buildNumber = " + buildNumber); + return buildNumber; + } + buildNumber = System.getProperty("runtime.java.build"); + if (buildNumber != null) { + System.out.println("buildNumber2 = " + buildNumber); + return buildNumber; + } + + switch (version) { + case 24: + // latest explicitly found build number for 24 + return "29"; + case 25: + return "3"; + default: + throw new IllegalArgumentException("Unsupported version " + version); + } + } } private static final Pattern VERSION_PATTERN = Pattern.compile( @@ -87,8 +111,8 @@ public String url(String os, String arch, String extension) { // package private so it can be replaced by tests List builds = List.of( getBundledJdkBuild(), - // 23 early access - new EarlyAccessJdkBuild(JavaLanguageVersion.of(23), "23", "24") + new EarlyAccessJdkBuild(JavaLanguageVersion.of(24)), + new EarlyAccessJdkBuild(JavaLanguageVersion.of(25)) ); private JdkBuild getBundledJdkBuild() { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java index 081c28c14fd91..9080f62f19937 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.util; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ParamsUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ParamsUtils.java new file mode 100644 index 0000000000000..0afe654bc5fbc --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ParamsUtils.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.util; + +import org.elasticsearch.gradle.internal.info.BuildParameterExtension; +import org.elasticsearch.gradle.internal.info.BuildParameterService; +import org.gradle.api.Project; +import org.gradle.api.provider.Property; +import org.gradle.api.services.BuildServiceRegistration; + +public class ParamsUtils { + + public static Property loadBuildParams(Project project) { + BuildServiceRegistration buildParamsRegistrations = (BuildServiceRegistration< + BuildParameterService, + BuildParameterService.Params>) project.getGradle().getSharedServices().getRegistrations().getByName("buildParams"); + Property buildParams = buildParamsRegistrations.getParameters().getBuildParams(); + return buildParams; + } + +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/SerializableFunction.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/SerializableFunction.java index 1f2842ed396d2..c4542648d5e41 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/SerializableFunction.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/SerializableFunction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.util; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/SourceDirectoryCommandLineArgumentProvider.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/SourceDirectoryCommandLineArgumentProvider.java index 30141f021935b..834a301c88b9e 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/SourceDirectoryCommandLineArgumentProvider.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/SourceDirectoryCommandLineArgumentProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.util; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ports/AvailablePortAllocator.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ports/AvailablePortAllocator.java index 61b8e559a9486..6b2a93247113b 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ports/AvailablePortAllocator.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ports/AvailablePortAllocator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.util.ports; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ports/DefaultPortDetector.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ports/DefaultPortDetector.java index dcc1c117031fb..b93fdb4735a3d 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ports/DefaultPortDetector.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ports/DefaultPortDetector.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.util.ports; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ports/DefaultReservedPortRangeFactory.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ports/DefaultReservedPortRangeFactory.java index c89f52de61af8..191024990fe2f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ports/DefaultReservedPortRangeFactory.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ports/DefaultReservedPortRangeFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.util.ports; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ports/PortDetector.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ports/PortDetector.java index 7119145d4badb..a3f840c6c4285 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ports/PortDetector.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ports/PortDetector.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.util.ports; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ports/ReservedPortRange.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ports/ReservedPortRange.java index 811fc10a00ae8..e381075c54271 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ports/ReservedPortRange.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ports/ReservedPortRange.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.util.ports; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ports/ReservedPortRangeFactory.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ports/ReservedPortRangeFactory.java index afcb61c326427..7c08a50c31309 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ports/ReservedPortRangeFactory.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/ports/ReservedPortRangeFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.util.ports; diff --git a/build-tools-internal/src/main/resources/changelog-schema.json b/build-tools-internal/src/main/resources/changelog-schema.json index 593716954780b..451701d74d690 100644 --- a/build-tools-internal/src/main/resources/changelog-schema.json +++ b/build-tools-internal/src/main/resources/changelog-schema.json @@ -32,6 +32,7 @@ "CRUD", "Client", "Cluster Coordination", + "Codec", "Data streams", "DLM", "Discovery-Plugins", @@ -283,6 +284,7 @@ "Cluster and node setting", "Command line tool", "CRUD", + "ES|QL", "Index setting", "Ingest", "JVM option", diff --git a/build-tools-internal/src/main/resources/checkstyle_suppressions.xml b/build-tools-internal/src/main/resources/checkstyle_suppressions.xml index fd01993951959..5fdfebf6849e7 100644 --- a/build-tools-internal/src/main/resources/checkstyle_suppressions.xml +++ b/build-tools-internal/src/main/resources/checkstyle_suppressions.xml @@ -35,6 +35,8 @@ + + diff --git a/build-tools-internal/src/main/resources/fips_java.policy b/build-tools-internal/src/main/resources/fips_java.policy index c259b0bc908d8..781e1247db7a5 100644 --- a/build-tools-internal/src/main/resources/fips_java.policy +++ b/build-tools-internal/src/main/resources/fips_java.policy @@ -5,6 +5,7 @@ grant { permission java.security.SecurityPermission "getProperty.jdk.tls.disabledAlgorithms"; permission java.security.SecurityPermission "getProperty.jdk.certpath.disabledAlgorithms"; permission java.security.SecurityPermission "getProperty.jdk.tls.server.defaultDHEParameters"; + permission java.security.SecurityPermission "getProperty.org.bouncycastle.ec.max_f2m_field_size"; permission java.lang.RuntimePermission "getProtectionDomain"; permission java.util.PropertyPermission "java.runtime.name", "read"; permission org.bouncycastle.crypto.CryptoServicesPermission "tlsAlgorithmsEnabled"; @@ -20,6 +21,6 @@ grant { }; // rely on the caller's socket permissions, the JSSE TLS implementation here is always allowed to connect -grant codeBase "file:${jdk.module.path}/bctls-fips-1.0.17.jar" { +grant codeBase "file:${jdk.module.path}/bctls-fips-1.0.19.jar" { permission java.net.SocketPermission "*", "connect"; }; diff --git a/build-tools-internal/src/main/resources/forbidden/es-all-signatures.txt b/build-tools-internal/src/main/resources/forbidden/es-all-signatures.txt index c49082af9363b..1158ac573792e 100644 --- a/build-tools-internal/src/main/resources/forbidden/es-all-signatures.txt +++ b/build-tools-internal/src/main/resources/forbidden/es-all-signatures.txt @@ -1,8 +1,9 @@ # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". java.nio.file.Paths @ Use org.elasticsearch.core.PathUtils.get() instead. java.nio.file.Path#of(java.net.URI) @ Use org.elasticsearch.core.PathUtils.get() instead. diff --git a/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt b/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt index 7475e77bc0805..73de821d6aca6 100644 --- a/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt +++ b/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt @@ -1,8 +1,9 @@ # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # For third-party dependencies, please put signatures in third-party.txt instead of here. @@ -161,7 +162,7 @@ org.elasticsearch.cluster.ClusterFeatures#nodeFeatures() @defaultMessage ClusterFeatures#allNodeFeatures is for internal use only. Use FeatureService#clusterHasFeature to determine if a feature is present on the cluster. org.elasticsearch.cluster.ClusterFeatures#allNodeFeatures() @defaultMessage ClusterFeatures#clusterHasFeature is for internal use only. Use FeatureService#clusterHasFeature to determine if a feature is present on the cluster. -org.elasticsearch.cluster.ClusterFeatures#clusterHasFeature(org.elasticsearch.features.NodeFeature) +org.elasticsearch.cluster.ClusterFeatures#clusterHasFeature(org.elasticsearch.cluster.node.DiscoveryNodes, org.elasticsearch.features.NodeFeature) @defaultMessage Do not construct this records outside the source files they are declared in org.elasticsearch.cluster.SnapshotsInProgress$ShardSnapshotStatus#(java.lang.String, org.elasticsearch.cluster.SnapshotsInProgress$ShardState, org.elasticsearch.repositories.ShardGeneration, java.lang.String, org.elasticsearch.repositories.ShardSnapshotResult) diff --git a/build-tools-internal/src/main/resources/forbidden/es-test-signatures.txt b/build-tools-internal/src/main/resources/forbidden/es-test-signatures.txt index db872971dc339..a53ea61f454fa 100644 --- a/build-tools-internal/src/main/resources/forbidden/es-test-signatures.txt +++ b/build-tools-internal/src/main/resources/forbidden/es-test-signatures.txt @@ -1,8 +1,9 @@ # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". com.carrotsearch.randomizedtesting.RandomizedTest#globalTempDir() @ Use newTempDirPath() instead com.carrotsearch.randomizedtesting.annotations.Seed @ Don't commit hardcoded seeds diff --git a/build-tools-internal/src/main/resources/forbidden/http-signatures.txt b/build-tools-internal/src/main/resources/forbidden/http-signatures.txt index 45b8313e0d13c..518ffab479e52 100644 --- a/build-tools-internal/src/main/resources/forbidden/http-signatures.txt +++ b/build-tools-internal/src/main/resources/forbidden/http-signatures.txt @@ -1,8 +1,9 @@ # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". @defaultMessage Explicitly specify the ContentType of HTTP entities when creating org.apache.http.entity.StringEntity#(java.lang.String) diff --git a/build-tools-internal/src/main/resources/forbidden/jdk-signatures.txt b/build-tools-internal/src/main/resources/forbidden/jdk-signatures.txt index 67a41d158c73c..74844d416c28e 100644 --- a/build-tools-internal/src/main/resources/forbidden/jdk-signatures.txt +++ b/build-tools-internal/src/main/resources/forbidden/jdk-signatures.txt @@ -1,8 +1,9 @@ # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". @defaultMessage Convert to URI java.net.URL#getPath() diff --git a/build-tools-internal/src/main/resources/forbidden/third-party-audit.txt b/build-tools-internal/src/main/resources/forbidden/third-party-audit.txt index 4628253672bf6..7161aa503aeae 100644 --- a/build-tools-internal/src/main/resources/forbidden/third-party-audit.txt +++ b/build-tools-internal/src/main/resources/forbidden/third-party-audit.txt @@ -1,8 +1,9 @@ # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # Checks that we run against bytecode of third-party dependencies # diff --git a/build-tools-internal/src/main/resources/license-headers/sspl+elastic-license-header.txt b/build-tools-internal/src/main/resources/license-headers/sspl+elastic-license-header.txt index 5c2d5b68ae2e0..ad21076a9ca93 100644 --- a/build-tools-internal/src/main/resources/license-headers/sspl+elastic-license-header.txt +++ b/build-tools-internal/src/main/resources/license-headers/sspl+elastic-license-header.txt @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ diff --git a/build-tools-internal/src/main/resources/minimumGradleVersion b/build-tools-internal/src/main/resources/minimumGradleVersion index 8d04a0f38fab0..9c57ca327c7b7 100644 --- a/build-tools-internal/src/main/resources/minimumGradleVersion +++ b/build-tools-internal/src/main/resources/minimumGradleVersion @@ -1 +1 @@ -8.10 \ No newline at end of file +8.12 \ No newline at end of file diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/BwcVersionsSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/BwcVersionsSpec.groovy index 39a9af38e6a9c..4d033564a42b4 100644 --- a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/BwcVersionsSpec.groovy +++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/BwcVersionsSpec.groovy @@ -1,195 +1,273 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal import spock.lang.Specification -import org.elasticsearch.gradle.Architecture -import org.elasticsearch.gradle.ElasticsearchDistribution import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.internal.BwcVersions.UnreleasedVersionInfo - class BwcVersionsSpec extends Specification { List versionLines = [] - def "current version is next major with last minor staged"() { + def "current version is next major"() { + given: + addVersion('7.17.10', '8.9.0') + addVersion('8.14.0', '9.9.0') + addVersion('8.14.1', '9.9.0') + addVersion('8.14.2', '9.9.0') + addVersion('8.15.0', '9.9.0') + addVersion('8.15.1', '9.9.0') + addVersion('8.15.2', '9.9.0') + addVersion('8.16.0', '9.10.0') + addVersion('8.16.1', '9.10.0') + addVersion('8.17.0', '9.10.0') + addVersion('9.0.0', '10.0.0') + + when: + def bwc = new BwcVersions(versionLines, v('9.0.0'), ['main', '8.x', '8.16', '8.15', '7.17']) + def unreleased = bwc.unreleased.collectEntries { [it, bwc.unreleasedInfo(it)] } + + then: + unreleased == [ + (v('8.15.2')): new UnreleasedVersionInfo(v('8.15.2'), '8.15', ':distribution:bwc:bugfix2'), + (v('8.16.1')): new UnreleasedVersionInfo(v('8.16.1'), '8.16', ':distribution:bwc:bugfix'), + (v('8.17.0')): new UnreleasedVersionInfo(v('8.17.0'), '8.x', ':distribution:bwc:minor'), + (v('9.0.0')): new UnreleasedVersionInfo(v('9.0.0'), 'main', ':distribution'), + ] + bwc.wireCompatible == [v('8.17.0'), v('9.0.0')] + bwc.indexCompatible == [v('8.14.0'), v('8.14.1'), v('8.14.2'), v('8.15.0'), v('8.15.1'), v('8.15.2'), v('8.16.0'), v('8.16.1'), v('8.17.0'), v('9.0.0')] + } + + def "current version is next major with staged minor"() { + given: + addVersion('7.17.10', '8.9.0') + addVersion('8.14.0', '9.9.0') + addVersion('8.14.1', '9.9.0') + addVersion('8.14.2', '9.9.0') + addVersion('8.15.0', '9.9.0') + addVersion('8.15.1', '9.9.0') + addVersion('8.15.2', '9.9.0') + addVersion('8.16.0', '9.10.0') + addVersion('8.16.1', '9.10.0') + addVersion('8.17.0', '9.10.0') + addVersion('8.18.0', '9.10.0') + addVersion('9.0.0', '10.0.0') + + when: + def bwc = new BwcVersions(versionLines, v('9.0.0'), ['main', '8.x', '8.17', '8.16', '8.15', '7.17']) + def unreleased = bwc.unreleased.collectEntries { [it, bwc.unreleasedInfo(it)] } + + then: + unreleased == [ + (v('8.15.2')): new UnreleasedVersionInfo(v('8.15.2'), '8.15', ':distribution:bwc:bugfix2'), + (v('8.16.1')): new UnreleasedVersionInfo(v('8.16.1'), '8.16', ':distribution:bwc:bugfix'), + (v('8.17.0')): new UnreleasedVersionInfo(v('8.17.0'), '8.17', ':distribution:bwc:staged'), + (v('8.18.0')): new UnreleasedVersionInfo(v('8.18.0'), '8.x', ':distribution:bwc:minor'), + (v('9.0.0')): new UnreleasedVersionInfo(v('9.0.0'), 'main', ':distribution'), + ] + bwc.wireCompatible == [v('8.18.0'), v('9.0.0')] + bwc.indexCompatible == [v('8.14.0'), v('8.14.1'), v('8.14.2'), v('8.15.0'), v('8.15.1'), v('8.15.2'), v('8.16.0'), v('8.16.1'), v('8.17.0'), v('8.18.0'), v('9.0.0')] + } + + def "current version is first new minor in major series"() { + given: + addVersion('7.17.10', '8.9.0') + addVersion('8.16.0', '9.10.0') + addVersion('8.16.1', '9.10.0') + addVersion('8.17.0', '9.10.0') + addVersion('8.18.0', '9.10.0') + addVersion('9.0.0', '10.0.0') + addVersion('9.1.0', '10.0.0') + + when: + def bwc = new BwcVersions(versionLines, v('9.1.0'), ['main', '9.0', '8.18']) + def unreleased = bwc.unreleased.collectEntries { [it, bwc.unreleasedInfo(it)] } + + then: + unreleased == [ + (v('8.18.0')): new UnreleasedVersionInfo(v('8.18.0'), '8.18', ':distribution:bwc:maintenance'), + (v('9.0.0')): new UnreleasedVersionInfo(v('9.0.0'), '9.0', ':distribution:bwc:staged'), + (v('9.1.0')): new UnreleasedVersionInfo(v('9.1.0'), 'main', ':distribution'), + ] + bwc.wireCompatible == [v('8.18.0'), v('9.0.0'), v('9.1.0')] + bwc.indexCompatible == [v('8.16.0'), v('8.16.1'), v('8.17.0'), v('8.18.0'), v('9.0.0'), v('9.1.0')] + } + + def "current version is new minor with single bugfix"() { given: - addVersion('7.14.0', '8.9.0') - addVersion('7.14.1', '8.9.0') - addVersion('7.14.2', '8.9.0') - addVersion('7.15.0', '8.9.0') - addVersion('7.15.1', '8.9.0') - addVersion('7.15.2', '8.9.0') - addVersion('7.16.0', '8.10.0') - addVersion('7.16.1', '8.10.0') - addVersion('7.16.2', '8.10.0') - addVersion('7.17.0', '8.10.0') - addVersion('8.0.0', '9.0.0') - addVersion('8.1.0', '9.0.0') + addVersion('7.17.10', '8.9.0') + addVersion('8.16.0', '9.10.0') + addVersion('8.16.1', '9.10.0') + addVersion('8.17.0', '9.10.0') + addVersion('8.18.0', '9.10.0') + addVersion('9.0.0', '10.0.0') + addVersion('9.0.1', '10.0.0') + addVersion('9.1.0', '10.0.0') when: - def bwc = new BwcVersions(versionLines, v('8.1.0')) + def bwc = new BwcVersions(versionLines, v('9.1.0'), ['main', '9.0', '8.18']) def unreleased = bwc.unreleased.collectEntries { [it, bwc.unreleasedInfo(it)] } then: unreleased == [ - (v('7.16.2')): new UnreleasedVersionInfo(v('7.16.2'), '7.16', ':distribution:bwc:bugfix'), - (v('7.17.0')): new UnreleasedVersionInfo(v('7.17.0'), '7.17', ':distribution:bwc:staged'), - (v('8.0.0')): new UnreleasedVersionInfo(v('8.0.0'), '8.0', ':distribution:bwc:minor'), - (v('8.1.0')): new UnreleasedVersionInfo(v('8.1.0'), 'main', ':distribution') + (v('8.18.0')): new UnreleasedVersionInfo(v('8.18.0'), '8.18', ':distribution:bwc:maintenance'), + (v('9.0.1')): new UnreleasedVersionInfo(v('9.0.1'), '9.0', ':distribution:bwc:bugfix'), + (v('9.1.0')): new UnreleasedVersionInfo(v('9.1.0'), 'main', ':distribution'), ] - bwc.wireCompatible == [v('7.17.0'), v('8.0.0'), v('8.1.0')] - bwc.indexCompatible == osFiltered([v('7.14.0'), v('7.14.1'), v('7.14.2'), v('7.15.0'), v('7.15.1'), v('7.15.2'), v('7.16.0'), v('7.16.1'), v('7.16.2'), v('7.17.0'), v('8.0.0'), v('8.1.0')]) + bwc.wireCompatible == [v('8.18.0'), v('9.0.0'), v('9.0.1'), v('9.1.0')] + bwc.indexCompatible == [v('8.16.0'), v('8.16.1'), v('8.17.0'), v('8.18.0'), v('9.0.0'), v('9.0.1'), v('9.1.0')] } - def "current version is next minor with next major and last minor both staged"() { + def "current version is new minor with single bugfix and staged minor"() { given: - addVersion('7.14.0', '8.9.0') - addVersion('7.14.1', '8.9.0') - addVersion('7.14.2', '8.9.0') - addVersion('7.15.0', '8.9.0') - addVersion('7.15.1', '8.9.0') - addVersion('7.15.2', '8.9.0') - addVersion('7.16.0', '8.10.0') - addVersion('7.16.1', '8.10.0') - addVersion('7.17.0', '8.10.0') - addVersion('8.0.0', '9.0.0') - addVersion('8.1.0', '9.1.0') + addVersion('7.17.10', '8.9.0') + addVersion('8.16.0', '9.10.0') + addVersion('8.16.1', '9.10.0') + addVersion('8.17.0', '9.10.0') + addVersion('8.18.0', '9.10.0') + addVersion('9.0.0', '10.0.0') + addVersion('9.0.1', '10.0.0') + addVersion('9.1.0', '10.0.0') + addVersion('9.2.0', '10.0.0') when: - def bwc = new BwcVersions(versionLines, v('8.1.0')) + def bwc = new BwcVersions(versionLines, v('9.2.0'), ['main', '9.1', '9.0', '8.18']) def unreleased = bwc.unreleased.collectEntries { [it, bwc.unreleasedInfo(it)] } then: unreleased == [ - (v('7.16.1')): new UnreleasedVersionInfo(v('7.16.1'), '7.16', ':distribution:bwc:bugfix'), - (v('7.17.0')): new UnreleasedVersionInfo(v('7.17.0'), '7.17', ':distribution:bwc:staged'), - (v('8.0.0')): new UnreleasedVersionInfo(v('8.0.0'), '8.0', ':distribution:bwc:minor'), - (v('8.1.0')): new UnreleasedVersionInfo(v('8.1.0'), 'main', ':distribution') + (v('8.18.0')): new UnreleasedVersionInfo(v('8.18.0'), '8.18', ':distribution:bwc:maintenance'), + (v('9.0.1')): new UnreleasedVersionInfo(v('9.0.1'), '9.0', ':distribution:bwc:bugfix'), + (v('9.1.0')): new UnreleasedVersionInfo(v('9.1.0'), '9.1', ':distribution:bwc:staged'), + (v('9.2.0')): new UnreleasedVersionInfo(v('9.2.0'), 'main', ':distribution'), ] - bwc.wireCompatible == [v('7.17.0'), v('8.0.0'), v('8.1.0')] - bwc.indexCompatible == osFiltered([v('7.14.0'), v('7.14.1'), v('7.14.2'), v('7.15.0'), v('7.15.1'), v('7.15.2'), v('7.16.0'), v('7.16.1'), v('7.17.0'), v('8.0.0'), v('8.1.0')]) + bwc.wireCompatible == [v('8.18.0'), v('9.0.0'), v('9.0.1'), v('9.1.0'), v('9.2.0')] + bwc.indexCompatible == [v('8.16.0'), v('8.16.1'), v('8.17.0'), v('8.18.0'), v('9.0.0'), v('9.0.1'), v('9.1.0'), v('9.2.0')] } - def "current is next minor with upcoming minor staged"() { + def "current version is next minor"() { given: - addVersion('7.14.0', '8.9.0') - addVersion('7.14.1', '8.9.0') - addVersion('7.14.2', '8.9.0') - addVersion('7.15.0', '8.9.0') - addVersion('7.15.1', '8.9.0') - addVersion('7.15.2', '8.9.0') - addVersion('7.16.0', '8.10.0') - addVersion('7.16.1', '8.10.0') - addVersion('7.17.0', '8.10.0') - addVersion('7.17.1', '8.10.0') - addVersion('8.0.0', '9.0.0') - addVersion('8.1.0', '9.1.0') + addVersion('7.16.3', '8.9.0') + addVersion('7.17.0', '8.9.0') + addVersion('7.17.1', '8.9.0') + addVersion('8.14.0', '9.9.0') + addVersion('8.14.1', '9.9.0') + addVersion('8.14.2', '9.9.0') + addVersion('8.15.0', '9.9.0') + addVersion('8.15.1', '9.9.0') + addVersion('8.15.2', '9.9.0') + addVersion('8.16.0', '9.10.0') + addVersion('8.16.1', '9.10.0') + addVersion('8.17.0', '9.10.0') + addVersion('8.17.1', '9.10.0') + addVersion('8.18.0', '9.10.0') when: - def bwc = new BwcVersions(versionLines, v('8.1.0')) + def bwc = new BwcVersions(versionLines, v('8.18.0'), ['main', '8.x', '8.17', '8.16', '7.17']) def unreleased = bwc.unreleased.collectEntries { [it, bwc.unreleasedInfo(it)] } then: unreleased == [ - (v('7.17.1')): new UnreleasedVersionInfo(v('7.17.1'), '7.17', ':distribution:bwc:bugfix'), - (v('8.0.0')): new UnreleasedVersionInfo(v('8.0.0'), '8.0', ':distribution:bwc:staged'), - (v('8.1.0')): new UnreleasedVersionInfo(v('8.1.0'), 'main', ':distribution') + (v('7.17.1')): new UnreleasedVersionInfo(v('7.17.1'), '7.17', ':distribution:bwc:maintenance'), + (v('8.16.1')): new UnreleasedVersionInfo(v('8.16.1'), '8.16', ':distribution:bwc:bugfix2'), + (v('8.17.1')): new UnreleasedVersionInfo(v('8.17.1'), '8.17', ':distribution:bwc:bugfix'), + (v('8.18.0')): new UnreleasedVersionInfo(v('8.18.0'), '8.x', ':distribution'), ] - bwc.wireCompatible == [v('7.17.0'), v('7.17.1'), v('8.0.0'), v('8.1.0')] - bwc.indexCompatible == osFiltered([v('7.14.0'), v('7.14.1'), v('7.14.2'), v('7.15.0'), v('7.15.1'), v('7.15.2'), v('7.16.0'), v('7.16.1'), v('7.17.0'), v('7.17.1'), v('8.0.0'), v('8.1.0')]) + bwc.wireCompatible == [v('7.17.0'), v('7.17.1'), v('8.14.0'), v('8.14.1'), v('8.14.2'), v('8.15.0'), v('8.15.1'), v('8.15.2'), v('8.16.0'), v('8.16.1'), v('8.17.0'), v('8.17.1'), v('8.18.0')] + bwc.indexCompatible == [v('7.16.3'), v('7.17.0'), v('7.17.1'), v('8.14.0'), v('8.14.1'), v('8.14.2'), v('8.15.0'), v('8.15.1'), v('8.15.2'), v('8.16.0'), v('8.16.1'), v('8.17.0'), v('8.17.1'), v('8.18.0')] } - def "current version is staged major"() { + def "current version is new minor with staged minor"() { given: - addVersion('7.14.0', '8.9.0') - addVersion('7.14.1', '8.9.0') - addVersion('7.14.2', '8.9.0') - addVersion('7.15.0', '8.9.0') - addVersion('7.15.1', '8.9.0') - addVersion('7.15.2', '8.9.0') - addVersion('7.16.0', '8.10.0') - addVersion('7.16.1', '8.10.0') - addVersion('7.17.0', '8.10.0') - addVersion('7.17.1', '8.10.0') - addVersion('8.0.0', '9.0.0') + addVersion('7.16.3', '8.9.0') + addVersion('7.17.0', '8.9.0') + addVersion('7.17.1', '8.9.0') + addVersion('8.14.0', '9.9.0') + addVersion('8.14.1', '9.9.0') + addVersion('8.14.2', '9.9.0') + addVersion('8.15.0', '9.9.0') + addVersion('8.15.1', '9.9.0') + addVersion('8.15.2', '9.9.0') + addVersion('8.16.0', '9.10.0') + addVersion('8.16.1', '9.10.0') + addVersion('8.17.0', '9.10.0') + addVersion('8.18.0', '9.10.0') when: - def bwc = new BwcVersions(versionLines, v('8.0.0')) + def bwc = new BwcVersions(versionLines, v('8.18.0'), ['main', '8.x', '8.17', '8.16', '8.15', '7.17']) def unreleased = bwc.unreleased.collectEntries { [it, bwc.unreleasedInfo(it)] } then: unreleased == [ - (v('7.17.1')): new UnreleasedVersionInfo(v('7.17.1'), '7.17', ':distribution:bwc:bugfix'), - (v('8.0.0')): new UnreleasedVersionInfo(v('8.0.0'), 'main', ':distribution'), + (v('7.17.1')): new UnreleasedVersionInfo(v('7.17.1'), '7.17', ':distribution:bwc:maintenance'), + (v('8.15.2')): new UnreleasedVersionInfo(v('8.15.2'), '8.15', ':distribution:bwc:bugfix2'), + (v('8.16.1')): new UnreleasedVersionInfo(v('8.16.1'), '8.16', ':distribution:bwc:bugfix'), + (v('8.17.0')): new UnreleasedVersionInfo(v('8.17.0'), '8.17', ':distribution:bwc:staged'), + (v('8.18.0')): new UnreleasedVersionInfo(v('8.18.0'), '8.x', ':distribution'), ] - bwc.wireCompatible == [v('7.17.0'), v('7.17.1'), v('8.0.0')] - bwc.indexCompatible == osFiltered([v('7.14.0'), v('7.14.1'), v('7.14.2'), v('7.15.0'), v('7.15.1'), v('7.15.2'), v('7.16.0'), v('7.16.1'), v('7.17.0'), v('7.17.1'), v('8.0.0')]) + bwc.wireCompatible == [v('7.17.0'), v('7.17.1'), v('8.14.0'), v('8.14.1'), v('8.14.2'), v('8.15.0'), v('8.15.1'), v('8.15.2'), v('8.16.0'), v('8.16.1'), v('8.17.0'), v('8.18.0')] + bwc.indexCompatible == [v('7.16.3'), v('7.17.0'), v('7.17.1'), v('8.14.0'), v('8.14.1'), v('8.14.2'), v('8.15.0'), v('8.15.1'), v('8.15.2'), v('8.16.0'), v('8.16.1'), v('8.17.0'), v('8.18.0')] } - def "current version is next bugfix"() { + def "current version is first bugfix"() { given: - addVersion('7.14.0', '8.9.0') - addVersion('7.14.1', '8.9.0') - addVersion('7.14.2', '8.9.0') - addVersion('7.15.0', '8.9.0') - addVersion('7.15.1', '8.9.0') - addVersion('7.15.2', '8.9.0') - addVersion('7.16.0', '8.10.0') - addVersion('7.16.1', '8.10.0') - addVersion('7.17.0', '8.10.0') - addVersion('7.17.1', '8.10.0') - addVersion('8.0.0', '9.0.0') - addVersion('8.0.1', '9.0.0') + addVersion('7.16.3', '8.9.0') + addVersion('7.17.0', '8.9.0') + addVersion('7.17.1', '8.9.0') + addVersion('8.14.0', '9.9.0') + addVersion('8.14.1', '9.9.0') + addVersion('8.14.2', '9.9.0') + addVersion('8.15.0', '9.9.0') + addVersion('8.15.1', '9.9.0') + addVersion('8.15.2', '9.9.0') + addVersion('8.16.0', '9.10.0') + addVersion('8.16.1', '9.10.0') when: - def bwc = new BwcVersions(versionLines, v('8.0.1')) + def bwc = new BwcVersions(versionLines, v('8.16.1'), ['main', '8.x', '8.17', '8.16', '8.15', '7.17']) def unreleased = bwc.unreleased.collectEntries { [it, bwc.unreleasedInfo(it)] } then: unreleased == [ (v('7.17.1')): new UnreleasedVersionInfo(v('7.17.1'), '7.17', ':distribution:bwc:maintenance'), - (v('8.0.1')): new UnreleasedVersionInfo(v('8.0.1'), 'main', ':distribution'), + (v('8.15.2')): new UnreleasedVersionInfo(v('8.15.2'), '8.15', ':distribution:bwc:bugfix'), + (v('8.16.1')): new UnreleasedVersionInfo(v('8.16.1'), '8.16', ':distribution'), ] - bwc.wireCompatible == [v('7.17.0'), v('7.17.1'), v('8.0.0'), v('8.0.1')] - bwc.indexCompatible == osFiltered([v('7.14.0'), v('7.14.1'), v('7.14.2'), v('7.15.0'), v('7.15.1'), v('7.15.2'), v('7.16.0'), v('7.16.1'), v('7.17.0'), v('7.17.1'), v('8.0.0'), v('8.0.1')]) + bwc.wireCompatible == [v('7.17.0'), v('7.17.1'), v('8.14.0'), v('8.14.1'), v('8.14.2'), v('8.15.0'), v('8.15.1'), v('8.15.2'), v('8.16.0'), v('8.16.1')] + bwc.indexCompatible == [v('7.16.3'), v('7.17.0'), v('7.17.1'), v('8.14.0'), v('8.14.1'), v('8.14.2'), v('8.15.0'), v('8.15.1'), v('8.15.2'), v('8.16.0'), v('8.16.1')] } - def "current version is next minor with no staged releases"() { + def "current version is second bugfix"() { given: - addVersion('7.14.0', '8.9.0') - addVersion('7.14.1', '8.9.0') - addVersion('7.14.2', '8.9.0') - addVersion('7.15.0', '8.9.0') - addVersion('7.15.1', '8.9.0') - addVersion('7.15.2', '8.9.0') - addVersion('7.16.0', '8.10.0') - addVersion('7.16.1', '8.10.0') - addVersion('7.17.0', '8.10.0') - addVersion('7.17.1', '8.10.0') - addVersion('8.0.0', '9.0.0') - addVersion('8.0.1', '9.0.0') - addVersion('8.1.0', '9.1.0') + addVersion('7.16.3', '8.9.0') + addVersion('7.17.0', '8.9.0') + addVersion('7.17.1', '8.9.0') + addVersion('8.14.0', '9.9.0') + addVersion('8.14.1', '9.9.0') + addVersion('8.14.2', '9.9.0') + addVersion('8.15.0', '9.9.0') + addVersion('8.15.1', '9.9.0') + addVersion('8.15.2', '9.9.0') when: - def bwc = new BwcVersions(versionLines, v('8.1.0')) + def bwc = new BwcVersions(versionLines, v('8.15.2'), ['main', '8.x', '8.17', '8.16', '8.15', '7.17']) def unreleased = bwc.unreleased.collectEntries { [it, bwc.unreleasedInfo(it)] } then: unreleased == [ (v('7.17.1')): new UnreleasedVersionInfo(v('7.17.1'), '7.17', ':distribution:bwc:maintenance'), - (v('8.0.1')): new UnreleasedVersionInfo(v('8.0.1'), '8.0', ':distribution:bwc:bugfix'), - (v('8.1.0')): new UnreleasedVersionInfo(v('8.1.0'), 'main', ':distribution') + (v('8.15.2')): new UnreleasedVersionInfo(v('8.15.2'), '8.15', ':distribution'), ] - bwc.wireCompatible == [v('7.17.0'), v('7.17.1'), v('8.0.0'), v('8.0.1'), v('8.1.0')] - bwc.indexCompatible == osFiltered([v('7.14.0'), v('7.14.1'), v('7.14.2'), v('7.15.0'), v('7.15.1'), v('7.15.2'), v('7.16.0'), v('7.16.1'), v('7.17.0'), v('7.17.1'), v('8.0.0'), v('8.0.1'), v('8.1.0')]) + bwc.wireCompatible == [v('7.17.0'), v('7.17.1'), v('8.14.0'), v('8.14.1'), v('8.14.2'), v('8.15.0'), v('8.15.1'), v('8.15.2')] + bwc.indexCompatible == [v('7.16.3'), v('7.17.0'), v('7.17.1'), v('8.14.0'), v('8.14.1'), v('8.14.2'), v('8.15.0'), v('8.15.1'), v('8.15.2')] } private void addVersion(String elasticsearch, String lucene) { @@ -202,12 +280,4 @@ class BwcVersionsSpec extends Specification { return Version.fromString(version) } - private boolean osxAarch64() { - Architecture.current() == Architecture.AARCH64 && - ElasticsearchDistribution.CURRENT_PLATFORM.equals(ElasticsearchDistribution.Platform.DARWIN) - } - - private List osFiltered(ArrayList versions) { - return osxAarch64() ? versions.findAll {it.onOrAfter("7.16.0")} : versions - } } diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/AbstractSnippetParserSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/AbstractSnippetParserSpec.groovy index 8690c738f0d95..51201146474c3 100644 --- a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/AbstractSnippetParserSpec.groovy +++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/AbstractSnippetParserSpec.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.doc diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/AsciidocParserSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/AsciidocParserSpec.groovy index a80215cd82f0d..7933d78597943 100644 --- a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/AsciidocParserSpec.groovy +++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/AsciidocParserSpec.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.doc diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/DocSnippetTaskSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/DocSnippetTaskSpec.groovy index 2b6582bd633ef..d9086321fa419 100644 --- a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/DocSnippetTaskSpec.groovy +++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/DocSnippetTaskSpec.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.doc diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/DocTestUtils.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/DocTestUtils.groovy index 350d8638c8005..96e351b6e873d 100644 --- a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/DocTestUtils.groovy +++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/DocTestUtils.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.doc; diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/MdxSnippetParserSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/MdxSnippetParserSpec.groovy index 020b920de3d0e..8a5b5ff2c6b1c 100644 --- a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/MdxSnippetParserSpec.groovy +++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/MdxSnippetParserSpec.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.doc diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/RestTestsFromDocSnippetTaskSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/RestTestsFromDocSnippetTaskSpec.groovy index dde1931afaa41..3ebf0bb494c6c 100644 --- a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/RestTestsFromDocSnippetTaskSpec.groovy +++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/RestTestsFromDocSnippetTaskSpec.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.doc diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/SnippetBuilderSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/SnippetBuilderSpec.groovy index 278728ec176c1..4a1e1c33fadb2 100644 --- a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/SnippetBuilderSpec.groovy +++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/SnippetBuilderSpec.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.doc diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/info/BuildParameterExtensionSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/info/BuildParameterExtensionSpec.groovy new file mode 100644 index 0000000000000..ce63069a873ab --- /dev/null +++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/info/BuildParameterExtensionSpec.groovy @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.info + +import spock.lang.Ignore +import spock.lang.Specification + +import org.gradle.api.JavaVersion +import org.gradle.api.Project +import org.gradle.api.provider.Provider +import org.gradle.api.provider.ProviderFactory +import org.gradle.testfixtures.ProjectBuilder +import org.junit.Assert + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executors +import java.util.concurrent.Future +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger + +import static org.junit.Assert.fail + +class BuildParameterExtensionSpec extends Specification { + + ProjectBuilder projectBuilder = new ProjectBuilder() + + @Ignore + def "#getterName is cached anc concurrently accessible"() { + given: + def project = projectBuilder.build() + def providers = project.getProviders(); + def buildParams = extension(project, providers) + int numberOfThreads = 10; + when: + var service = Executors.newFixedThreadPool(numberOfThreads) + var latch = new CountDownLatch(numberOfThreads) + def testedProvider = buildParams."$getterName"() + def futures = (1..numberOfThreads).collect { + service.submit( + () -> { + try { + testedProvider.get() + } catch (AssertionError e) { + latch.countDown() + Assert.fail("Accessing cached provider more than once") + } + latch.countDown() + } + ) + } + latch.await(10, TimeUnit.SECONDS) + + then: + futures.collect { it.state() }.any() { it == Future.State.FAILED } == false + + where: + getterName << [ + "getRuntimeJavaHome", + "getJavaToolChainSpec", + "getRuntimeJavaDetails", + "getRuntimeJavaVersion", + "getBwcVersionsProvider" + ] + } + + private BuildParameterExtension extension(Project project, ProviderFactory providers) { + return project.getExtensions().create( + BuildParameterExtension.class, "buildParameters", DefaultBuildParameterExtension.class, + providers, + providerMock(), + providerMock(), + providerMock(), + true, + providerMock(), + [ + Mock(JavaHome), + Mock(JavaHome), + ], + JavaVersion.VERSION_11, + JavaVersion.VERSION_11, + JavaVersion.VERSION_11, + providerMock(), + providerMock(), + "testSeed", + false, + 5, + true, + // cannot use Mock here because of the way the provider is used by gradle internal property api + providerMock() + ) + } + + private Provider providerMock() { + Provider provider = Mock(Provider) + AtomicInteger counter = new AtomicInteger(0) + provider.getOrNull() >> { + println "accessing provider" + return counter.get() == 1 ? fail("Accessing cached provider more than once") : counter.incrementAndGet() + } + provider.get() >> { + fail("Accessing cached provider directly") + } + return provider + + } +} diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/test/rerun/executer/RerunTestResultProcessorTestSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/test/rerun/executer/RerunTestResultProcessorTestSpec.groovy index 507a99f080df2..edfebb83be4c6 100644 --- a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/test/rerun/executer/RerunTestResultProcessorTestSpec.groovy +++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/test/rerun/executer/RerunTestResultProcessorTestSpec.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rerun.executer diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/AbstractToolchainResolverSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/AbstractToolchainResolverSpec.groovy index de0d969c730dc..cea96437129a6 100644 --- a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/AbstractToolchainResolverSpec.groovy +++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/AbstractToolchainResolverSpec.groovy @@ -1,13 +1,16 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.toolchain +import spock.lang.Unroll + import org.gradle.api.provider.Property import org.gradle.jvm.toolchain.JavaLanguageVersion import org.gradle.jvm.toolchain.JavaToolchainDownload @@ -25,6 +28,7 @@ import static org.gradle.platform.OperatingSystem.MAC_OS abstract class AbstractToolchainResolverSpec extends Specification { + @Unroll def "resolves #os #arch #vendor jdk #langVersion"() { given: def resolver = resolverImplementation() diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolverSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolverSpec.groovy index fe4a644ddfc1d..5abb78b062c39 100644 --- a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolverSpec.groovy +++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolverSpec.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.toolchain diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/ArchivedOracleJdkToolchainResolverSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/ArchivedOracleJdkToolchainResolverSpec.groovy index dd6e7b324e745..444deedfd9488 100644 --- a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/ArchivedOracleJdkToolchainResolverSpec.groovy +++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/ArchivedOracleJdkToolchainResolverSpec.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.toolchain diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolverSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolverSpec.groovy index 82bcbca3785d6..4993bf00f2af5 100644 --- a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolverSpec.groovy +++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolverSpec.groovy @@ -1,22 +1,28 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.toolchain +import spock.util.environment.RestoreSystemProperties import org.gradle.api.services.BuildServiceParameters import org.gradle.jvm.toolchain.JavaLanguageVersion +import org.gradle.jvm.toolchain.JavaToolchainDownload + import static org.gradle.jvm.toolchain.JvmVendorSpec.ORACLE -import static org.gradle.platform.Architecture.* +import static org.gradle.platform.Architecture.AARCH64 +import static org.gradle.platform.Architecture.X86_64 import static org.gradle.platform.OperatingSystem.* class OracleOpenJdkToolchainResolverSpec extends AbstractToolchainResolverSpec { + OracleOpenJdkToolchainResolver resolverImplementation() { var toolChain = new OracleOpenJdkToolchainResolver() { @Override @@ -24,10 +30,13 @@ class OracleOpenJdkToolchainResolverSpec extends AbstractToolchainResolverSpec { return null } } - toolChain.builds = [ - new OracleOpenJdkToolchainResolver.ReleasedJdkBuild(JavaLanguageVersion.of(20), "20", "36", "bdc68b4b9cbc4ebcb30745c85038d91d"), - new OracleOpenJdkToolchainResolver.EarlyAccessJdkBuild(JavaLanguageVersion.of(21), "21", "6") - ] + toolChain.builds = toolChain.builds.findAll { it instanceof OracleOpenJdkToolchainResolver.EarlyAccessJdkBuild } + [ + new OracleOpenJdkToolchainResolver.ReleasedJdkBuild( + JavaLanguageVersion.of(20), + "20", + "36", + "bdc68b4b9cbc4ebcb30745c85038d91d" + )] toolChain } @@ -43,23 +52,67 @@ class OracleOpenJdkToolchainResolverSpec extends AbstractToolchainResolverSpec { [20, anyVendor(), LINUX, AARCH64, "https://download.oracle.com/java/GA/jdk20/bdc68b4b9cbc4ebcb30745c85038d91d/36/GPL/openjdk-20_linux-aarch64_bin.tar.gz"], [20, anyVendor(), WINDOWS, X86_64, "https://download.oracle.com/java/GA/jdk20/bdc68b4b9cbc4ebcb30745c85038d91d/36/GPL/openjdk-20_windows-x64_bin.zip"], // https://download.java.net/java/early_access/jdk23/23/GPL/openjdk-23-ea+23_macos-aarch64_bin.tar.gz - [21, ORACLE, MAC_OS, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_macos-x64_bin.tar.gz"], - [21, ORACLE, MAC_OS, AARCH64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_macos-aarch64_bin.tar.gz"], - [21, ORACLE, LINUX, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_linux-x64_bin.tar.gz"], - [21, ORACLE, LINUX, AARCH64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_linux-aarch64_bin.tar.gz"], - [21, ORACLE, WINDOWS, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_windows-x64_bin.zip"], - [21, anyVendor(), MAC_OS, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_macos-x64_bin.tar.gz"], - [21, anyVendor(), MAC_OS, AARCH64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_macos-aarch64_bin.tar.gz"], - [21, anyVendor(), LINUX, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_linux-x64_bin.tar.gz"], - [21, anyVendor(), LINUX, AARCH64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_linux-aarch64_bin.tar.gz"], - [21, anyVendor(), WINDOWS, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_windows-x64_bin.zip"] - ] + [24, ORACLE, MAC_OS, X86_64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_macos-x64_bin.tar.gz"], + [24, ORACLE, MAC_OS, AARCH64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_macos-aarch64_bin.tar.gz"], + [24, ORACLE, LINUX, X86_64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_linux-x64_bin.tar.gz"], + [24, ORACLE, LINUX, AARCH64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_linux-aarch64_bin.tar.gz"], + [24, ORACLE, WINDOWS, X86_64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_windows-x64_bin.zip"], + [24, anyVendor(), MAC_OS, X86_64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_macos-x64_bin.tar.gz"], + [24, anyVendor(), MAC_OS, AARCH64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_macos-aarch64_bin.tar.gz"], + [24, anyVendor(), LINUX, X86_64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_linux-x64_bin.tar.gz"], + [24, anyVendor(), LINUX, AARCH64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_linux-aarch64_bin.tar.gz"], + [24, anyVendor(), WINDOWS, X86_64, "https://download.java.net/java/early_access/jdk24/29/GPL/openjdk-24-ea+29_windows-x64_bin.zip"]] + } + + @RestoreSystemProperties + def "can provide build number for ea versions"() { + given: + System.setProperty('runtime.java.build', "42") + System.setProperty('runtime.java.25.build', "13") + def resolver = resolverImplementation() + + when: + Optional download = resolver.resolve( + request( + JavaLanguageVersion.of(version), + vendor, + platform(os, arch) + ) + ) + + then: + download.get().uri == URI.create(expectedUrl) + + where: + version | vendor | os | arch | expectedUrl + 24 | ORACLE | MAC_OS | X86_64 | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_macos-x64_bin.tar.gz" + 24 | ORACLE | MAC_OS | AARCH64 | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_macos-aarch64_bin.tar.gz" + 24 | ORACLE | LINUX | X86_64 | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_linux-x64_bin.tar.gz" + 24 | ORACLE | LINUX | AARCH64 | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_linux-aarch64_bin.tar.gz" + 24 | ORACLE | WINDOWS | X86_64 | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_windows-x64_bin.zip" + 24 | anyVendor() | MAC_OS | X86_64 | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_macos-x64_bin.tar.gz" + 24 | anyVendor() | MAC_OS | AARCH64 | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_macos-aarch64_bin.tar.gz" + 24 | anyVendor() | LINUX | X86_64 | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_linux-x64_bin.tar.gz" + 24 | anyVendor() | LINUX | AARCH64 | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_linux-aarch64_bin.tar.gz" + 24 | anyVendor() | WINDOWS | X86_64 | urlPrefix(24) + "42/GPL/openjdk-24-ea+42_windows-x64_bin.zip" + 25 | ORACLE | MAC_OS | X86_64 | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_macos-x64_bin.tar.gz" + 25 | ORACLE | MAC_OS | AARCH64 | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_macos-aarch64_bin.tar.gz" + 25 | ORACLE | LINUX | X86_64 | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_linux-x64_bin.tar.gz" + 25 | ORACLE | LINUX | AARCH64 | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_linux-aarch64_bin.tar.gz" + 25 | ORACLE | WINDOWS | X86_64 | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_windows-x64_bin.zip" + 25 | anyVendor() | MAC_OS | X86_64 | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_macos-x64_bin.tar.gz" + 25 | anyVendor() | MAC_OS | AARCH64 | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_macos-aarch64_bin.tar.gz" + 25 | anyVendor() | LINUX | X86_64 | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_linux-x64_bin.tar.gz" + 25 | anyVendor() | LINUX | AARCH64 | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_linux-aarch64_bin.tar.gz" + 25 | anyVendor() | WINDOWS | X86_64 | urlPrefix(25) + "13/GPL/openjdk-25-ea+13_windows-x64_bin.zip" + } + + private static String urlPrefix(int i) { + return "https://download.java.net/java/early_access/jdk" + i + "/" } def unsupportedRequests() { - [ - [20, ORACLE, WINDOWS, AARCH64] - ] + [[20, ORACLE, WINDOWS, AARCH64]] } } diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/AbstractDistributionDownloadPluginTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/AbstractDistributionDownloadPluginTests.java index 8b5cae2825441..7512fa20814c6 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/AbstractDistributionDownloadPluginTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/AbstractDistributionDownloadPluginTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle; @@ -15,6 +16,7 @@ import java.io.File; import java.util.Arrays; +import java.util.List; public class AbstractDistributionDownloadPluginTests { protected static Project rootProject; @@ -27,22 +29,27 @@ public class AbstractDistributionDownloadPluginTests { protected static final Version BWC_STAGED_VERSION = Version.fromString("1.0.0"); protected static final Version BWC_BUGFIX_VERSION = Version.fromString("1.0.1"); protected static final Version BWC_MAINTENANCE_VERSION = Version.fromString("0.90.1"); + protected static final List DEVELOPMENT_BRANCHES = Arrays.asList("main", "1.1", "1.0", "0.90"); protected static final BwcVersions BWC_MINOR = new BwcVersions( BWC_MAJOR_VERSION, - Arrays.asList(BWC_BUGFIX_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION) + Arrays.asList(BWC_BUGFIX_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION), + DEVELOPMENT_BRANCHES ); protected static final BwcVersions BWC_STAGED = new BwcVersions( BWC_MAJOR_VERSION, - Arrays.asList(BWC_MAINTENANCE_VERSION, BWC_STAGED_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION) + Arrays.asList(BWC_MAINTENANCE_VERSION, BWC_STAGED_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION), + DEVELOPMENT_BRANCHES ); protected static final BwcVersions BWC_BUGFIX = new BwcVersions( BWC_MAJOR_VERSION, - Arrays.asList(BWC_BUGFIX_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION) + Arrays.asList(BWC_BUGFIX_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION), + DEVELOPMENT_BRANCHES ); protected static final BwcVersions BWC_MAINTENANCE = new BwcVersions( BWC_MINOR_VERSION, - Arrays.asList(BWC_MAINTENANCE_VERSION, BWC_BUGFIX_VERSION, BWC_MINOR_VERSION) + Arrays.asList(BWC_MAINTENANCE_VERSION, BWC_BUGFIX_VERSION, BWC_MINOR_VERSION), + DEVELOPMENT_BRANCHES ); protected static String projectName(String base, boolean bundledJdk) { diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/ConcatFilesTaskTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/ConcatFilesTaskTests.java index 1e912f23a9923..bb2ba43329731 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/ConcatFilesTaskTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/ConcatFilesTaskTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/EmptyDirTaskTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/EmptyDirTaskTests.java index 6aea053738c09..083804d84b405 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/EmptyDirTaskTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/EmptyDirTaskTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginTests.java index de896b459bda5..c21d9a2b2bb3a 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/JdkDownloadPluginTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/JdkDownloadPluginTests.java index df19e34f9d370..6462e249d6b03 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/JdkDownloadPluginTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/JdkDownloadPluginTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/checkstyle/SnipptLengthCheckTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/checkstyle/SnipptLengthCheckTests.java index efdf7563d74d2..e31fdf3d7db4f 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/checkstyle/SnipptLengthCheckTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/checkstyle/SnipptLengthCheckTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.checkstyle; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/docker/DockerSupportServiceTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/docker/DockerSupportServiceTests.java index ec0c155abbdcc..7e7a211efadc4 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/docker/DockerSupportServiceTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/docker/DockerSupportServiceTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.docker; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/docker/TransformLog4jConfigFilterTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/docker/TransformLog4jConfigFilterTests.java index 827f8fd6288ba..3d0cd8cd03d59 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/docker/TransformLog4jConfigFilterTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/docker/TransformLog4jConfigFilterTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.docker; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTaskTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTaskTests.java index b909970638753..936a8379a5621 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTaskTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTaskTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/precommit/FilePermissionsTaskTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/precommit/FilePermissionsTaskTests.java index c200ea850e611..168b2da109852 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/precommit/FilePermissionsTaskTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/precommit/FilePermissionsTaskTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/precommit/ForbiddenPatternsTaskTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/precommit/ForbiddenPatternsTaskTests.java index 919f3c78ffb9a..947558f1e7a31 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/precommit/ForbiddenPatternsTaskTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/precommit/ForbiddenPatternsTaskTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.precommit; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.java index 0a811b38bb203..5d51f3514cbc3 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.release; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ExtractCurrentVersionsTaskTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ExtractCurrentVersionsTaskTests.java index 1dd4675756f94..a3a0c79d1690a 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ExtractCurrentVersionsTaskTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ExtractCurrentVersionsTaskTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.release; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTaskTest.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTaskTest.java index 125adb9cb3a8d..78bdb8017f476 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTaskTest.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTaskTest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.release; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/PruneChangelogsTaskTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/PruneChangelogsTaskTests.java index 90f4d6b1d353b..4fe924a7a724d 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/PruneChangelogsTaskTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/PruneChangelogsTaskTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.release; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.java index db39c6eea7e86..4e8320d8c0b65 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.release; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java index f8a27c41770dc..cbb53698c586b 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.release; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGeneratorTest.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGeneratorTest.java index 8bb3f868d9de0..2a32040035ec7 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGeneratorTest.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexGeneratorTest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.release; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/SetCompatibleVersionsTaskTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/SetCompatibleVersionsTaskTests.java index eecb953a44eb6..5ae252e3d8d44 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/SetCompatibleVersionsTaskTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/SetCompatibleVersionsTaskTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.release; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/TagVersionsTaskTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/TagVersionsTaskTests.java index 28f92faf49b06..23216a601a5d0 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/TagVersionsTaskTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/TagVersionsTaskTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.release; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/UpdateVersionsTaskTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/UpdateVersionsTaskTests.java index 97441990d47c2..9e4f1cd3a913d 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/UpdateVersionsTaskTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/UpdateVersionsTaskTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.release; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/AssertObjectNodes.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/AssertObjectNodes.java index ae3a61f4a57f9..81be69addb41f 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/AssertObjectNodes.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/AssertObjectNodes.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/TransformTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/TransformTests.java index b608abb7c8a9c..78eaaecd5363e 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/TransformTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/TransformTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/do_/ReplaceKeyInDoTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/do_/ReplaceKeyInDoTests.java index 82a9719ed83b4..22bd009ee1492 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/do_/ReplaceKeyInDoTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/do_/ReplaceKeyInDoTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.do_; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/feature/InjectFeatureTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/feature/InjectFeatureTests.java index 05811f81a2b6c..127f27ccf65bd 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/feature/InjectFeatureTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/feature/InjectFeatureTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.feature; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/header/InjectHeaderTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/header/InjectHeaderTests.java index b1210aa728a04..3ba9ab0f697e0 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/header/InjectHeaderTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/header/InjectHeaderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.header; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/length/ReplaceKeyInLengthTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/length/ReplaceKeyInLengthTests.java index 3c2606157be6c..c1c31843d399f 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/length/ReplaceKeyInLengthTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/length/ReplaceKeyInLengthTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.length; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/length/ReplaceValueInLengthTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/length/ReplaceValueInLengthTests.java index f44ed1ec9a789..17d7bb1c65f3e 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/length/ReplaceValueInLengthTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/length/ReplaceValueInLengthTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.length; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/AddMatchTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/AddMatchTests.java index 126a6034009d6..21dce6f6c95d6 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/AddMatchTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/AddMatchTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.match; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/RemoveMatchTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/RemoveMatchTests.java index 3b30cfcb7361e..665c5b90c8073 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/RemoveMatchTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/RemoveMatchTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.match; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/ReplaceKeyInMatchTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/ReplaceKeyInMatchTests.java index 9151fc13b8843..24bf55d15167c 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/ReplaceKeyInMatchTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/ReplaceKeyInMatchTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.match; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/ReplaceValueInMatchTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/ReplaceValueInMatchTests.java index 6c8ed274c4fbd..f1ed68f1f3d64 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/ReplaceValueInMatchTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/ReplaceValueInMatchTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.match; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/SkipTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/SkipTests.java index 32225164a0d4b..c62351e7e0c9d 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/SkipTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/SkipTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.skip; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceTextualTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceTextualTests.java index f4af8e8320c62..0fcbd0a94b333 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceTextualTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceTextualTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.text; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectAllowedWarningsRegexTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectAllowedWarningsRegexTests.java index 7c37a6137476f..76df1cfde5525 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectAllowedWarningsRegexTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectAllowedWarningsRegexTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.warnings; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectAllowedWarningsTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectAllowedWarningsTests.java index a6e7a42d8639f..e6cae5da6d7e3 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectAllowedWarningsTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectAllowedWarningsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.warnings; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectWarningsRegexTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectWarningsRegexTests.java index 0132fa5c0c52d..9a999815eff63 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectWarningsRegexTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectWarningsRegexTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.warnings; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectWarningsTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectWarningsTests.java index ef3ab0e84678d..a858149098aa1 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectWarningsTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectWarningsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.warnings; diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/RemoveWarningsTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/RemoveWarningsTests.java index 6ff9430d79c62..dcc0711afd22c 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/RemoveWarningsTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/RemoveWarningsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test.rest.transform.warnings; diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index 1dd9fb95bd17b..9d404cc4127b3 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -1,8 +1,8 @@ -elasticsearch = 8.16.0 -lucene = 9.11.1 +elasticsearch = 8.18.0 +lucene = 9.12.1 bundled_jdk_vendor = openjdk -bundled_jdk = 22.0.1+8@c7ec1332f7bb44aeba2eb341ae18aca4 +bundled_jdk = 23+37@3c5b90190c68498b986a97f276efd28a # optional dependencies spatial4j = 0.7 jts = 1.15.0 @@ -14,9 +14,12 @@ log4j = 2.19.0 slf4j = 2.0.6 ecsLogging = 1.2.0 jna = 5.12.1 -netty = 4.1.109.Final +netty = 4.1.115.Final commons_lang3 = 3.9 google_oauth_client = 1.34.1 +awsv1sdk = 1.12.270 +awsv2sdk = 2.28.13 +reactive_streams = 1.0.4 antlr4 = 4.13.1 # bouncy castle version for non-fips. fips jars use a different version @@ -30,10 +33,10 @@ httpcore = 4.4.13 httpasyncclient = 4.1.5 commonslogging = 1.2 commonscodec = 1.15 -protobuf = 3.21.9 +protobuf = 3.25.5 # test dependencies -randomizedrunner = 2.8.0 +randomizedrunner = 2.8.2 junit = 4.13.2 junit5 = 5.7.1 hamcrest = 2.1 diff --git a/build-tools/build.gradle b/build-tools/build.gradle index 7ba5e9f6faa62..e457999fedfee 100644 --- a/build-tools/build.gradle +++ b/build-tools/build.gradle @@ -1,16 +1,14 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ buildscript { repositories { - maven { - url 'https://jitpack.io' - } mavenCentral() } } @@ -116,9 +114,6 @@ configurations { } repositories { - maven { - url 'https://jitpack.io' - } mavenCentral() gradlePluginPortal() } diff --git a/build-tools/reaper/src/main/java/org/elasticsearch/gradle/reaper/Reaper.java b/build-tools/reaper/src/main/java/org/elasticsearch/gradle/reaper/Reaper.java index c2adaf59ead1b..2696962532cc9 100644 --- a/build-tools/reaper/src/main/java/org/elasticsearch/gradle/reaper/Reaper.java +++ b/build-tools/reaper/src/main/java/org/elasticsearch/gradle/reaper/Reaper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.reaper; diff --git a/build-tools/settings.gradle b/build-tools/settings.gradle index 7590b8b6b054e..fce1f3b33d524 100644 --- a/build-tools/settings.gradle +++ b/build-tools/settings.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ pluginManagement { includeBuild "../build-conventions" diff --git a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/DistributionDownloadPluginFuncTest.groovy b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/DistributionDownloadPluginFuncTest.groovy index 3a06bdf917ff6..1332a2fdf241c 100644 --- a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/DistributionDownloadPluginFuncTest.groovy +++ b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/DistributionDownloadPluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle diff --git a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/LoggedExecFuncTest.groovy b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/LoggedExecFuncTest.groovy index 5a92f61c70d8c..a8299a6cc28d1 100644 --- a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/LoggedExecFuncTest.groovy +++ b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/LoggedExecFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle @@ -20,7 +21,7 @@ class LoggedExecFuncTest extends AbstractGradleFuncTest { def setup() { buildFile << """ - // we need apply any custom plugin + // we need apply any custom plugin // to add build-logic to the build classpath plugins { id 'elasticsearch.distribution-download' @@ -54,7 +55,7 @@ class LoggedExecFuncTest extends AbstractGradleFuncTest { import org.elasticsearch.gradle.LoggedExec tasks.register('loggedExec', LoggedExec) { commandLine 'ls', 'wtf' - getSpoolOutput().set($spooling) + getSpoolOutput().set($spooling) } """ when: @@ -81,7 +82,7 @@ class LoggedExecFuncTest extends AbstractGradleFuncTest { println 'OUTPUT ' + output } } - + """ when: def result = gradleRunner("loggedExec").build() @@ -106,7 +107,7 @@ class LoggedExecFuncTest extends AbstractGradleFuncTest { result.task(':loggedExec').outcome == TaskOutcome.FAILED assertOutputContains(result.output, '''\ FAILURE: Build failed with an exception. - + * What went wrong: Execution failed for task ':loggedExec'. > Capturing output is not supported when spoolOutput is true.'''.stripIndent()) @@ -138,13 +139,13 @@ class LoggedExecFuncTest extends AbstractGradleFuncTest { def "can provide standard input"() { setup: file('script.sh') << """ -#!/bin/bash - -# Read the user input - -echo "Enter the user input: " -read userInput -echo "The user input is \$userInput" +#!/bin/bash + +# Read the user input + +echo "Enter the user input: " +read userInput +echo "The user input is \$userInput" """ buildFile << """ import org.elasticsearch.gradle.LoggedExec diff --git a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy index 22efa8d08d3e7..bbc47bedeffc6 100644 --- a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy +++ b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle @@ -134,7 +135,7 @@ class TestClustersPluginFuncTest extends AbstractGradleFuncTest { esplugin { name = 'test-$pluginType' - classname 'org.acme.TestModule' + classname = 'org.acme.TestModule' description = "test $pluginType description" } diff --git a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/jarhell/JarHellPluginFuncTest.groovy b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/jarhell/JarHellPluginFuncTest.groovy index 89b9141eb2b2a..2e1a831b9b471 100644 --- a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/jarhell/JarHellPluginFuncTest.groovy +++ b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/jarhell/JarHellPluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.jarhell @@ -22,7 +23,7 @@ class JarHellPluginFuncTest extends AbstractJavaGradleFuncTest { // bring the build tools on the test classpath id 'elasticsearch.esplugin' apply false } - + repositories { maven { name = "local-test" @@ -55,7 +56,7 @@ class JarHellPluginFuncTest extends AbstractJavaGradleFuncTest { buildFile << """ apply plugin: org.elasticsearch.gradle.jarhell.JarHellPlugin apply plugin: 'java' - + """ when: @@ -79,7 +80,7 @@ class JarHellPluginFuncTest extends AbstractJavaGradleFuncTest { buildFile << """ apply plugin: org.elasticsearch.gradle.jarhell.JarHellPlugin apply plugin: 'java' - + dependencies { jarHell "org.elasticsearch:elasticsearch-core:$version" } diff --git a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/plugin/PluginBuildPluginFuncTest.groovy b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/plugin/PluginBuildPluginFuncTest.groovy index 9c501295391f5..72c67514f3a5c 100644 --- a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/plugin/PluginBuildPluginFuncTest.groovy +++ b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/plugin/PluginBuildPluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.plugin diff --git a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/plugin/StablePluginBuildPluginFuncTest.groovy b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/plugin/StablePluginBuildPluginFuncTest.groovy index 9b0a44ad97103..697d2027a707a 100644 --- a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/plugin/StablePluginBuildPluginFuncTest.groovy +++ b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/plugin/StablePluginBuildPluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.plugin diff --git a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/reaper/ReaperPluginFuncTest.groovy b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/reaper/ReaperPluginFuncTest.groovy index 8cae82cccdf43..14a5594b115f7 100644 --- a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/reaper/ReaperPluginFuncTest.groovy +++ b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/reaper/ReaperPluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.reaper @@ -19,12 +20,12 @@ class ReaperPluginFuncTest extends AbstractGradleFuncTest { plugins { id 'elasticsearch.reaper' } - + import org.elasticsearch.gradle.ReaperPlugin; import org.elasticsearch.gradle.util.GradleUtils; - + def serviceProvider = GradleUtils.getBuildService(project.getGradle().getSharedServices(), ReaperPlugin.REAPER_SERVICE_NAME); - + tasks.register("launchReaper") { doLast { def reaper = serviceProvider.get() diff --git a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/GradleTestPolicySetupPluginFuncTest.groovy b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/GradleTestPolicySetupPluginFuncTest.groovy index 2353b7e2f7d34..82af5b78f63e8 100644 --- a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/GradleTestPolicySetupPluginFuncTest.groovy +++ b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/GradleTestPolicySetupPluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.test @@ -17,7 +18,7 @@ class GradleTestPolicySetupPluginFuncTest extends AbstractGradleFuncTest { given: file("src/test/java/org/acme/SysPropTest.java") << """ package org.acme; - + import static org.junit.Assert.*; import org.junit.After; import org.junit.Before; @@ -39,11 +40,11 @@ class GradleTestPolicySetupPluginFuncTest extends AbstractGradleFuncTest { id "elasticsearch.test-gradle-policy" id "java" } - + repositories { mavenCentral() } - + dependencies { testImplementation "junit:junit:4.13" } @@ -60,4 +61,4 @@ class GradleTestPolicySetupPluginFuncTest extends AbstractGradleFuncTest { then: // still up-to-date result.task(":test").outcome == TaskOutcome.UP_TO_DATE } -} \ No newline at end of file +} diff --git a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/JavaRestTestPluginFuncTest.groovy b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/JavaRestTestPluginFuncTest.groovy index 1d3ac0030a0b5..51d2dfa74b03d 100644 --- a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/JavaRestTestPluginFuncTest.groovy +++ b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/JavaRestTestPluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.test diff --git a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/YamlRestTestPluginFuncTest.groovy b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/YamlRestTestPluginFuncTest.groovy index 1598d6ab06ff0..0bffa5e9e61f6 100644 --- a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/YamlRestTestPluginFuncTest.groovy +++ b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/YamlRestTestPluginFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.test diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/AbstractLazyPropertyCollection.java b/build-tools/src/main/java/org/elasticsearch/gradle/AbstractLazyPropertyCollection.java index c19a698f3f60d..17e3e038cdacc 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/AbstractLazyPropertyCollection.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/AbstractLazyPropertyCollection.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/Architecture.java b/build-tools/src/main/java/org/elasticsearch/gradle/Architecture.java index 34874b62d9489..c7e6546e66b9a 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/Architecture.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/Architecture.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/DistributionDependency.java b/build-tools/src/main/java/org/elasticsearch/gradle/DistributionDependency.java index 44071305628b8..44654ba12ca86 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/DistributionDependency.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/DistributionDependency.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java index d4747c9a6c38e..4c7290457e7df 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle; @@ -88,7 +89,9 @@ public void apply(Project project) { private void setupDistributionContainer(Project project) { distributionsContainer = project.container(ElasticsearchDistribution.class, name -> { var fileConfiguration = project.getConfigurations().create(DISTRO_CONFIG_PREFIX + name); + fileConfiguration.setCanBeConsumed(false); var extractedConfiguration = project.getConfigurations().create(DISTRO_EXTRACTED_CONFIG_PREFIX + name); + extractedConfiguration.setCanBeConsumed(false); extractedConfiguration.getAttributes() .attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE); var distribution = new ElasticsearchDistribution( diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/DistributionResolution.java b/build-tools/src/main/java/org/elasticsearch/gradle/DistributionResolution.java index 0d8177dea5cb6..7414b915dbaef 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/DistributionResolution.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/DistributionResolution.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java b/build-tools/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java index afb90ba1ca62e..3f9669af0568b 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/ElasticsearchDistributionType.java b/build-tools/src/main/java/org/elasticsearch/gradle/ElasticsearchDistributionType.java index c229b2983e539..94c00518b325f 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/ElasticsearchDistributionType.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/ElasticsearchDistributionType.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/FileSupplier.java b/build-tools/src/main/java/org/elasticsearch/gradle/FileSupplier.java index 300749698162a..eb615f7137492 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/FileSupplier.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/FileSupplier.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/FileSystemOperationsAware.java b/build-tools/src/main/java/org/elasticsearch/gradle/FileSystemOperationsAware.java index 23a188fa13557..039d3828b63da 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/FileSystemOperationsAware.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/FileSystemOperationsAware.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/LazyFileOutputStream.java b/build-tools/src/main/java/org/elasticsearch/gradle/LazyFileOutputStream.java index dd7dd45649b0c..c3da389fc30d4 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/LazyFileOutputStream.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/LazyFileOutputStream.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle; @@ -38,6 +39,12 @@ public void write(byte b[], int off, int len) throws IOException { bootstrap(); delegate.write(b, off, len); } + + @Override + public void write(byte b[]) throws IOException { + bootstrap(); + delegate.write(b); + } }; } diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/LazyPropertyList.java b/build-tools/src/main/java/org/elasticsearch/gradle/LazyPropertyList.java index f44e3ea2979cd..a63bed6562643 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/LazyPropertyList.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/LazyPropertyList.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/LazyPropertyMap.java b/build-tools/src/main/java/org/elasticsearch/gradle/LazyPropertyMap.java index db6d07e2b1d7b..2586d8c7ba279 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/LazyPropertyMap.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/LazyPropertyMap.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java b/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java index 3a425d11ccf17..505e9a5b114d1 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/NamedComponentScannerMock.java b/build-tools/src/main/java/org/elasticsearch/gradle/NamedComponentScannerMock.java index cfd3ab566b33d..e358a035834f7 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/NamedComponentScannerMock.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/NamedComponentScannerMock.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/OS.java b/build-tools/src/main/java/org/elasticsearch/gradle/OS.java index 60a534bff540b..d5649f94cf1a4 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/OS.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/OS.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/PropertyNormalization.java b/build-tools/src/main/java/org/elasticsearch/gradle/PropertyNormalization.java index 941e4b77f114d..059cd6bd4ff8c 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/PropertyNormalization.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/PropertyNormalization.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/ReaperPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/ReaperPlugin.java index 7fa1e451fff0b..31fdc77cce87c 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/ReaperPlugin.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/ReaperPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/ReaperService.java b/build-tools/src/main/java/org/elasticsearch/gradle/ReaperService.java index afd8b13c839f8..d09c4a79bae56 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/ReaperService.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/ReaperService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/Version.java b/build-tools/src/main/java/org/elasticsearch/gradle/Version.java index 27a9f0b352f3b..f0bc936e683a2 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/Version.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/Version.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/VersionProperties.java b/build-tools/src/main/java/org/elasticsearch/gradle/VersionProperties.java index c4312d113ffd8..88c7ea3684226 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/VersionProperties.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/VersionProperties.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/dependencies/CompileOnlyResolvePlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/dependencies/CompileOnlyResolvePlugin.java index 68bb0c6e5c36c..23d6205bb6f98 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/dependencies/CompileOnlyResolvePlugin.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/dependencies/CompileOnlyResolvePlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.dependencies; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/distribution/ArchiveElasticsearchDistributionType.java b/build-tools/src/main/java/org/elasticsearch/gradle/distribution/ArchiveElasticsearchDistributionType.java index b19ea81d6b496..f04390004a6d7 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/distribution/ArchiveElasticsearchDistributionType.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/distribution/ArchiveElasticsearchDistributionType.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.distribution; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/distribution/ElasticsearchDistributionTypes.java b/build-tools/src/main/java/org/elasticsearch/gradle/distribution/ElasticsearchDistributionTypes.java index 0c19e000034a8..8a53d56e3c4ce 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/distribution/ElasticsearchDistributionTypes.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/distribution/ElasticsearchDistributionTypes.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.distribution; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/distribution/IntegTestZipElasticsearchDistributionType.java b/build-tools/src/main/java/org/elasticsearch/gradle/distribution/IntegTestZipElasticsearchDistributionType.java index 441b2c173ec48..5b1058ae08c63 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/distribution/IntegTestZipElasticsearchDistributionType.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/distribution/IntegTestZipElasticsearchDistributionType.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.distribution; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/jarhell/JarHellPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/jarhell/JarHellPlugin.java index fd3da7b5a02df..6b8bd9833d043 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/jarhell/JarHellPlugin.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/jarhell/JarHellPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.jarhell; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/jarhell/JarHellTask.java b/build-tools/src/main/java/org/elasticsearch/gradle/jarhell/JarHellTask.java index e07a82b662eca..5a6eae905fc71 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/jarhell/JarHellTask.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/jarhell/JarHellTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.jarhell; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/BasePluginBuildPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/BasePluginBuildPlugin.java index d824d1f5cd334..42e576012c0c9 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/BasePluginBuildPlugin.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/BasePluginBuildPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.plugin; @@ -23,6 +24,8 @@ import org.gradle.api.Task; import org.gradle.api.Transformer; import org.gradle.api.artifacts.type.ArtifactTypeDefinition; +import org.gradle.api.attributes.Attribute; +import org.gradle.api.attributes.LibraryElements; import org.gradle.api.file.CopySpec; import org.gradle.api.file.FileCollection; import org.gradle.api.file.RegularFile; @@ -125,9 +128,27 @@ private TaskProvider createBundleTasks(final Project project, PluginPropert // know about the plugin (used by test security code to statically initialize the plugin in unit tests) var testSourceSet = project.getExtensions().getByType(SourceSetContainer.class).getByName("test"); Map map = Map.of("builtBy", buildProperties); - testSourceSet.getOutput().dir(map, new File(project.getBuildDir(), "generated-resources")); + + File generatedResources = new File(project.getBuildDir(), "generated-resources"); + testSourceSet.getOutput().dir(map, generatedResources); testSourceSet.getResources().srcDir(pluginMetadata); + // expose the plugin properties and metadata for other plugins to use in their tests. + // See TestWithDependenciesPlugin for how this is used. + project.getConfigurations().create("pluginMetadata", conf -> { + conf.getAttributes().attribute(Attribute.of("pluginMetadata", Boolean.class), true); + conf.getAttributes() + .attribute( + LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE, + project.getObjects().named(LibraryElements.class, LibraryElements.RESOURCES) + ); + }); + + project.getArtifacts().add("pluginMetadata", new File(project.getBuildDir(), "generated-descriptor"), artifact -> { + artifact.builtBy(buildProperties); + }); + project.getArtifacts().add("pluginMetadata", pluginMetadata); + // getAttributes().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, "plugin-metadata"); var bundleSpec = createBundleSpec(project, pluginMetadata, buildProperties); extension.setBundleSpec(bundleSpec); // create the actual bundle task, which zips up all the files for the plugin @@ -166,7 +187,7 @@ private static CopySpec createBundleSpec( copySpec.exclude("plugin-security.codebases"); }); bundleSpec.from( - (Callable>) () -> project.getPluginManager().hasPlugin("com.github.johnrengelman.shadow") + (Callable>) () -> project.getPluginManager().hasPlugin("com.gradleup.shadow") ? project.getTasks().named("shadowJar") : project.getTasks().named("jar") ); diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/GenerateNamedComponentsTask.java b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/GenerateNamedComponentsTask.java index 7945bce426cc9..1b63d1230ff91 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/GenerateNamedComponentsTask.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/GenerateNamedComponentsTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.plugin; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/GeneratePluginPropertiesTask.java b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/GeneratePluginPropertiesTask.java index 6e5c5e50db34a..e144122f97770 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/GeneratePluginPropertiesTask.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/GeneratePluginPropertiesTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.plugin; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java index 3fde5ed8dd8fa..d5fec104c063e 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.plugin; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.java b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.java index ceb791ce09b33..e87d2e50ee121 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.plugin; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/StablePluginBuildPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/StablePluginBuildPlugin.java index ef2d1631d5604..086aa3839e342 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/StablePluginBuildPlugin.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/StablePluginBuildPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.plugin; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/test/GradleTestPolicySetupPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/test/GradleTestPolicySetupPlugin.java index 9593a281686e7..2068ee4447971 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/test/GradleTestPolicySetupPlugin.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/test/GradleTestPolicySetupPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.test; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/test/JavaRestTestPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/test/JavaRestTestPlugin.java index 7ed7d669c6a2d..de86e525a82f6 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/test/JavaRestTestPlugin.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/test/JavaRestTestPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.test; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/test/SystemPropertyCommandLineArgumentProvider.java b/build-tools/src/main/java/org/elasticsearch/gradle/test/SystemPropertyCommandLineArgumentProvider.java index 069e51330f0d8..70be689ca637f 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/test/SystemPropertyCommandLineArgumentProvider.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/test/SystemPropertyCommandLineArgumentProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.test; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/test/YamlRestTestPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/test/YamlRestTestPlugin.java index 04985e532992b..cbd39a5878acd 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/test/YamlRestTestPlugin.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/test/YamlRestTestPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.test; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/DefaultTestClustersTask.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/DefaultTestClustersTask.java index e80d2ed64cabd..10ad64aa8e59f 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/DefaultTestClustersTask.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/DefaultTestClustersTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.testclusters; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java index 54962ac241f75..77393fe16b4c2 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.testclusters; @@ -75,6 +76,7 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named { private final LinkedHashMap> waitConditions = new LinkedHashMap<>(); private final transient Project project; private final Provider reaper; + private final Provider testClustersRegistryProvider; private final FileSystemOperations fileSystemOperations; private final ArchiveOperations archiveOperations; private final ExecOperations execOperations; @@ -86,11 +88,14 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named { private boolean shared = false; + private int claims = 0; + public ElasticsearchCluster( String path, String clusterName, Project project, Provider reaper, + Provider testClustersRegistryProvider, FileSystemOperations fileSystemOperations, ArchiveOperations archiveOperations, ExecOperations execOperations, @@ -103,6 +108,7 @@ public ElasticsearchCluster( this.clusterName = clusterName; this.project = project; this.reaper = reaper; + this.testClustersRegistryProvider = testClustersRegistryProvider; this.fileSystemOperations = fileSystemOperations; this.archiveOperations = archiveOperations; this.execOperations = execOperations; @@ -119,6 +125,7 @@ public ElasticsearchCluster( clusterName + "-0", project, reaper, + testClustersRegistryProvider, fileSystemOperations, archiveOperations, execOperations, @@ -176,6 +183,7 @@ public void setNumberOfNodes(int numberOfNodes) { clusterName + "-" + i, project, reaper, + testClustersRegistryProvider, fileSystemOperations, archiveOperations, execOperations, @@ -407,6 +415,7 @@ public void setPreserveDataDir(boolean preserveDataDir) { public void freeze() { nodes.forEach(ElasticsearchNode::freeze); configurationFrozen.set(true); + nodes.whenObjectAdded(node -> { throw new IllegalStateException("Cannot add nodes to test cluster after is has been frozen"); }); } private void checkFrozen() { @@ -662,4 +671,11 @@ public String toString() { return "cluster{" + path + ":" + clusterName + "}"; } + int addClaim() { + return ++this.claims; + } + + int removeClaim() { + return --this.claims; + } } diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java index 0b62243e66afe..4cb67e249b0b0 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.testclusters; @@ -97,7 +98,7 @@ public class ElasticsearchNode implements TestClusterConfiguration { private static final int ES_DESTROY_TIMEOUT = 20; private static final TimeUnit ES_DESTROY_TIMEOUT_UNIT = TimeUnit.SECONDS; - private static final int NODE_UP_TIMEOUT = 2; + private static final int NODE_UP_TIMEOUT = 3; private static final TimeUnit NODE_UP_TIMEOUT_UNIT = TimeUnit.MINUTES; private static final int ADDITIONAL_CONFIG_TIMEOUT = 15; private static final TimeUnit ADDITIONAL_CONFIG_TIMEOUT_UNIT = TimeUnit.SECONDS; @@ -123,6 +124,8 @@ public class ElasticsearchNode implements TestClusterConfiguration { private final String name; transient private final Project project; private final Provider reaperServiceProvider; + private final Provider testClustersRegistryProvider; + private final FileSystemOperations fileSystemOperations; private final ArchiveOperations archiveOperations; private final ExecOperations execOperations; @@ -163,7 +166,6 @@ public class ElasticsearchNode implements TestClusterConfiguration { private final List distributions = new ArrayList<>(); private int currentDistro = 0; private TestDistribution testDistribution; - private volatile Process esProcess; private Function nameCustomization = s -> s; private boolean isWorkingDirConfigured = false; private String httpPort = "0"; @@ -178,6 +180,7 @@ public class ElasticsearchNode implements TestClusterConfiguration { String name, Project project, Provider reaperServiceProvider, + Provider testClustersRegistryProvider, FileSystemOperations fileSystemOperations, ArchiveOperations archiveOperations, ExecOperations execOperations, @@ -190,6 +193,7 @@ public class ElasticsearchNode implements TestClusterConfiguration { this.name = name; this.project = project; this.reaperServiceProvider = reaperServiceProvider; + this.testClustersRegistryProvider = testClustersRegistryProvider; this.fileSystemOperations = fileSystemOperations; this.archiveOperations = archiveOperations; this.execOperations = execOperations; @@ -891,11 +895,13 @@ private void startElasticsearchProcess() { } } LOGGER.info("Running `{}` in `{}` for {} env: {}", command, workingDir, this, environment); + Process esProcess; try { esProcess = processBuilder.start(); } catch (IOException e) { throw new TestClustersException("Failed to start ES process for " + this, e); } + testClustersRegistryProvider.get().storeProcess(id(), esProcess); reaperServiceProvider.get().registerPid(toString(), esProcess.pid()); } @@ -981,6 +987,7 @@ public synchronized void stop(boolean tailLogs) { } catch (IOException e) { throw new UncheckedIOException(e); } + Process esProcess = testClustersRegistryProvider.get().getProcess(id()); if (esProcess == null && tailLogs) { // This is a special case. If start() throws an exception the plugin will still call stop // Another exception here would eat the orriginal. @@ -1573,6 +1580,7 @@ public List getFeatureFlags() { @Override @Internal public boolean isProcessAlive() { + Process esProcess = testClustersRegistryProvider.get().getProcess(id()); requireNonNull(esProcess, "Can't wait for `" + this + "` as it's not started. Does the task have `useCluster` ?"); return esProcess.isAlive(); } @@ -1601,6 +1609,10 @@ public int hashCode() { @Override public String toString() { + return id() + " (" + System.identityHashCode(this) + ")"; + } + + private String id() { return "node{" + path + ":" + name + "}"; } @@ -1701,7 +1713,7 @@ public CharSequence[] getArgs() { } } - private record FeatureFlag(String feature, Version from, Version until) { + public record FeatureFlag(String feature, Version from, Version until) { @Input public String getFeature() { diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/MockApmServer.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/MockApmServer.java index 7c1d4b6015d2e..b64f372364f22 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/MockApmServer.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/MockApmServer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.testclusters; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java index cb601778a20e9..54bffd2a14b3d 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.testclusters; @@ -41,6 +42,7 @@ public abstract class RunTask extends DefaultTestClustersTask { private Boolean debug = false; private Boolean cliDebug = false; + private Boolean entitlementsEnabled = false; private Boolean apmServerEnabled = false; private Boolean preserveData = false; @@ -68,6 +70,14 @@ public void setCliDebug(boolean enabled) { this.cliDebug = enabled; } + @Option( + option = "entitlements", + description = "Use the Entitlements agent system in place of SecurityManager to enforce sandbox policies." + ) + public void setEntitlementsEnabled(boolean enabled) { + this.entitlementsEnabled = enabled; + } + @Input public Boolean getDebug() { return debug; @@ -78,6 +88,11 @@ public Boolean getCliDebug() { return cliDebug; } + @Input + public Boolean getEntitlementsEnabled() { + return entitlementsEnabled; + } + @Input public Boolean getApmServerEnabled() { return apmServerEnabled; @@ -225,6 +240,9 @@ else if (node.getSettingKeys().contains("telemetry.metrics.enabled") == false) { if (cliDebug) { enableCliDebug(); } + if (entitlementsEnabled) { + enableEntitlements(); + } } @TaskAction diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/SslTrustResolver.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/SslTrustResolver.java index 4c2cbf4defb74..1110ce57aa980 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/SslTrustResolver.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/SslTrustResolver.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.testclusters; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/StandaloneRestIntegTestTask.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/StandaloneRestIntegTestTask.java index 5e6b33aa980f0..6b9295bb1cff2 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/StandaloneRestIntegTestTask.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/StandaloneRestIntegTestTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.testclusters; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java index b6ead59296eb4..49dd22fb35815 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.testclusters; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterInfo.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterInfo.java new file mode 100644 index 0000000000000..07663de7a9df9 --- /dev/null +++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterInfo.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.gradle.testclusters; + +import java.io.File; +import java.util.List; + +public class TestClusterInfo { + private final List allHttpSocketURI; + private final List allTransportPortURI; + private final List auditLogs; + + public TestClusterInfo(List allHttpSocketURI, List allTransportPortURI, List auditLogs) { + this.allHttpSocketURI = allHttpSocketURI; + this.allTransportPortURI = allTransportPortURI; + this.auditLogs = auditLogs; + } + + public List getAllHttpSocketURI() { + return allHttpSocketURI; + } + + public List getAllTransportPortURI() { + return allTransportPortURI; + } + + public List getAuditLogs() { + return auditLogs; + } +} diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterValueSource.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterValueSource.java new file mode 100644 index 0000000000000..8ecadcdc6d2b1 --- /dev/null +++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterValueSource.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.testclusters; + +import org.gradle.api.provider.Property; +import org.gradle.api.provider.ValueSource; +import org.gradle.api.provider.ValueSourceParameters; +import org.jetbrains.annotations.Nullable; + +public abstract class TestClusterValueSource implements ValueSource { + + @Nullable + @Override + public TestClusterInfo obtain() { + String clusterName = getParameters().getClusterName().get(); + String path = getParameters().getPath().get(); + return getParameters().getService().get().getClusterDetails(path, clusterName); + } + + interface Parameters extends ValueSourceParameters { + Property getClusterName(); + + Property getPath(); + + Property getService(); + } +} diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersAware.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersAware.java index 3fef77688c48d..9e5fc1f09ac9e 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersAware.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersAware.java @@ -1,12 +1,14 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.testclusters; +import org.elasticsearch.gradle.ElasticsearchDistribution; import org.gradle.api.Task; import org.gradle.api.provider.Property; import org.gradle.api.provider.Provider; @@ -33,10 +35,15 @@ default void useCluster(ElasticsearchCluster cluster) { if (cluster.getPath().equals(getProject().getPath()) == false) { throw new TestClustersException("Task " + getPath() + " can't use test cluster from" + " another project " + cluster); } - - cluster.getNodes() - .all(node -> node.getDistributions().forEach(distro -> dependsOn(getProject().provider(() -> distro.maybeFreeze())))); - dependsOn(cluster.getPluginAndModuleConfigurations()); + if (cluster.getName().equals(getName())) { + for (ElasticsearchNode node : cluster.getNodes()) { + for (ElasticsearchDistribution distro : node.getDistributions()) { + ElasticsearchDistribution frozenDistro = distro.maybeFreeze(); + dependsOn(frozenDistro); + } + } + dependsOn(cluster.getPluginAndModuleConfigurations()); + } getClusters().add(cluster); } @@ -73,4 +80,12 @@ default void enableCliDebug() { } } } + + default void enableEntitlements() { + for (ElasticsearchCluster cluster : getClusters()) { + for (ElasticsearchNode node : cluster.getNodes()) { + node.cliJvmArgs("-Des.entitlements.enabled=true"); + } + } + } } diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersException.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersException.java index e66edbb497a9d..3e0b1b281f19a 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersException.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.testclusters; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java index d4ae65d43893a..ada31bc11a653 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.testclusters; @@ -25,6 +26,7 @@ import org.gradle.api.invocation.Gradle; import org.gradle.api.logging.Logger; import org.gradle.api.logging.Logging; +import org.gradle.api.provider.Property; import org.gradle.api.provider.Provider; import org.gradle.api.provider.ProviderFactory; import org.gradle.api.services.BuildService; @@ -105,15 +107,22 @@ public void apply(Project project) { runtimeJavaProvider = providerFactory.provider( () -> System.getenv("RUNTIME_JAVA_HOME") == null ? Jvm.current().getJavaHome() : new File(System.getenv("RUNTIME_JAVA_HOME")) ); + + // register cluster registry as a global build service + Provider testClustersRegistryProvider = project.getGradle() + .getSharedServices() + .registerIfAbsent(REGISTRY_SERVICE_NAME, TestClustersRegistry.class, noop()); + // enable the DSL to describe clusters - NamedDomainObjectContainer container = createTestClustersContainerExtension(project, reaperServiceProvider); + NamedDomainObjectContainer container = createTestClustersContainerExtension( + project, + testClustersRegistryProvider, + reaperServiceProvider + ); // provide a task to be able to list defined clusters. createListClustersTask(project, container); - // register cluster registry as a global build service - project.getGradle().getSharedServices().registerIfAbsent(REGISTRY_SERVICE_NAME, TestClustersRegistry.class, noop()); - // register throttle so we only run at most max-workers/2 nodes concurrently Provider testClustersThrottleProvider = project.getGradle() .getSharedServices() @@ -144,6 +153,7 @@ private void configureArtifactTransforms(Project project) { private NamedDomainObjectContainer createTestClustersContainerExtension( Project project, + Provider testClustersRegistryProvider, Provider reaper ) { // Create an extensions that allows describing clusters @@ -154,6 +164,7 @@ private NamedDomainObjectContainer createTestClustersConta name, project, reaper, + testClustersRegistryProvider, getFileSystemOperations(), getArchiveOperations(), getExecOperations(), @@ -198,7 +209,9 @@ public void apply(Project project) { Provider testClusterTasksService = project.getGradle() .getSharedServices() - .registerIfAbsent(TEST_CLUSTER_TASKS_SERVICE, TaskEventsService.class, spec -> {}); + .registerIfAbsent(TEST_CLUSTER_TASKS_SERVICE, TaskEventsService.class, spec -> { + spec.getParameters().getRegistry().set(registryProvider); + }); TestClustersRegistry registry = registryProvider.get(); // When we know what tasks will run, we claim the clusters of those task to differentiate between clusters @@ -208,7 +221,7 @@ public void apply(Project project) { configureClaimClustersHook(project.getGradle(), registry); // Before each task, we determine if a cluster needs to be started for that task. - configureStartClustersHook(project.getGradle(), registry, testClusterTasksService); + configureStartClustersHook(project.getGradle()); // After each task we determine if there are clusters that are no longer needed. getEventsListenerRegistry().onTaskCompletion(testClusterTasksService); @@ -227,12 +240,7 @@ private static void configureClaimClustersHook(Gradle gradle, TestClustersRegist }); } - private void configureStartClustersHook( - Gradle gradle, - TestClustersRegistry registry, - Provider testClusterTasksService - ) { - testClusterTasksService.get().registry(registry); + private void configureStartClustersHook(Gradle gradle) { gradle.getTaskGraph().whenReady(taskExecutionGraph -> { taskExecutionGraph.getAllTasks() .stream() @@ -248,19 +256,14 @@ private void configureStartClustersHook( } } - static public abstract class TaskEventsService implements BuildService, OperationCompletionListener { + static public abstract class TaskEventsService implements BuildService, OperationCompletionListener { Map tasksMap = new HashMap<>(); - private TestClustersRegistry registryProvider; public void register(TestClustersAware task) { tasksMap.put(task.getPath(), task); } - public void registry(TestClustersRegistry registry) { - this.registryProvider = registry; - } - @Override public void onFinish(FinishEvent finishEvent) { if (finishEvent instanceof TaskFinishEvent taskFinishEvent) { @@ -272,11 +275,18 @@ public void onFinish(FinishEvent finishEvent) { if (task.getDidWork()) { task.getClusters() .forEach( - cluster -> registryProvider.stopCluster(cluster, taskFinishEvent.getResult() instanceof TaskFailureResult) + cluster -> getParameters().getRegistry() + .get() + .stopCluster(cluster, taskFinishEvent.getResult() instanceof TaskFailureResult) ); } } } } + + // Some parameters for the web server + interface Params extends BuildServiceParameters { + Property getRegistry(); + } } } diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersRegistry.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersRegistry.java index 1d6efdabcd59f..8d2a9217e7d0c 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersRegistry.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersRegistry.java @@ -1,14 +1,17 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.testclusters; import org.gradle.api.logging.Logger; import org.gradle.api.logging.Logging; +import org.gradle.api.provider.Provider; +import org.gradle.api.provider.ProviderFactory; import org.gradle.api.services.BuildService; import org.gradle.api.services.BuildServiceParameters; @@ -16,20 +19,23 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; + +import javax.inject.Inject; public abstract class TestClustersRegistry implements BuildService { private static final Logger logger = Logging.getLogger(TestClustersRegistry.class); private static final String TESTCLUSTERS_INSPECT_FAILURE = "testclusters.inspect.failure"; private final Boolean allowClusterToSurvive = Boolean.valueOf(System.getProperty(TESTCLUSTERS_INSPECT_FAILURE, "false")); - private final Map claimsInventory = new HashMap<>(); - private final Set runningClusters = new HashSet<>(); + private final Map nodeProcesses = new HashMap<>(); + + @Inject + public abstract ProviderFactory getProviderFactory(); public void claimCluster(ElasticsearchCluster cluster) { - cluster.freeze(); - int claim = claimsInventory.getOrDefault(cluster, 0) + 1; - claimsInventory.put(cluster, claim); - if (claim > 1) { + int claims = cluster.addClaim(); + if (claims > 1) { cluster.setShared(true); } } @@ -42,6 +48,13 @@ public void maybeStartCluster(ElasticsearchCluster cluster) { cluster.start(); } + public Provider getClusterInfo(String clusterName) { + return getProviderFactory().of(TestClusterValueSource.class, spec -> { + spec.getParameters().getService().set(TestClustersRegistry.this); + spec.getParameters().getClusterName().set(clusterName); + }); + } + public void stopCluster(ElasticsearchCluster cluster, boolean taskFailed) { if (taskFailed) { // If the task fails, and other tasks use this cluster, the other task will likely never be @@ -66,8 +79,7 @@ public void stopCluster(ElasticsearchCluster cluster, boolean taskFailed) { runningClusters.remove(cluster); } } else { - int currentClaims = claimsInventory.getOrDefault(cluster, 0) - 1; - claimsInventory.put(cluster, currentClaims); + int currentClaims = cluster.removeClaim(); if (currentClaims <= 0 && runningClusters.contains(cluster)) { cluster.stop(false); runningClusters.remove(cluster); @@ -75,4 +87,33 @@ public void stopCluster(ElasticsearchCluster cluster, boolean taskFailed) { } } + public TestClusterInfo getClusterDetails(String path, String clusterName) { + ElasticsearchCluster cluster = runningClusters.stream() + .filter(c -> c.getPath().equals(path)) + .filter(c -> c.getName().equals(clusterName)) + .findFirst() + .orElseThrow(); + return new TestClusterInfo( + cluster.getAllHttpSocketURI(), + cluster.getAllTransportPortURI(), + cluster.getNodes().stream().map(n -> n.getAuditLog()).collect(Collectors.toList()) + ); + } + + public void restart(String path, String clusterName) { + ElasticsearchCluster cluster = runningClusters.stream() + .filter(c -> c.getPath().equals(path)) + .filter(c -> c.getName().equals(clusterName)) + .findFirst() + .orElseThrow(); + cluster.restart(); + } + + public void storeProcess(String id, Process esProcess) { + nodeProcesses.put(id, esProcess); + } + + public Process getProcess(String id) { + return nodeProcesses.get(id); + } } diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersThrottle.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersThrottle.java index 4dfd950e0aaf1..b5c6937b4659c 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersThrottle.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersThrottle.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.testclusters; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestDistribution.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestDistribution.java index 6a1b35150892b..269332a59d86e 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestDistribution.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestDistribution.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.testclusters; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/WaitForHttpResource.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/WaitForHttpResource.java index 550dd0fdcf8fb..8eb385a0deb11 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/WaitForHttpResource.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/WaitForHttpResource.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.testclusters; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/transform/SymbolicLinkPreservingUntarTransform.java b/build-tools/src/main/java/org/elasticsearch/gradle/transform/SymbolicLinkPreservingUntarTransform.java index ba93f36d6dbee..fd386274eddc8 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/transform/SymbolicLinkPreservingUntarTransform.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/transform/SymbolicLinkPreservingUntarTransform.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.transform; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/transform/UnpackTransform.java b/build-tools/src/main/java/org/elasticsearch/gradle/transform/UnpackTransform.java index a7f3f5968871a..9d80359cafd88 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/transform/UnpackTransform.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/transform/UnpackTransform.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.transform; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/transform/UnzipTransform.java b/build-tools/src/main/java/org/elasticsearch/gradle/transform/UnzipTransform.java index add869bb8f4ef..b52ecaa3cfae6 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/transform/UnzipTransform.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/transform/UnzipTransform.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.transform; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/util/FileUtils.java b/build-tools/src/main/java/org/elasticsearch/gradle/util/FileUtils.java index 534d28c02c08d..c4183e430ad59 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/util/FileUtils.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/util/FileUtils.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.util; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/util/GradleUtils.java b/build-tools/src/main/java/org/elasticsearch/gradle/util/GradleUtils.java index cab4d55210864..8068a8fd9801f 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/util/GradleUtils.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/util/GradleUtils.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.util; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/util/Pair.java b/build-tools/src/main/java/org/elasticsearch/gradle/util/Pair.java index 1ffe226910863..5ea0edcd40842 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/util/Pair.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/util/Pair.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.util; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/util/PermissionUtils.java b/build-tools/src/main/java/org/elasticsearch/gradle/util/PermissionUtils.java index d35c0f2957fa6..459a57c0b08ab 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/util/PermissionUtils.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/util/PermissionUtils.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.util; diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/util/PlatformUtils.java b/build-tools/src/main/java/org/elasticsearch/gradle/util/PlatformUtils.java new file mode 100644 index 0000000000000..2f093a19032c8 --- /dev/null +++ b/build-tools/src/main/java/org/elasticsearch/gradle/util/PlatformUtils.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.util; + +import java.util.stream.Collectors; + +public class PlatformUtils { + + public static String normalize(String input) { + return input.lines() + .map(it -> it.replace('\\', '/')) + .map(it -> it.replaceAll("\\d+\\.\\d\\ds", "0.00s")) + .map(it -> it.replace("file:/./", "file:./")) + .collect(Collectors.joining("\n")); + } +} diff --git a/build-tools/src/test/groovy/org/elasticsearch/gradle/DistributionDownloadPluginSpec.groovy b/build-tools/src/test/groovy/org/elasticsearch/gradle/DistributionDownloadPluginSpec.groovy index 6ec3db1d908e3..aeb0bcf148446 100644 --- a/build-tools/src/test/groovy/org/elasticsearch/gradle/DistributionDownloadPluginSpec.groovy +++ b/build-tools/src/test/groovy/org/elasticsearch/gradle/DistributionDownloadPluginSpec.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle diff --git a/build-tools/src/test/groovy/org/elasticsearch/gradle/VersionSpec.groovy b/build-tools/src/test/groovy/org/elasticsearch/gradle/VersionSpec.groovy index 281caeafd7251..96a7c63d665ed 100644 --- a/build-tools/src/test/groovy/org/elasticsearch/gradle/VersionSpec.groovy +++ b/build-tools/src/test/groovy/org/elasticsearch/gradle/VersionSpec.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle diff --git a/build-tools/src/test/groovy/org/elasticsearch/gradle/testclusters/SslTrustResolverSpec.groovy b/build-tools/src/test/groovy/org/elasticsearch/gradle/testclusters/SslTrustResolverSpec.groovy index f1a87cd9ff266..9948d99d9f430 100644 --- a/build-tools/src/test/groovy/org/elasticsearch/gradle/testclusters/SslTrustResolverSpec.groovy +++ b/build-tools/src/test/groovy/org/elasticsearch/gradle/testclusters/SslTrustResolverSpec.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.testclusters diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy index 5ea970c533474..fe23204d5601c 100644 --- a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy +++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.fixtures @@ -55,7 +56,7 @@ abstract class AbstractGradleFuncTest extends Specification { propertiesFile << "org.gradle.java.installations.fromEnv=JAVA_HOME,RUNTIME_JAVA_HOME,JAVA15_HOME,JAVA14_HOME,JAVA13_HOME,JAVA12_HOME,JAVA11_HOME,JAVA8_HOME" - def nativeLibsProject = subProject(":libs:elasticsearch-native:elasticsearch-native-libraries") + def nativeLibsProject = subProject(":libs:native:native-libraries") nativeLibsProject << """ plugins { id 'base' @@ -155,33 +156,34 @@ abstract class AbstractGradleFuncTest extends Specification { File internalBuild( List extraPlugins = [], - String bugfix = "7.15.2", - String bugfixLucene = "8.9.0", - String staged = "7.16.0", - String stagedLucene = "8.10.0", - String minor = "8.0.0", - String minorLucene = "9.0.0" + String maintenance = "7.16.10", + String bugfix2 = "8.1.3", + String bugfix = "8.2.1", + String staged = "8.3.0", + String minor = "8.4.0", + String current = "9.0.0" ) { buildFile << """plugins { id 'elasticsearch.global-build-info' ${extraPlugins.collect { p -> "id '$p'" }.join('\n')} } import org.elasticsearch.gradle.Architecture - import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.internal.BwcVersions import org.elasticsearch.gradle.Version - Version currentVersion = Version.fromString("8.1.0") + Version currentVersion = Version.fromString("${current}") def versionList = [ + Version.fromString("$maintenance"), + Version.fromString("$bugfix2"), Version.fromString("$bugfix"), Version.fromString("$staged"), Version.fromString("$minor"), currentVersion ] - BwcVersions versions = new BwcVersions(currentVersion, versionList) - BuildParams.init { it.setBwcVersions(provider(() -> versions)) } + BwcVersions versions = new BwcVersions(currentVersion, versionList, ['main', '8.x', '8.3', '8.2', '8.1', '7.16']) + buildParams.setBwcVersions(project.provider { versions} ) """ } diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractJavaGradleFuncTest.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractJavaGradleFuncTest.groovy index bf9c26da8ba13..17f181d4aaf80 100644 --- a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractJavaGradleFuncTest.groovy +++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractJavaGradleFuncTest.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.fixtures diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/DistributionDownloadFixture.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/DistributionDownloadFixture.groovy index 172b94bb8b945..b23bad1697e0d 100644 --- a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/DistributionDownloadFixture.groovy +++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/DistributionDownloadFixture.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.fixtures diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/RestrictedBuildServiceAwareGradleRunner.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/RestrictedBuildServiceAwareGradleRunner.groovy index d8739b9b380cf..01a304031e156 100644 --- a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/RestrictedBuildServiceAwareGradleRunner.groovy +++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/RestrictedBuildServiceAwareGradleRunner.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.fixtures diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/TestClasspathUtils.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/TestClasspathUtils.groovy index e41ace83b75bc..7f0ca4ac0b29e 100644 --- a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/TestClasspathUtils.groovy +++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/TestClasspathUtils.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.fixtures diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/WiremockFixture.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/WiremockFixture.groovy index a3281780e16e0..d0d968d2735f4 100644 --- a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/WiremockFixture.groovy +++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/WiremockFixture.groovy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.fixtures diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/BuildConfigurationAwareGradleRunner.java b/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/BuildConfigurationAwareGradleRunner.java index 1a2a7a564b4f1..631bc12a6a38a 100644 --- a/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/BuildConfigurationAwareGradleRunner.java +++ b/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/BuildConfigurationAwareGradleRunner.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test; diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/InMemoryJavaCompiler.java b/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/InMemoryJavaCompiler.java index 7e13013c8ddc4..b463f9605f01e 100644 --- a/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/InMemoryJavaCompiler.java +++ b/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/InMemoryJavaCompiler.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test; diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/InternalAwareGradleRunner.java b/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/InternalAwareGradleRunner.java index 38c4a8105b112..5a5eeb4f66f8b 100644 --- a/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/InternalAwareGradleRunner.java +++ b/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/InternalAwareGradleRunner.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test; diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/JarUtils.java b/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/JarUtils.java index 6c6f01a8cce7b..4a78e80a3ab4b 100644 --- a/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/JarUtils.java +++ b/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/JarUtils.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test; diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/NormalizeOutputGradleRunner.java b/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/NormalizeOutputGradleRunner.java index 7c496d1f9597d..d241420192dff 100644 --- a/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/NormalizeOutputGradleRunner.java +++ b/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/NormalizeOutputGradleRunner.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test; diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/StableApiJarMocks.java b/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/StableApiJarMocks.java index a625fef619039..b69de626c2b29 100644 --- a/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/StableApiJarMocks.java +++ b/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/StableApiJarMocks.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test; diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/TestResultExtension.java b/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/TestResultExtension.java index c08f25843c721..5ecb8c1ce660a 100644 --- a/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/TestResultExtension.java +++ b/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/TestResultExtension.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test; diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/TestUtils.java b/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/TestUtils.java index e4513bfe4c0f0..0eae8f5f0f9a6 100644 --- a/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/TestUtils.java +++ b/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/TestUtils.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gradle.internal.test; diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/Extensible.java b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/Extensible.java index 34984b90d3c73..3e0099df45459 100644 --- a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/Extensible.java +++ b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/Extensible.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin; diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/NamedComponent.java b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/NamedComponent.java index 53b195313ecc1..b765348bf05f4 100644 --- a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/NamedComponent.java +++ b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/NamedComponent.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin; diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/ExtensibleClass.java b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/ExtensibleClass.java index 50ea17993d7c7..b6e38ec13939c 100644 --- a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/ExtensibleClass.java +++ b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/ExtensibleClass.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.scanner.test_classes; diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/ExtensibleInterface.java b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/ExtensibleInterface.java index 784136bf6f5a5..7553c3e2e9acb 100644 --- a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/ExtensibleInterface.java +++ b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/ExtensibleInterface.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.scanner.test_classes; diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/ImplementingExtensible.java b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/ImplementingExtensible.java index 9b0cbbb8d6277..0462f8d636750 100644 --- a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/ImplementingExtensible.java +++ b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/ImplementingExtensible.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.scanner.test_classes; diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/SubClass.java b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/SubClass.java index 3dbcd4bac3218..8c6ee765e37f1 100644 --- a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/SubClass.java +++ b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/SubClass.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.scanner.test_classes; diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/TestNamedComponent.java b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/TestNamedComponent.java index 9a6b9b648e3a2..4d584079848e6 100644 --- a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/TestNamedComponent.java +++ b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/TestNamedComponent.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.scanner.test_classes; diff --git a/build-tools/src/testFixtures/resources/META-INF/services/org.spockframework.runtime.extension.IGlobalExtension b/build-tools/src/testFixtures/resources/META-INF/services/org.spockframework.runtime.extension.IGlobalExtension index 7d1e63d129ca1..816b14ab9bc34 100644 --- a/build-tools/src/testFixtures/resources/META-INF/services/org.spockframework.runtime.extension.IGlobalExtension +++ b/build-tools/src/testFixtures/resources/META-INF/services/org.spockframework.runtime.extension.IGlobalExtension @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # -org.elasticsearch.gradle.internal.test.TestResultExtension \ No newline at end of file +org.elasticsearch.gradle.internal.test.TestResultExtension diff --git a/build.gradle b/build.gradle index 01fdace570ce0..3c0b456a14d4b 100644 --- a/build.gradle +++ b/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ @@ -15,7 +16,6 @@ import com.fasterxml.jackson.databind.ObjectMapper import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.internal.BaseInternalPluginBuildPlugin import org.elasticsearch.gradle.internal.ResolveAllDependencies -import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.util.GradleUtils import org.gradle.plugins.ide.eclipse.model.AccessRule import org.gradle.plugins.ide.eclipse.model.ProjectDependency @@ -28,10 +28,6 @@ import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure buildscript { repositories { - maven { - url 'https://jitpack.io' - } - mavenCentral() } } @@ -135,21 +131,21 @@ tasks.register("updateCIBwcVersions") { } doLast { - writeVersions(file(".ci/bwcVersions"), filterIntermediatePatches(BuildParams.bwcVersions.allIndexCompatible)) - writeVersions(file(".ci/snapshotBwcVersions"), filterIntermediatePatches(BuildParams.bwcVersions.unreleasedIndexCompatible)) + writeVersions(file(".ci/bwcVersions"), filterIntermediatePatches(buildParams.bwcVersions.allIndexCompatible)) + writeVersions(file(".ci/snapshotBwcVersions"), filterIntermediatePatches(buildParams.bwcVersions.unreleasedIndexCompatible)) expandBwcList( ".buildkite/pipelines/intake.yml", ".buildkite/pipelines/intake.template.yml", - filterIntermediatePatches(BuildParams.bwcVersions.unreleasedIndexCompatible) + filterIntermediatePatches(buildParams.bwcVersions.unreleasedIndexCompatible) ) writeBuildkitePipeline( ".buildkite/pipelines/periodic.yml", ".buildkite/pipelines/periodic.template.yml", [ - new ListExpansion(versions: filterIntermediatePatches(BuildParams.bwcVersions.unreleasedIndexCompatible), variable: "BWC_LIST"), + new ListExpansion(versions: filterIntermediatePatches(buildParams.bwcVersions.unreleasedIndexCompatible), variable: "BWC_LIST"), ], [ - new StepExpansion(templatePath: ".buildkite/pipelines/periodic.bwc.template.yml", versions: filterIntermediatePatches(BuildParams.bwcVersions.allIndexCompatible), variable: "BWC_STEPS"), + new StepExpansion(templatePath: ".buildkite/pipelines/periodic.bwc.template.yml", versions: filterIntermediatePatches(buildParams.bwcVersions.allIndexCompatible), variable: "BWC_STEPS"), ] ) @@ -157,7 +153,7 @@ tasks.register("updateCIBwcVersions") { ".buildkite/pipelines/periodic-packaging.yml", ".buildkite/pipelines/periodic-packaging.template.yml", ".buildkite/pipelines/periodic-packaging.bwc.template.yml", - filterIntermediatePatches(BuildParams.bwcVersions.allIndexCompatible) + filterIntermediatePatches(buildParams.bwcVersions.allIndexCompatible) ) } } @@ -179,19 +175,19 @@ tasks.register("verifyVersions") { // Fetch the metadata and parse the xml into Version instances because it's more straight forward here // rather than bwcVersion ( VersionCollection ). new URL('https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s -> - BuildParams.bwcVersions.compareToAuthoritative( + buildParams.bwcVersions.compareToAuthoritative( new XmlParser().parse(s) .versioning.versions.version .collect { it.text() }.findAll { it ==~ /\d+\.\d+\.\d+/ } .collect { Version.fromString(it) } ) } - verifyCiYaml(file(".ci/bwcVersions"), filterIntermediatePatches(BuildParams.bwcVersions.allIndexCompatible)) - verifyCiYaml(file(".ci/snapshotBwcVersions"), BuildParams.bwcVersions.unreleasedIndexCompatible) + verifyCiYaml(file(".ci/bwcVersions"), filterIntermediatePatches(buildParams.bwcVersions.allIndexCompatible)) + verifyCiYaml(file(".ci/snapshotBwcVersions"), buildParams.bwcVersions.unreleasedIndexCompatible) // Make sure backport bot config file is up to date JsonNode backportConfig = new ObjectMapper().readTree(file(".backportrc.json")) - BuildParams.bwcVersions.forPreviousUnreleased { unreleasedVersion -> + buildParams.bwcVersions.forPreviousUnreleased { unreleasedVersion -> boolean valid = backportConfig.get("targetBranchChoices").elements().any { branchChoice -> if (branchChoice.isObject()) { return branchChoice.get("name").textValue() == unreleasedVersion.branch @@ -203,11 +199,11 @@ tasks.register("verifyVersions") { throw new GradleException("No branch choice exists for development branch ${unreleasedVersion.branch} in .backportrc.json.") } } - String versionMapping = backportConfig.get("branchLabelMapping").fields().find { it.value.textValue() == 'main' }.key + String versionMapping = backportConfig.get("branchLabelMapping").fields().find { it.value.textValue() == '8.x' }.key String expectedMapping = "^v${versions.elasticsearch.replaceAll('-SNAPSHOT', '')}\$" if (versionMapping != expectedMapping) { throw new GradleException( - "Backport label mapping for branch 'main' is '${versionMapping}' but should be " + + "Backport label mapping for branch '8.x' is '${versionMapping}' but should be " + "'${expectedMapping}'. Update .backportrc.json." ) } @@ -294,7 +290,10 @@ allprojects { if (project.path.contains(":distribution:docker")) { enabled = false } - + if (project.path.contains(":libs:cli")) { + // ensure we resolve p2 dependencies for the spotless eclipse formatter + dependsOn "spotlessJavaCheck" + } } plugins.withId('lifecycle-base') { @@ -368,24 +367,24 @@ tasks.register("verifyBwcTestsEnabled") { } tasks.register("branchConsistency") { - description 'Ensures this branch is internally consistent. For example, that versions constants match released versions.' - group 'Verification' + description = 'Ensures this branch is internally consistent. For example, that versions constants match released versions.' + group = 'Verification' dependsOn ":verifyVersions", ":verifyBwcTestsEnabled" } tasks.named("wrapper").configure { distributionType = 'ALL' + def minimumGradleVersionFile = project.file('build-tools-internal/src/main/resources/minimumGradleVersion') doLast { // copy wrapper properties file to build-tools-internal to allow seamless idea integration def file = new File("build-tools-internal/gradle/wrapper/gradle-wrapper.properties") - Files.copy(wrapper.getPropertiesFile().toPath(), file.toPath(), REPLACE_EXISTING) + Files.copy(getPropertiesFile().toPath(), file.toPath(), REPLACE_EXISTING) // copy wrapper properties file to plugins/examples to allow seamless idea integration def examplePluginsWrapperProperties = new File("plugins/examples/gradle/wrapper/gradle-wrapper.properties") - Files.copy(wrapper.getPropertiesFile().toPath(), examplePluginsWrapperProperties.toPath(), REPLACE_EXISTING) - + Files.copy(getPropertiesFile().toPath(), examplePluginsWrapperProperties.toPath(), REPLACE_EXISTING) // Update build-tools to reflect the Gradle upgrade // TODO: we can remove this once we have tests to make sure older versions work. - project.file('build-tools-internal/src/main/resources/minimumGradleVersion').text = gradleVersion + minimumGradleVersionFile.text = gradleVersion println "Updated minimum Gradle Version" } } @@ -408,6 +407,13 @@ gradle.projectsEvaluated { } } +tasks.named("validateChangelogs").configure { + def triggeredTaskNames = gradle.startParameter.taskNames + onlyIf { + triggeredTaskNames.any { it.startsWith("checkPart") || it == 'functionalTests' } == false + } +} + tasks.named("precommit") { dependsOn gradle.includedBuild('build-tools').task(':precommit') dependsOn gradle.includedBuild('build-tools-internal').task(':precommit') diff --git a/client/benchmark/build.gradle b/client/benchmark/build.gradle index 6773ca5532ee4..d87fad0dfba02 100644 --- a/client/benchmark/build.gradle +++ b/client/benchmark/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.java' diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/AbstractBenchmark.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/AbstractBenchmark.java index 6218b4d9a0d36..62c02da397592 100644 --- a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/AbstractBenchmark.java +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/AbstractBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.client.benchmark; diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/BenchmarkMain.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/BenchmarkMain.java index e2afd828fd7af..526671b929953 100644 --- a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/BenchmarkMain.java +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/BenchmarkMain.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.client.benchmark; diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/BenchmarkRunner.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/BenchmarkRunner.java index a442f20f3f588..5001622504073 100644 --- a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/BenchmarkRunner.java +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/BenchmarkRunner.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.client.benchmark; diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/BenchmarkTask.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/BenchmarkTask.java index a50442c543d87..1ac0807426807 100644 --- a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/BenchmarkTask.java +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/BenchmarkTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.client.benchmark; diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/Metrics.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/Metrics.java index deff0c7d23ef9..345c4173b84ce 100644 --- a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/Metrics.java +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/Metrics.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.client.benchmark.metrics; diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/MetricsCalculator.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/MetricsCalculator.java index 2f9a6e65655de..358a9f8fc49c0 100644 --- a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/MetricsCalculator.java +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/MetricsCalculator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.client.benchmark.metrics; diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/Sample.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/Sample.java index ae8da1444d3c2..89e9232c5c445 100644 --- a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/Sample.java +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/Sample.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.client.benchmark.metrics; diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/SampleRecorder.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/SampleRecorder.java index b7b684661b1ed..6d01fe9a7fcc8 100644 --- a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/SampleRecorder.java +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/metrics/SampleRecorder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.client.benchmark.metrics; diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/bulk/BulkBenchmarkTask.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/bulk/BulkBenchmarkTask.java index 69edd9c8f86ff..d6cb9d86d8afb 100644 --- a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/bulk/BulkBenchmarkTask.java +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/bulk/BulkBenchmarkTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.client.benchmark.ops.bulk; diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/bulk/BulkRequestExecutor.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/bulk/BulkRequestExecutor.java index 1eb062c294271..8095749962da8 100644 --- a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/bulk/BulkRequestExecutor.java +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/bulk/BulkRequestExecutor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.client.benchmark.ops.bulk; diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/search/SearchBenchmarkTask.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/search/SearchBenchmarkTask.java index a00d78bcf77f4..16a912338d384 100644 --- a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/search/SearchBenchmarkTask.java +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/search/SearchBenchmarkTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.client.benchmark.ops.search; diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/search/SearchRequestExecutor.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/search/SearchRequestExecutor.java index f3659c8ac829c..731c72ca6d0d3 100644 --- a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/search/SearchRequestExecutor.java +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/search/SearchRequestExecutor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.client.benchmark.ops.search; diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/rest/RestClientBenchmark.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/rest/RestClientBenchmark.java index 0793b8c721c7b..814a251810e75 100644 --- a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/rest/RestClientBenchmark.java +++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/rest/RestClientBenchmark.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.client.benchmark.rest; diff --git a/client/client-benchmark-noop-api-plugin/build.gradle b/client/client-benchmark-noop-api-plugin/build.gradle index 839cd9052b4f5..9a93cb38b2881 100644 --- a/client/client-benchmark-noop-api-plugin/build.gradle +++ b/client/client-benchmark-noop-api-plugin/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ group = 'org.elasticsearch.plugin' @@ -11,13 +12,13 @@ group = 'org.elasticsearch.plugin' apply plugin: 'elasticsearch.internal-es-plugin' esplugin { - name 'client-benchmark-noop-api' - description 'Stubbed out Elasticsearch actions that can be used for client-side benchmarking' - classname 'org.elasticsearch.plugin.noop.NoopPlugin' + name = 'client-benchmark-noop-api' + description = 'Stubbed out Elasticsearch actions that can be used for client-side benchmarking' + classname ='org.elasticsearch.plugin.noop.NoopPlugin' } // Not published so no need to assemble tasks.named("assemble").configure { enabled = false } tasks.named("dependencyLicenses").configure { enabled = false } tasks.named("dependenciesInfo").configure { enabled = false } -tasks.named("uploadSnykDependencyGraph").configure { enabled = false } \ No newline at end of file +tasks.named("uploadSnykDependencyGraph").configure { enabled = false } diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/NoopPlugin.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/NoopPlugin.java index c25b422a980a3..383b0dd420590 100644 --- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/NoopPlugin.java +++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/NoopPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.noop; diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/RestNoopBulkAction.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/RestNoopBulkAction.java index 6ad1bac8d6e32..12f039a5a122f 100644 --- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/RestNoopBulkAction.java +++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/RestNoopBulkAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.noop.action.bulk; diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/TransportNoopBulkAction.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/TransportNoopBulkAction.java index 01ec6c118bf24..3d3d0640a2b2f 100644 --- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/TransportNoopBulkAction.java +++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/TransportNoopBulkAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.noop.action.bulk; diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/RestNoopSearchAction.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/RestNoopSearchAction.java index 0e91a063596e3..bdac4a8a7e837 100644 --- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/RestNoopSearchAction.java +++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/RestNoopSearchAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.noop.action.search; diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/TransportNoopSearchAction.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/TransportNoopSearchAction.java index 871cdb860a9a9..66a0e785af9ea 100644 --- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/TransportNoopSearchAction.java +++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/TransportNoopSearchAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.noop.action.search; diff --git a/client/rest/build.gradle b/client/rest/build.gradle index 6006fae1c2d84..3fb2aa6595869 100644 --- a/client/rest/build.gradle +++ b/client/rest/build.gradle @@ -36,7 +36,7 @@ base { } // LLRC is licenses under Apache 2.0 -projectLicenses.set(['The Apache Software License, Version 2.0': 'http://www.apache.org/licenses/LICENSE-2.0']) +projectLicenses.set(['The Apache Software License, Version 2.0': providers.provider(() -> 'http://www.apache.org/licenses/LICENSE-2.0')]) licenseFile.set(rootProject.file('licenses/APACHE-LICENSE-2.0.txt')) dependencies { @@ -79,7 +79,7 @@ tasks.named('forbiddenApisTest').configure { } // JarHell is part of es server, which we don't want to pull in -// TODO: Not anymore. Now in :libs:elasticsearch-core +// TODO: Not anymore. Now in :libs:core tasks.named("jarHell").configure { enabled = false } diff --git a/client/sniffer/build.gradle b/client/sniffer/build.gradle index 901917c7b25f8..9b1cb1140311b 100644 --- a/client/sniffer/build.gradle +++ b/client/sniffer/build.gradle @@ -32,7 +32,7 @@ base { } // rest client sniffer is licenses under Apache 2.0 -projectLicenses.set(['The Apache Software License, Version 2.0': 'http://www.apache.org/licenses/LICENSE-2.0']) +projectLicenses.set(['The Apache Software License, Version 2.0': providers.provider(() -> 'http://www.apache.org/licenses/LICENSE-2.0')]) licenseFile.set(rootProject.file('licenses/APACHE-LICENSE-2.0.txt')) dependencies { @@ -73,7 +73,7 @@ tasks.named("dependencyLicenses").configure { } // JarHell is part of es server, which we don't want to pull in -// TODO: Not anymore. Now in :libs:elasticsearch-core +// TODO: Not anymore. Now in :libs:core tasks.named("jarHell").configure { enabled = false } tasks.named("testTestingConventions").configure { diff --git a/client/test/build.gradle b/client/test/build.gradle index 8d457948b91b4..e39b7587b69d5 100644 --- a/client/test/build.gradle +++ b/client/test/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.internal.conventions.precommit.LicenseHeadersTask @@ -17,7 +18,7 @@ java { group = "${group}.client.test" // rest client sniffer is licenses under Apache 2.0 -projectLicenses.set(['The Apache Software License, Version 2.0': 'http://www.apache.org/licenses/LICENSE-2.0']) +projectLicenses.set(['The Apache Software License, Version 2.0': providers.provider(() -> 'http://www.apache.org/licenses/LICENSE-2.0')]) licenseFile.set(rootProject.file('licenses/APACHE-LICENSE-2.0.txt')) dependencies { @@ -53,7 +54,7 @@ tasks.named('forbiddenApisTest').configure { tasks.named("thirdPartyAudit").configure { enabled = false } // JarHell is part of es server, which we don't want to pull in -// TODO: Not anymore. Now in :libs:elasticsearch-core +// TODO: Not anymore. Now in :libs:core tasks.named("jarHell").configure { enabled = false } // TODO: should we have licenses for our test deps? diff --git a/dev-tools/prepare_release_update_documentation.py b/dev-tools/prepare_release_update_documentation.py index 0ddda9d41ebcf..3135889aaf410 100644 --- a/dev-tools/prepare_release_update_documentation.py +++ b/dev-tools/prepare_release_update_documentation.py @@ -1,8 +1,9 @@ # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # Prepare a release: Update the documentation and commit # diff --git a/dev-tools/publish_zstd_binaries.sh b/dev-tools/publish_zstd_binaries.sh index ad41353856c87..e1a5b5732adbf 100755 --- a/dev-tools/publish_zstd_binaries.sh +++ b/dev-tools/publish_zstd_binaries.sh @@ -1,10 +1,11 @@ #!/usr/bin/env bash # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # set -e diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle index 815ac5d4c2dd8..7bbfb0f313e55 100644 --- a/distribution/archives/build.gradle +++ b/distribution/archives/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import java.nio.file.Files @@ -63,7 +64,7 @@ CopySpec archiveFiles(String distributionType, String os, String architecture, b include 'README.asciidoc' } from(rootProject.file('licenses')) { - include isTestDistro ? 'SSPL-1.0+ELASTIC-LICENSE-2.0.txt' : 'ELASTIC-LICENSE-2.0.txt' + include isTestDistro ? 'AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt' : 'ELASTIC-LICENSE-2.0.txt' filePermissions { unix(0644) } diff --git a/distribution/archives/integ-test-zip/build.gradle b/distribution/archives/integ-test-zip/build.gradle index 639136b8df15e..6fbb73cf64aad 100644 --- a/distribution/archives/integ-test-zip/build.gradle +++ b/distribution/archives/integ-test-zip/build.gradle @@ -1,14 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.apache.tools.ant.filters.ReplaceTokens -apply plugin: 'elasticsearch.legacy-java-rest-test' +apply plugin: 'elasticsearch.internal-java-rest-test' // The integ-test-distribution is published to maven apply plugin: 'elasticsearch.publish' @@ -34,19 +35,3 @@ publishing { } } } - -tasks.named("javaRestTest").configure { - dependsOn "assemble" - /* - * There are two unique things going on here: - * 1. These tests can be run against an external cluster with - * -Dtests.rest.cluster=whatever and -Dtest.cluster=whatever - * 2. *One* of these tests is incompatible with that and should be skipped - * when running against an external cluster. - */ - if (project.providers.systemProperty("tests.rest.cluster").isPresent()) { - nonInputProperties.systemProperty 'tests.logfile', testClusters.named('javaRestTest').map(c -> c.singleNode().serverLog) - } else { - systemProperty 'tests.logfile', '--external--' - } -} diff --git a/distribution/archives/integ-test-zip/src/javaRestTest/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java b/distribution/archives/integ-test-zip/src/javaRestTest/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java index c1a93c983a68f..b28f7c020be53 100644 --- a/distribution/archives/integ-test-zip/src/javaRestTest/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java +++ b/distribution/archives/integ-test-zip/src/javaRestTest/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java @@ -1,15 +1,18 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.test.rest; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.junit.ClassRule; import java.io.IOException; @@ -22,6 +25,14 @@ */ public class CreatedLocationHeaderIT extends ESRestTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + public void testCreate() throws IOException { locationTestCase("PUT", "test/_doc/1"); } diff --git a/distribution/archives/integ-test-zip/src/javaRestTest/java/org/elasticsearch/test/rest/JsonLogsFormatAndParseIT.java b/distribution/archives/integ-test-zip/src/javaRestTest/java/org/elasticsearch/test/rest/JsonLogsFormatAndParseIT.java index e4efa6d909335..aef7a876ffadf 100644 --- a/distribution/archives/integ-test-zip/src/javaRestTest/java/org/elasticsearch/test/rest/JsonLogsFormatAndParseIT.java +++ b/distribution/archives/integ-test-zip/src/javaRestTest/java/org/elasticsearch/test/rest/JsonLogsFormatAndParseIT.java @@ -1,44 +1,45 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.test.rest; import org.elasticsearch.common.logging.JsonLogsIntegTestCase; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.LogType; import org.hamcrest.Matcher; +import org.junit.ClassRule; -import java.io.BufferedReader; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.security.AccessController; -import java.security.PrivilegedAction; +import java.io.InputStream; import static org.hamcrest.Matchers.is; public class JsonLogsFormatAndParseIT extends JsonLogsIntegTestCase { + + private static final String NODE_NAME = "test-node-0"; + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .withNode(localNodeSpecBuilder -> localNodeSpecBuilder.name(NODE_NAME)) + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + @Override protected Matcher nodeNameMatcher() { - return is("integTest-0"); + return is(NODE_NAME); } @Override - protected BufferedReader openReader(Path logFile) { - assumeFalse( - "Skipping test because it is being run against an external cluster.", - logFile.getFileName().toString().equals("--external--") - ); - return AccessController.doPrivileged((PrivilegedAction) () -> { - try { - return Files.newBufferedReader(logFile, StandardCharsets.UTF_8); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); + protected InputStream openLogsStream() { + return cluster.getNodeLog(0, LogType.SERVER_JSON); } } diff --git a/distribution/archives/integ-test-zip/src/javaRestTest/java/org/elasticsearch/test/rest/NodeRestUsageIT.java b/distribution/archives/integ-test-zip/src/javaRestTest/java/org/elasticsearch/test/rest/NodeRestUsageIT.java index 5041b82da9c55..17bdf2a748bd7 100644 --- a/distribution/archives/integ-test-zip/src/javaRestTest/java/org/elasticsearch/test/rest/NodeRestUsageIT.java +++ b/distribution/archives/integ-test-zip/src/javaRestTest/java/org/elasticsearch/test/rest/NodeRestUsageIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.test.rest; @@ -14,6 +15,8 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.junit.ClassRule; import java.io.IOException; import java.util.Collections; @@ -27,6 +30,14 @@ public class NodeRestUsageIT extends ESRestTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + @SuppressWarnings("unchecked") public void testWithRestUsage() throws IOException { // First get the current usage figures diff --git a/distribution/archives/integ-test-zip/src/javaRestTest/java/org/elasticsearch/test/rest/RequestsWithoutContentIT.java b/distribution/archives/integ-test-zip/src/javaRestTest/java/org/elasticsearch/test/rest/RequestsWithoutContentIT.java index ba305333f2fb9..abb92ebad9d52 100644 --- a/distribution/archives/integ-test-zip/src/javaRestTest/java/org/elasticsearch/test/rest/RequestsWithoutContentIT.java +++ b/distribution/archives/integ-test-zip/src/javaRestTest/java/org/elasticsearch/test/rest/RequestsWithoutContentIT.java @@ -1,15 +1,18 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.test.rest; import org.elasticsearch.client.Request; import org.elasticsearch.client.ResponseException; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.junit.ClassRule; import java.io.IOException; @@ -17,6 +20,14 @@ public class RequestsWithoutContentIT extends ESRestTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + public void testIndexMissingBody() throws IOException { ResponseException responseException = expectThrows( ResponseException.class, @@ -26,10 +37,9 @@ public void testIndexMissingBody() throws IOException { } public void testBulkMissingBody() throws IOException { - ResponseException responseException = expectThrows( - ResponseException.class, - () -> client().performRequest(new Request(randomBoolean() ? "POST" : "PUT", "/_bulk")) - ); + Request request = new Request(randomBoolean() ? "POST" : "PUT", "/_bulk"); + request.setJsonEntity(""); + ResponseException responseException = expectThrows(ResponseException.class, () -> client().performRequest(request)); assertResponseException(responseException, "request body is required"); } diff --git a/distribution/archives/integ-test-zip/src/javaRestTest/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseIT.java b/distribution/archives/integ-test-zip/src/javaRestTest/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseIT.java index 19afb4932ff2c..37efc2d7f2b43 100644 --- a/distribution/archives/integ-test-zip/src/javaRestTest/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseIT.java +++ b/distribution/archives/integ-test-zip/src/javaRestTest/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.test.rest; @@ -16,8 +17,10 @@ import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.ResponseListener; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.junit.After; import org.junit.Before; +import org.junit.ClassRule; import java.io.IOException; import java.util.Map; @@ -31,6 +34,15 @@ * Tests that wait for refresh is fired if the index is closed. */ public class WaitForRefreshAndCloseIT extends ESRestTestCase { + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + @Before public void setupIndex() throws IOException { Request request = new Request("PUT", "/test"); diff --git a/distribution/build.gradle b/distribution/build.gradle index 47367ab0261a2..5fc2846b031fb 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ @@ -13,7 +14,6 @@ import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.internal.ConcatFilesTask import org.elasticsearch.gradle.internal.DependenciesInfoPlugin import org.elasticsearch.gradle.internal.NoticeTask -import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.internal.test.HistoricalFeaturesMetadataPlugin import java.nio.file.Files @@ -207,7 +207,7 @@ project.rootProject.subprojects.findAll { it.parent.path == ':modules' }.each { distro.copyModule(processDefaultOutputsTaskProvider, module) dependencies.add('featuresMetadata', module) - if (module.name.startsWith('transport-') || (BuildParams.snapshotBuild == false && module.name == 'apm')) { + if (module.name.startsWith('transport-') || (buildParams.snapshotBuild == false && module.name == 'apm')) { distro.copyModule(processIntegTestOutputsTaskProvider, module) } @@ -261,7 +261,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { * Properties to expand when copying packaging files * *****************************************************************************/ configurations { - ['libs', 'libsVersionChecker', 'libsCliLauncher', 'libsServerCli', 'libsWindowsServiceCli', 'libsPluginCli', 'libsKeystoreCli', 'libsSecurityCli', 'libsGeoIpCli', 'libsAnsiConsole', 'libsNative'].each { + ['libs', 'libsVersionChecker', 'libsCliLauncher', 'libsServerCli', 'libsWindowsServiceCli', 'libsPluginCli', 'libsKeystoreCli', 'libsSecurityCli', 'libsGeoIpCli', 'libsAnsiConsole', 'libsNative', 'libsEntitlementAgent', 'libsEntitlementBridge'].each { create(it) { canBeConsumed = false canBeResolved = true @@ -274,7 +274,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { } all { resolutionStrategy.dependencySubstitution { - substitute module("org.apache.logging.log4j:log4j-core") using project(":libs:elasticsearch-log4j") because "patched to remove JndiLookup clas"} + substitute module("org.apache.logging.log4j:log4j-core") using project(":libs:log4j") because "patched to remove JndiLookup clas"} } } @@ -290,7 +290,9 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { libsKeystoreCli project(path: ':distribution:tools:keystore-cli') libsSecurityCli project(':x-pack:plugin:security:cli') libsGeoIpCli project(':distribution:tools:geoip-cli') - libsNative project(':libs:elasticsearch-native:elasticsearch-native-libraries') + libsNative project(':libs:native:native-libraries') + libsEntitlementAgent project(':libs:entitlement:agent') + libsEntitlementBridge project(':libs:entitlement:bridge') } project.ext { @@ -335,6 +337,12 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { include (os + '-' + architecture + '/*') } } + into('entitlement-agent') { + from(configurations.libsEntitlementAgent) + } + into('entitlement-bridge') { + from(configurations.libsEntitlementBridge) + } } } @@ -369,7 +377,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { exclude "**/platform/${excludePlatform}/**" } } - if (BuildParams.isSnapshotBuild()) { + if (buildParams.isSnapshotBuild()) { from(buildExternalTestModulesTaskProvider) } if (project.path.startsWith(':distribution:packages')) { @@ -510,7 +518,7 @@ subprojects { String licenseText if (isTestDistro) { - licenseText = rootProject.file('licenses/SSPL-1.0+ELASTIC-LICENSE-2.0.txt').getText('UTF-8') + licenseText = rootProject.file('licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt').getText('UTF-8') } else { licenseText = rootProject.file('licenses/ELASTIC-LICENSE-2.0.txt').getText('UTF-8') } @@ -639,8 +647,8 @@ subprojects { // in the final log4j2.properties configuration, as it appears in the // archive distribution. artifacts.add('log4jConfig', file("${defaultOutputs}/log4j2.properties")) { - type 'file' - name 'log4j2.properties' + type = 'file' + name = 'log4j2.properties' builtBy 'buildDefaultLog4jConfig' } diff --git a/distribution/bwc/bugfix2/build.gradle b/distribution/bwc/bugfix2/build.gradle new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/distribution/bwc/build.gradle b/distribution/bwc/build.gradle index 437cd730fff7d..1d34a433956c8 100644 --- a/distribution/bwc/build.gradle +++ b/distribution/bwc/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin:"elasticsearch.internal-distribution-bwc-setup" diff --git a/distribution/docker/README.md b/distribution/docker/README.md index eb0e7b296097d..49facab461edc 100644 --- a/distribution/docker/README.md +++ b/distribution/docker/README.md @@ -7,25 +7,29 @@ the [DockerBase] enum. * UBI - the same as the default image, but based upon [RedHat's UBI images][ubi], specifically their minimal flavour. * Wolfi - the same as the default image, but based upon [Wolfi](https://github.com/wolfi-dev) + * Cloud ESS - this directly extends the Wolfi image, and adds all ES plugins + that the ES build generates in an archive directory. It also sets an + environment variable that points at this directory. This allows plugins to + be installed from the archive instead of the internet, speeding up + deployment times. Furthermore this image has + * `filebeat` and `metricbeat` included + * `wget` included + * The `ENTRYPOINT` is just `/sbin/tini`, and the `CMD` is + `/app/elasticsearch.sh`. In normal use this file would be bind-mounted + in, but the image ships a stub version of this file so that the image + can still be tested. * Iron Bank - this is the US Department of Defence's repository of digitally signed, binary container images including both Free and Open-Source software (FOSS) and Commercial off-the-shelf (COTS). In practice, this is another UBI build, this time on the regular UBI image, with extra hardening. See below for more details. - * Cloud - this is mostly the same as the default image, with some notable differences: * `filebeat` and `metricbeat` are included * `wget` is included * The `ENTRYPOINT` is just `/bin/tini`, and the `CMD` is - `/app/elasticsearc.sh`. In normal use this file would be bind-mounted + `/app/elasticsearch.sh`. In normal use this file would be bind-mounted in, but the image ships a stub version of this file so that the image can still be tested. - * Cloud ESS - this directly extends the Cloud image, and adds all ES plugins - that the ES build generates in an archive directory. It also sets an - environment variable that points at this directory. This allows plugins to - be installed from the archive instead of the internet, speeding up - deployment times. - The long-term goal is for both Cloud images to be retired in favour of the default image. diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 30974ed2396a8..36aaf4e5625b7 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -1,4 +1,3 @@ -import org.elasticsearch.gradle.Architecture import org.elasticsearch.gradle.LoggedExec import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.internal.DockerBase @@ -8,8 +7,9 @@ import org.elasticsearch.gradle.internal.docker.DockerSupportPlugin import org.elasticsearch.gradle.internal.docker.DockerSupportService import org.elasticsearch.gradle.internal.docker.ShellRetry import org.elasticsearch.gradle.internal.docker.TransformLog4jConfigFilter -import org.elasticsearch.gradle.internal.info.BuildParams +import org.elasticsearch.gradle.internal.docker.* import org.elasticsearch.gradle.util.GradleUtils +import org.elasticsearch.gradle.Architecture import java.nio.file.Path import java.time.temporal.ChronoUnit @@ -28,7 +28,7 @@ repositories { // other Docker variants, the need for the main image to be rebuildable by Docker Hub // means that the Dockerfile itself has to fetch the binary. ivy { - url 'https://github.com/' + url = 'https://github.com/' patternLayout { artifact '/[organisation]/[module]/releases/download/v[revision]/[module]-[classifier]' } @@ -45,12 +45,12 @@ if (useDra == false) { ivy { name = 'beats' if (useLocalArtifacts) { - url getLayout().getBuildDirectory().dir("artifacts").get().asFile + url = getLayout().getBuildDirectory().dir("artifacts").get().asFile patternLayout { artifact '/[organisation]/[module]-[revision]-[classifier].[ext]' } } else { - url "https://artifacts-snapshot.elastic.co/" + url = "https://artifacts-snapshot.elastic.co/" patternLayout { if (VersionProperties.isElasticsearchSnapshot()) { artifact '/[organization]/[revision]/downloads/[organization]/[module]/[module]-[revision]-[classifier].[ext]' @@ -99,9 +99,9 @@ String tiniArch = Architecture.current() == Architecture.AARCH64 ? 'arm64' : 'am dependencies { aarch64DockerSource project(":distribution:archives:linux-aarch64-tar") - aarch64DockerSourceTar project(path: ":distribution:archives:linux-aarch64-tar", configuration:"default") + aarch64DockerSourceTar project(path: ":distribution:archives:linux-aarch64-tar", configuration: "default") dockerSource project(":distribution:archives:linux-tar") - dockerSourceTar project(path: ":distribution:archives:linux-tar", configuration:"default") + dockerSourceTar project(path: ":distribution:archives:linux-tar", configuration: "default") log4jConfig project(path: ":distribution", configuration: 'log4jConfig') tini "krallin:tini:0.19.0:${tiniArch}" allPlugins project(path: ':plugins', configuration: 'allPlugins') @@ -112,14 +112,14 @@ dependencies { } ext.expansions = { Architecture architecture, DockerBase base -> - def (major,minor) = VersionProperties.elasticsearch.split("\\.") + def (major, minor) = VersionProperties.elasticsearch.split("\\.") // We tag our Docker images with various pieces of information, including a timestamp // for when the image was built. However, this makes it impossible completely cache // the image. When developing the Docker images, it's very tedious to completely rebuild // an image for every single change. Therefore, outside of CI, we fix the // build time to midnight so that the Docker build cache is usable. - def buildDate = BuildParams.isCi() ? BuildParams.buildDate : BuildParams.buildDate.truncatedTo(ChronoUnit.DAYS).toString() + def buildDate = buildParams.isCi() ? buildParams.buildDate : buildParams.buildDate.truncatedTo(ChronoUnit.DAYS).toString() return [ 'arch' : architecture.classifier, @@ -127,7 +127,7 @@ ext.expansions = { Architecture architecture, DockerBase base -> 'bin_dir' : base == DockerBase.IRON_BANK ? 'scripts' : 'bin', 'build_date' : buildDate, 'config_dir' : base == DockerBase.IRON_BANK ? 'scripts' : 'config', - 'git_revision' : BuildParams.gitRevision, + 'git_revision' : buildParams.gitRevision.get(), 'license' : base == DockerBase.IRON_BANK ? 'Elastic License 2.0' : 'Elastic-License-2.0', 'package_manager' : base.packageManager, 'docker_base' : base.name().toLowerCase(), @@ -216,7 +216,8 @@ elasticsearch_distributions { } interface Injected { - @Inject FileSystemOperations getFs() + @Inject + FileSystemOperations getFs() } tasks.named("preProcessFixture").configure { @@ -286,21 +287,10 @@ void addBuildDockerContextTask(Architecture architecture, DockerBase base) { } } - if (base == DockerBase.CLOUD) { - // If we're performing a release build, but `build.id` hasn't been set, we can - // infer that we're not at the Docker building stage of the build, and therefore - // we should skip the beats part of the build. - String buildId = providers.systemProperty('build.id').getOrNull() - boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null || useDra - - if (includeBeats) { - from configurations.getByName("filebeat_${architecture.classifier}") - from configurations.getByName("metricbeat_${architecture.classifier}") - } - // For some reason, the artifact name can differ depending on what repository we used. - rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz" - } - Provider serviceProvider = GradleUtils.getBuildService(project.gradle.sharedServices, DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME) + Provider serviceProvider = GradleUtils.getBuildService( + project.gradle.sharedServices, + DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME + ) onlyIf("$architecture supported") { serviceProvider.get().isArchitectureSupported(architecture) } } @@ -337,9 +327,9 @@ void addTransformDockerContextTask(Architecture architecture, DockerBase base) { into "${project.buildDir}/docker-context/${archiveName}" // Since we replaced the remote URL in the Dockerfile, copy in the required file - if(base == DockerBase.IRON_BANK) { + if (base == DockerBase.IRON_BANK) { from(architecture == Architecture.AARCH64 ? configurations.aarch64DockerSourceTar : configurations.dockerSourceTar) - from (configurations.tini) { + from(configurations.tini) { rename { _ -> 'tini' } } } else { @@ -349,7 +339,10 @@ void addTransformDockerContextTask(Architecture architecture, DockerBase base) { expansions(architecture, base).findAll { it.key != 'build_date' }.each { k, v -> inputs.property(k, { v.toString() }) } - Provider serviceProvider = GradleUtils.getBuildService(project.gradle.sharedServices, DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME) + Provider serviceProvider = GradleUtils.getBuildService( + project.gradle.sharedServices, + DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME + ) onlyIf("$architecture supported") { serviceProvider.get().isArchitectureSupported(architecture) } } @@ -373,7 +366,7 @@ private static List generateTags(DockerBase base, Architecture architect String image = "elasticsearch${base.suffix}" String namespace = 'elasticsearch' - if (base == DockerBase.CLOUD || base == DockerBase.CLOUD_ESS) { + if (base == base == DockerBase.CLOUD_ESS) { namespace += '-ci' } @@ -396,7 +389,7 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) { dockerContext.fileProvider(transformTask.map { Sync task -> task.getDestinationDir() }) - noCache = BuildParams.isCi() + noCache = buildParams.isCi() tags = generateTags(base, architecture) platforms.add(architecture.dockerPlatform) @@ -423,12 +416,15 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) { baseImages = [base.image] } - Provider serviceProvider = GradleUtils.getBuildService(project.gradle.sharedServices, DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME) + Provider serviceProvider = GradleUtils.getBuildService( + project.gradle.sharedServices, + DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME + ) onlyIf("$architecture supported") { serviceProvider.get().isArchitectureSupported(architecture) } } - if (base != DockerBase.IRON_BANK && base != DockerBase.CLOUD && base != DockerBase.CLOUD_ESS) { + if (base != DockerBase.IRON_BANK && base != DockerBase.CLOUD_ESS) { tasks.named("assemble").configure { dependsOn(buildDockerImageTask) } @@ -436,12 +432,12 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) { } void addBuildEssDockerImageTask(Architecture architecture) { - DockerBase base = DockerBase.CLOUD_ESS + DockerBase dockerBase = DockerBase.CLOUD_ESS String arch = architecture == Architecture.AARCH64 ? '-aarch64' : '' - String contextDir = "${project.buildDir}/docker-context/elasticsearch${base.suffix}-${VersionProperties.elasticsearch}-docker-build-context${arch}" + String contextDir = "${project.buildDir}/docker-context/elasticsearch${dockerBase.suffix}-${VersionProperties.elasticsearch}-docker-build-context${arch}" final TaskProvider buildContextTask = - tasks.register(taskName('build', architecture, base, 'DockerContext'), Sync) { + tasks.register(taskName('build', architecture, dockerBase, 'DockerContext'), Sync) { into contextDir final Path projectDir = project.projectDir.toPath() @@ -450,28 +446,52 @@ void addBuildEssDockerImageTask(Architecture architecture) { from configurations.allPlugins } - from(projectDir.resolve("src/docker/Dockerfile.cloud-ess")) { - expand([ - base_image: "elasticsearch${DockerBase.CLOUD.suffix}:${architecture.classifier}" - ]) + // If we're performing a release build, but `build.id` hasn't been set, we can + // infer that we're not at the Docker building stage of the build, and therefore + // we should skip the beats part of the build. + String buildId = providers.systemProperty('build.id').getOrNull() + boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null || useDra + + if (includeBeats) { + from configurations.getByName("filebeat_${architecture.classifier}") + from configurations.getByName("metricbeat_${architecture.classifier}") + } + // For some reason, the artifact name can differ depending on what repository we used. + rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz" + + String baseSuffix = DockerBase.WOLFI.suffix + from(projectDir.resolve("src/docker/Dockerfile.ess")) { + expand( + [ + base_image: "elasticsearch${baseSuffix}:${architecture.classifier}", + docker_base: "${dockerBase.name().toLowerCase()}", + version: "${VersionProperties.elasticsearch}", + retry: ShellRetry + ] + ) filter SquashNewlinesFilter - rename ~/Dockerfile\.cloud-ess$/, 'Dockerfile' + rename ~/Dockerfile\.ess$/, 'Dockerfile' } } final TaskProvider buildDockerImageTask = - tasks.register(taskName("build", architecture, base, "DockerImage"), DockerBuildTask) { + tasks.register(taskName("build", architecture, dockerBase, "DockerImage"), DockerBuildTask) { + + DockerBase base = DockerBase.WOLFI - TaskProvider buildCloudTask = tasks.named(taskName("build", architecture, DockerBase.CLOUD, "DockerImage")) - inputs.files(buildCloudTask) + TaskProvider buildBaseTask = tasks.named(taskName("build", architecture, base, "DockerImage")) + inputs.files(buildBaseTask) dockerContext.fileProvider(buildContextTask.map { it.getDestinationDir() }) - noCache = BuildParams.isCi() + noCache = buildParams.isCi() baseImages = [] - tags = generateTags(base, architecture) + tags = generateTags(dockerBase, architecture) platforms.add(architecture.dockerPlatform) - Provider serviceProvider = GradleUtils.getBuildService(project.gradle.sharedServices, DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME) + Provider serviceProvider = GradleUtils.getBuildService( + project.gradle.sharedServices, + DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME + ) onlyIf("$architecture supported") { serviceProvider.get().isArchitectureSupported(architecture) } } @@ -513,8 +533,6 @@ subprojects { Project subProject -> base = DockerBase.IRON_BANK } else if (subProject.name.contains('cloud-ess-')) { base = DockerBase.CLOUD_ESS - } else if (subProject.name.contains('cloud-')) { - base = DockerBase.CLOUD } else if (subProject.name.contains('wolfi-')) { base = DockerBase.WOLFI } @@ -522,10 +540,9 @@ subprojects { Project subProject -> final String arch = architecture == Architecture.AARCH64 ? '-aarch64' : '' final String extension = base == DockerBase.UBI ? 'ubi.tar' : (base == DockerBase.IRON_BANK ? 'ironbank.tar' : - (base == DockerBase.CLOUD ? 'cloud.tar' : (base == DockerBase.CLOUD_ESS ? 'cloud-ess.tar' : (base == DockerBase.WOLFI ? 'wolfi.tar' : - 'docker.tar')))) + 'docker.tar'))) final String artifactName = "elasticsearch${arch}${base.suffix}_test" final String exportTaskName = taskName("export", architecture, base, 'DockerImage') @@ -536,12 +553,16 @@ subprojects { Project subProject -> inputs.file("${parent.projectDir}/build/markers/${buildTaskName}.marker") executable = 'docker' outputs.file(tarFile) + outputs.doNotCacheIf("Build cache is disabled for export tasks") { true } args "save", "-o", tarFile, "elasticsearch${base.suffix}:${architecture.classifier}" dependsOn(parent.path + ":" + buildTaskName) - Provider serviceProvider = GradleUtils.getBuildService(project.gradle.sharedServices, DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME) + Provider serviceProvider = GradleUtils.getBuildService( + project.gradle.sharedServices, + DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME + ) onlyIf("$architecture supported") { serviceProvider.get().isArchitectureSupported(architecture) } } @@ -565,8 +586,8 @@ subprojects { Project subProject -> } artifacts.add('default', file(tarFile)) { - type 'tar' - name artifactName + type = 'tar' + name = artifactName builtBy exportTaskName } } diff --git a/distribution/docker/cloud-docker-aarch64-export/build.gradle b/distribution/docker/cloud-docker-aarch64-export/build.gradle deleted file mode 100644 index 537b5a093683e..0000000000000 --- a/distribution/docker/cloud-docker-aarch64-export/build.gradle +++ /dev/null @@ -1,2 +0,0 @@ -// This file is intentionally blank. All configuration of the -// export is done in the parent project. diff --git a/distribution/docker/cloud-docker-export/build.gradle b/distribution/docker/cloud-docker-export/build.gradle deleted file mode 100644 index 537b5a093683e..0000000000000 --- a/distribution/docker/cloud-docker-export/build.gradle +++ /dev/null @@ -1,2 +0,0 @@ -// This file is intentionally blank. All configuration of the -// export is done in the parent project. diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile index 47f79749cbefa..37319e95a5970 100644 --- a/distribution/docker/src/docker/Dockerfile +++ b/distribution/docker/src/docker/Dockerfile @@ -22,7 +22,7 @@ <% if (docker_base == 'iron_bank') { %> ARG BASE_REGISTRY=registry1.dso.mil ARG BASE_IMAGE=ironbank/redhat/ubi/ubi9 -ARG BASE_TAG=9.4 +ARG BASE_TAG=9.5 <% } %> ################################################################################ @@ -163,9 +163,16 @@ RUN <%= retry.loop(package_manager, " ${package_manager} update && \n" + " ${package_manager} upgrade && \n" + " ${package_manager} add --no-cache \n" + - " bash ca-certificates curl libsystemd netcat-openbsd p11-kit p11-kit-trust shadow tini unzip zip zstd && \n" + + " bash java-cacerts curl libstdc++ libsystemd netcat-openbsd p11-kit p11-kit-trust posix-libc-utils shadow tini unzip zip zstd && \n" + " rm -rf /var/cache/apk/* " ) %> + +# Set Bash as the default shell for future commands +SHELL ["/bin/bash", "-c"] + +# Optionally set Bash as the default shell in the container at runtime +CMD ["/bin/bash"] + <% } else if (docker_base == "default" || docker_base == "cloud") { %> # Change default shell to bash, then install required packages with retries. @@ -224,7 +231,7 @@ COPY --from=builder --chown=0:0 /opt /opt <% } %> ENV PATH /usr/share/elasticsearch/bin:\$PATH - +ENV SHELL /bin/bash COPY ${bin_dir}/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh # 1. Sync the user and group permissions of /etc/passwd @@ -249,6 +256,8 @@ RUN chmod g=u /etc/passwd && \\ # stays up-to-date with changes to Ubuntu's store) COPY bin/docker-openjdk /etc/ca-certificates/update.d/docker-openjdk RUN /etc/ca-certificates/update.d/docker-openjdk +<% } else if (docker_base == 'wolfi') { %> +RUN ln -sf /etc/ssl/certs/java/cacerts /usr/share/elasticsearch/jdk/lib/security/cacerts <% } else { %> RUN ln -sf /etc/pki/ca-trust/extracted/java/cacerts /usr/share/elasticsearch/jdk/lib/security/cacerts <% } %> diff --git a/distribution/docker/src/docker/Dockerfile.cloud-ess b/distribution/docker/src/docker/Dockerfile.cloud-ess deleted file mode 100644 index f82752d67a284..0000000000000 --- a/distribution/docker/src/docker/Dockerfile.cloud-ess +++ /dev/null @@ -1,13 +0,0 @@ -FROM ${base_image} AS builder - -USER root - -COPY plugins/*.zip /opt/plugins/archive/ - -RUN chown root.root /opt/plugins/archive/* -RUN chmod 0444 /opt/plugins/archive/* - -FROM ${base_image} - -COPY --from=builder /opt/plugins /opt/plugins -ENV ES_PLUGIN_ARCHIVE_DIR /opt/plugins/archive diff --git a/distribution/docker/src/docker/Dockerfile.ess b/distribution/docker/src/docker/Dockerfile.ess new file mode 100644 index 0000000000000..197af28b93455 --- /dev/null +++ b/distribution/docker/src/docker/Dockerfile.ess @@ -0,0 +1,44 @@ +FROM ${base_image} AS builder + +USER root + +# Add plugins infrastructure +RUN mkdir -p /opt/plugins/archive +RUN chmod -R 0555 /opt/plugins + +COPY filebeat-${version}.tar.gz metricbeat-${version}.tar.gz /tmp/ +RUN set -eux ; \\ + for beat in filebeat metricbeat ; do \\ + if [ ! -s /tmp/\$beat-${version}.tar.gz ]; then \\ + echo "/tmp/\$beat-${version}.tar.gz is empty - cannot uncompress" 2>&1 ; \\ + exit 1 ; \\ + fi ; \\ + if ! tar tf /tmp/\$beat-${version}.tar.gz >/dev/null; then \\ + echo "/tmp/\$beat-${version}.tar.gz is corrupt - cannot uncompress" 2>&1 ; \\ + exit 1 ; \\ + fi ; \\ + mkdir -p /opt/\$beat ; \\ + tar xf /tmp/\$beat-${version}.tar.gz -C /opt/\$beat --strip-components=1 ; \\ + done + +COPY plugins/*.zip /opt/plugins/archive/ + +RUN chown 1000:1000 /opt/plugins/archive/* +RUN chmod 0444 /opt/plugins/archive/* + +FROM ${base_image} +USER root + +RUN <%= retry.loop("apk", "export DEBIAN_FRONTEND=noninteractive && apk update && apk update && apk add --no-cache wget") %> + +# tweak entry point for ESS specific wolfi image +ENTRYPOINT ["/sbin/tini", "--"] +CMD ["/app/elasticsearch.sh"] +# Generate a stub command that will be overwritten at runtime +RUN mkdir /app && \\ + echo -e '#!/bin/bash\\nexec /usr/local/bin/docker-entrypoint.sh eswrapper' > /app/elasticsearch.sh && \\ + chmod 0555 /app/elasticsearch.sh + +COPY --from=builder --chown=0:0 /opt /opt +USER 1000:0 +ENV ES_PLUGIN_ARCHIVE_DIR /opt/plugins/archive diff --git a/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml b/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml index f4364c5008c09..19b4a13dc9f22 100644 --- a/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml +++ b/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml @@ -1,21 +1,17 @@ --- apiVersion: v1 - # The repository name in registry1, excluding /ironbank/ name: "elastic/elasticsearch/elasticsearch" - # List of tags to push for the repository in registry1 # The most specific version should be the first tag and will be shown # on ironbank.dsop.io tags: - "${version}" - "latest" - # Build args passed to Dockerfile ARGs args: BASE_IMAGE: "redhat/ubi/ubi9" - BASE_TAG: "9.4" - + BASE_TAG: "9.5" # Docker image labels labels: org.opencontainers.image.title: "elasticsearch" @@ -34,7 +30,6 @@ labels: mil.dso.ironbank.image.type: "commercial" # Product the image belongs to for grouping multiple images mil.dso.ironbank.product.name: "elasticsearch" - # List of resources to make available to the offline build context resources: - filename: "elasticsearch-${version}-linux-x86_64.tar.gz" @@ -47,12 +42,14 @@ resources: validation: type: "sha256" value: "93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c" - # List of project maintainers maintainers: - - name: "Rory Hunter" - email: "rory.hunter@elastic.co" - username: "rory" + - name: "Mark Vieira" + email: "mark.vieira@elastic.co" + username: "mark-vieira" + - name: "Rene Gröschke" + email: "rene.groschke@elastic.co" + username: "breskeby" - email: "klepal_alexander@bah.com" name: "Alexander Klepal" username: "alexander.klepal" diff --git a/distribution/docker/src/yamlRestTest/java/org/elasticsearch/docker/test/DockerYmlTestSuiteIT.java b/distribution/docker/src/yamlRestTest/java/org/elasticsearch/docker/test/DockerYmlTestSuiteIT.java index 2cba6ea4d4bda..ab2a4bb0e73e7 100644 --- a/distribution/docker/src/yamlRestTest/java/org/elasticsearch/docker/test/DockerYmlTestSuiteIT.java +++ b/distribution/docker/src/yamlRestTest/java/org/elasticsearch/docker/test/DockerYmlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.docker.test; diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle index 2dfd24d97cbba..9cf718b0c992a 100644 --- a/distribution/packages/build.gradle +++ b/distribution/packages/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.apache.tools.ant.filters.ReplaceTokens @@ -43,7 +44,7 @@ import java.util.regex.Pattern */ plugins { - id "com.netflix.nebula.ospackage-base" version "11.9.1" + id "com.netflix.nebula.ospackage-base" version "11.10.1" } ['deb', 'rpm'].each { type -> @@ -87,21 +88,21 @@ def commonPackageConfig(String type, String architecture) { OS.current().equals(OS.WINDOWS) == false } dependsOn "process${type.capitalize()}Files" - packageName "elasticsearch" + packageName = "elasticsearch" if (type == 'deb') { if (architecture == 'x64') { - arch('amd64') + arch = 'amd64' } else { assert architecture == 'aarch64': architecture - arch('arm64') + arch = 'arm64' } } else { assert type == 'rpm': type if (architecture == 'x64') { - arch('X86_64') + arch = 'X86_64' } else { assert architecture == 'aarch64': architecture - arch('aarch64') + arch = 'aarch64' } } // Follow elasticsearch's file naming convention @@ -195,12 +196,12 @@ def commonPackageConfig(String type, String architecture) { configurationFile '/etc/elasticsearch/users_roles' from("${packagingFiles}") { dirPermissions { - unix(02750) + unix(0750) } into('/etc') permissionGroup 'elasticsearch' setgid true - includeEmptyDirs true + includeEmptyDirs = true createDirectoryEntry true include("elasticsearch") // empty dir, just to add directory entry include("elasticsearch/jvm.options.d") // empty dir, just to add directory entry @@ -208,14 +209,14 @@ def commonPackageConfig(String type, String architecture) { from("${packagingFiles}/etc/elasticsearch") { into('/etc/elasticsearch') dirPermissions { - unix(02750) + unix(0750) } setgid = true filePermissions { unix(0660) } permissionGroup 'elasticsearch' - includeEmptyDirs true + includeEmptyDirs = true createDirectoryEntry true fileType CONFIG | NOREPLACE } @@ -260,24 +261,24 @@ def commonPackageConfig(String type, String architecture) { // ========= empty dirs ========= // NOTE: these are created under packagingFiles as empty, but the permissions are set here - Closure copyEmptyDir = { path, u, g, mode -> + Closure copyEmptyDir = { path, u, g, gid, mode -> File file = new File(path) into(file.parent) { from "${packagingFiles}/${file.parent}" include file.name - includeEmptyDirs true + includeEmptyDirs = true createDirectoryEntry true user u permissionGroup g dirPermissions { unix(mode) } - setgid (mode == 02750) + setgid(gid) } } - copyEmptyDir('/var/log/elasticsearch', 'elasticsearch', 'elasticsearch', 02750) - copyEmptyDir('/var/lib/elasticsearch', 'elasticsearch', 'elasticsearch', 02750) - copyEmptyDir('/usr/share/elasticsearch/plugins', 'root', 'root', 0755) + copyEmptyDir('/var/log/elasticsearch', 'elasticsearch', 'elasticsearch', true, 0750) + copyEmptyDir('/var/lib/elasticsearch', 'elasticsearch', 'elasticsearch', true, 0750) + copyEmptyDir('/usr/share/elasticsearch/plugins', 'root', 'root', false, 0755) // the oss package conflicts with the default distribution and vice versa conflicts('elasticsearch-oss') @@ -289,18 +290,18 @@ def commonPackageConfig(String type, String architecture) { // this is package independent configuration ospackage { - maintainer 'Elasticsearch Team ' - summary 'Distributed RESTful search engine built for the cloud' - packageDescription ''' + maintainer = 'Elasticsearch Team ' + summary = 'Distributed RESTful search engine built for the cloud' + packageDescription = ''' Reference documentation can be found at https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html and the 'Elasticsearch: The Definitive Guide' book can be found at https://www.elastic.co/guide/en/elasticsearch/guide/current/index.html '''.stripIndent().trim() - url 'https://www.elastic.co/' + url = 'https://www.elastic.co/' // signing setup - if (project.hasProperty('signing.password') && BuildParams.isSnapshotBuild() == false) { + if (project.hasProperty('signing.password') && buildParams.isSnapshotBuild() == false) { signingKeyId = project.hasProperty('signing.keyId') ? project.property('signing.keyId') : 'D88E42B4' signingKeyPassphrase = project.property('signing.password') signingKeyRingFile = project.hasProperty('signing.secretKeyRingFile') ? @@ -311,10 +312,10 @@ ospackage { // version found on oldest supported distro, centos-6 requires('coreutils', '8.4', GREATER | EQUAL) - fileMode 0644 - dirMode 0755 - user 'root' - permissionGroup 'root' + fileMode = 0644 + dirMode = 0755 + user = 'root' + permissionGroup = 'root' into '/usr/share/elasticsearch' } @@ -330,11 +331,10 @@ Closure commonDebConfig(String architecture) { customFields['License'] = 'Elastic-License' archiveVersion = project.version.replace('-', '~') - packageGroup 'web' + packageGroup = 'web' // versions found on oldest supported distro, centos-6 requires('bash', '4.1', GREATER | EQUAL) - requires('lsb-base', '4', GREATER | EQUAL) requires 'libc6' requires 'adduser' @@ -359,24 +359,24 @@ Closure commonRpmConfig(String architecture) { return { configure(commonPackageConfig('rpm', architecture)) - license 'Elastic License' + license = 'Elastic License' - packageGroup 'Application/Internet' + packageGroup = 'Application/Internet' requires '/bin/bash' obsoletes packageName, '7.0.0', Flags.LESS prefix '/usr' - packager 'Elasticsearch' + packager = 'Elasticsearch' archiveVersion = project.version.replace('-', '_') release = '1' - os 'LINUX' - distribution 'Elasticsearch' - vendor 'Elasticsearch' + os = 'LINUX' + distribution = 'Elasticsearch' + vendor = 'Elasticsearch' // TODO ospackage doesn't support icon but we used to have one // without this the rpm will have parent dirs of any files we copy in, eg /etc/elasticsearch - addParentDirs false + addParentDirs = false } } diff --git a/distribution/packages/src/deb/lintian/elasticsearch b/distribution/packages/src/deb/lintian/elasticsearch index edd705b66caaa..1622d8d8aeb40 100644 --- a/distribution/packages/src/deb/lintian/elasticsearch +++ b/distribution/packages/src/deb/lintian/elasticsearch @@ -5,8 +5,6 @@ changelog-file-missing-in-native-package # we intentionally copy our copyright file for all deb packages -copyright-file-contains-full-apache-2-license -copyright-not-using-common-license-for-apache2 copyright-without-copyright-notice # we still put all our files under /usr/share/elasticsearch even after transition to platform dependent packages @@ -16,37 +14,23 @@ arch-dependent-file-in-usr-share missing-dep-on-jarwrapper # we prefer to not make our config and log files world readable -non-standard-file-perm etc/default/elasticsearch 0660 != 0644 -non-standard-dir-perm etc/elasticsearch/ 2750 != 0755 -non-standard-dir-perm etc/elasticsearch/jvm.options.d/ 2750 != 0755 -non-standard-file-perm etc/elasticsearch/* -non-standard-dir-perm var/lib/elasticsearch/ 2750 != 0755 -non-standard-dir-perm var/log/elasticsearch/ 2750 != 0755 - -# this lintian tag is simply wrong; contrary to the explanation, Debian systemd -# does actually look at /usr/lib/systemd/system -systemd-service-file-outside-lib usr/lib/systemd/system/elasticsearch.service +non-standard-file-perm 0660 != 0644 [etc/default/elasticsearch] +non-standard-dir-perm 2750 != 0755 [etc/elasticsearch/] +non-standard-dir-perm 2750 != 0755 [etc/elasticsearch/jvm.options.d/] +non-standard-file-perm 0660 != 0644 [etc/elasticsearch/*] +non-standard-dir-perm 2750 != 0755 [var/lib/elasticsearch/] +non-standard-dir-perm 2750 != 0755 [var/log/elasticsearch/] # the package scripts handle systemd directly and don't need to use deb helpers maintainer-script-calls-systemctl # bundled JDK embedded-library -unstripped-binary-or-object usr/share/elasticsearch/jdk/* -extra-license-file usr/share/elasticsearch/jdk/legal/* -hardening-no-pie usr/share/elasticsearch/jdk/bin/* -hardening-no-pie usr/share/elasticsearch/jdk/lib/* +unstripped-binary-or-object [usr/share/elasticsearch/jdk/*] # the system java version that lintian assumes is far behind what elasticsearch uses unknown-java-class-version -# elastic licensed modules contain elastic license -extra-license-file usr/share/elasticsearch/modules/* - -# This dependency appears to have a packaging flaw, and includes a -# generated source file alongside the compiled version -jar-contains-source usr/share/elasticsearch/modules/repository-gcs/api-common*.jar * - # There's no `License` field in Debian control files, but earlier versions # of `lintian` were more permissive. Override this warning so that we can # run `lintian` on different releases of Debian. The format of this override @@ -58,8 +42,27 @@ unknown-field License # indirectly to libc via libdl. This might not be best practice but we # don't build them ourselves and the license precludes us modifying them # to fix this. -library-not-linked-against-libc usr/share/elasticsearch/modules/x-pack-ml/platform/linux-x86_64/lib/libmkl_*.so +library-not-linked-against-libc [usr/share/elasticsearch/modules/x-pack-ml/platform/linux-x86_64/lib/libmkl_*.so*] + + +# Below is the copy of some of the above rules in format for Lintian versions <= 2.104 (Debian 11) +# Override syntax changes between Lintian versions in a non-backwards compatible way, so we handle it with +# duplication and ignoring some issues in the test code. + + +# we prefer to not make our config and log files world readable +non-standard-file-perm etc/default/elasticsearch 0660 != 0644 +non-standard-dir-perm etc/elasticsearch/ 2750 != 0755 +non-standard-dir-perm etc/elasticsearch/jvm.options.d/ 2750 != 0755 +non-standard-file-perm etc/elasticsearch/* +non-standard-dir-perm var/lib/elasticsearch/ 2750 != 0755 +non-standard-dir-perm var/log/elasticsearch/ 2750 != 0755 -# shared-lib-without-dependency-information (now shared-library-lacks-prerequisites) is falsely reported for libvec.so -# which has no dependencies (not even libc) besides the symbols in the base executable. -shared-lib-without-dependency-information usr/share/elasticsearch/lib/platform/linux-x64/libvec.so +# bundled JDK +unstripped-binary-or-object usr/share/elasticsearch/jdk/* + +# Intel MKL libraries are not linked directly to libc. They are linked +# indirectly to libc via libdl. This might not be best practice but we +# don't build them ourselves and the license precludes us modifying them +# to fix this. +library-not-linked-against-libc usr/share/elasticsearch/modules/x-pack-ml/platform/linux-x86_64/lib/libmkl_*.so* diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options index c5e905f461f45..340baedded785 100644 --- a/distribution/src/config/jvm.options +++ b/distribution/src/config/jvm.options @@ -9,7 +9,7 @@ ## should create one or more files in the jvm.options.d ## directory containing your adjustments. ## -## See https://www.elastic.co/guide/en/elasticsearch/reference/@project.minor.version@/jvm-options.html +## See https://www.elastic.co/guide/en/elasticsearch/reference/@project.minor.version@/advanced-configuration.html#set-jvm-options ## for more information. ## ################################################################ @@ -58,6 +58,10 @@ # result in less optimal vector performance 20-:--add-modules=jdk.incubator.vector +# Required to workaround performance issue in JDK 23, https://github.com/elastic/elasticsearch/issues/113030 +23:-XX:CompileCommand=dontinline,java/lang/invoke/MethodHandle.setAsTypeCache +23:-XX:CompileCommand=dontinline,java/lang/invoke/MethodHandle.asTypeUncached + ## heap dumps # generate a heap dump when an allocation from the Java heap fails; heap dumps diff --git a/distribution/tools/ansi-console/build.gradle b/distribution/tools/ansi-console/build.gradle index 2a1057b164524..d01d60fb50f07 100644 --- a/distribution/tools/ansi-console/build.gradle +++ b/distribution/tools/ansi-console/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.build' diff --git a/distribution/tools/ansi-console/src/main/java/org/elasticsearch/io/ansi/AnsiConsoleLoader.java b/distribution/tools/ansi-console/src/main/java/org/elasticsearch/io/ansi/AnsiConsoleLoader.java index 427978d3ec3ec..6bf18e8a3c060 100644 --- a/distribution/tools/ansi-console/src/main/java/org/elasticsearch/io/ansi/AnsiConsoleLoader.java +++ b/distribution/tools/ansi-console/src/main/java/org/elasticsearch/io/ansi/AnsiConsoleLoader.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.io.ansi; diff --git a/distribution/tools/ansi-console/src/test/java/org/elasticsearch/bootstrap/ConsoleLoaderTests.java b/distribution/tools/ansi-console/src/test/java/org/elasticsearch/bootstrap/ConsoleLoaderTests.java index f34ddae8a2ffb..5ff65e5677a0a 100644 --- a/distribution/tools/ansi-console/src/test/java/org/elasticsearch/bootstrap/ConsoleLoaderTests.java +++ b/distribution/tools/ansi-console/src/test/java/org/elasticsearch/bootstrap/ConsoleLoaderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.bootstrap; diff --git a/distribution/tools/ansi-console/src/test/java/org/elasticsearch/io/ansi/AnsiConsoleLoaderTests.java b/distribution/tools/ansi-console/src/test/java/org/elasticsearch/io/ansi/AnsiConsoleLoaderTests.java index 4b40f8fa568de..9f0f39710238b 100644 --- a/distribution/tools/ansi-console/src/test/java/org/elasticsearch/io/ansi/AnsiConsoleLoaderTests.java +++ b/distribution/tools/ansi-console/src/test/java/org/elasticsearch/io/ansi/AnsiConsoleLoaderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.io.ansi; diff --git a/distribution/tools/cli-launcher/src/main/java/org/elasticsearch/launcher/CliToolLauncher.java b/distribution/tools/cli-launcher/src/main/java/org/elasticsearch/launcher/CliToolLauncher.java index 981033aeccd8c..a3ef768c30100 100644 --- a/distribution/tools/cli-launcher/src/main/java/org/elasticsearch/launcher/CliToolLauncher.java +++ b/distribution/tools/cli-launcher/src/main/java/org/elasticsearch/launcher/CliToolLauncher.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.launcher; diff --git a/distribution/tools/cli-launcher/src/test/java/org/elasticsearch/launcher/CliToolLauncherTests.java b/distribution/tools/cli-launcher/src/test/java/org/elasticsearch/launcher/CliToolLauncherTests.java index 875da8cdd628c..df11a529d7a19 100644 --- a/distribution/tools/cli-launcher/src/test/java/org/elasticsearch/launcher/CliToolLauncherTests.java +++ b/distribution/tools/cli-launcher/src/test/java/org/elasticsearch/launcher/CliToolLauncherTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.launcher; diff --git a/distribution/tools/geoip-cli/build.gradle b/distribution/tools/geoip-cli/build.gradle index 1cd502fa91d51..26af3bb4f9911 100644 --- a/distribution/tools/geoip-cli/build.gradle +++ b/distribution/tools/geoip-cli/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.build' @@ -14,8 +15,8 @@ base { dependencies { compileOnly project(":server") - compileOnly project(":libs:elasticsearch-cli") - compileOnly project(":libs:elasticsearch-x-content") + compileOnly project(":libs:cli") + compileOnly project(":libs:x-content") testImplementation project(":test:framework") testImplementation "org.apache.commons:commons-compress:1.26.1" testImplementation "commons-io:commons-io:2.15.1" diff --git a/distribution/tools/geoip-cli/src/main/java/org/elasticsearch/geoip/GeoIpCli.java b/distribution/tools/geoip-cli/src/main/java/org/elasticsearch/geoip/GeoIpCli.java index 5edfe0a326fec..208eefab7c18f 100644 --- a/distribution/tools/geoip-cli/src/main/java/org/elasticsearch/geoip/GeoIpCli.java +++ b/distribution/tools/geoip-cli/src/main/java/org/elasticsearch/geoip/GeoIpCli.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geoip; diff --git a/distribution/tools/geoip-cli/src/main/java/org/elasticsearch/geoip/GeoIpCliProvider.java b/distribution/tools/geoip-cli/src/main/java/org/elasticsearch/geoip/GeoIpCliProvider.java index 2b6b0559a2898..fa0801470a3d4 100644 --- a/distribution/tools/geoip-cli/src/main/java/org/elasticsearch/geoip/GeoIpCliProvider.java +++ b/distribution/tools/geoip-cli/src/main/java/org/elasticsearch/geoip/GeoIpCliProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geoip; diff --git a/distribution/tools/geoip-cli/src/test/java/org/elasticsearch/geoip/GeoIpCliTests.java b/distribution/tools/geoip-cli/src/test/java/org/elasticsearch/geoip/GeoIpCliTests.java index b1d6a529f883d..7daec3365c379 100644 --- a/distribution/tools/geoip-cli/src/test/java/org/elasticsearch/geoip/GeoIpCliTests.java +++ b/distribution/tools/geoip-cli/src/test/java/org/elasticsearch/geoip/GeoIpCliTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geoip; @@ -19,7 +20,6 @@ import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; -import java.io.BufferedInputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -105,9 +105,7 @@ private void verifyOverview() throws Exception { private void verifyTarball(Map data) throws Exception { for (String tgz : List.of("a.tgz", "b.tgz")) { try ( - TarArchiveInputStream tis = new TarArchiveInputStream( - new GZIPInputStream(new BufferedInputStream(Files.newInputStream(target.resolve(tgz)))) - ) + TarArchiveInputStream tis = new TarArchiveInputStream(new GZIPInputStream(Files.newInputStream(target.resolve(tgz)), 8192)) ) { TarArchiveEntry entry = tis.getNextTarEntry(); assertNotNull(entry); diff --git a/distribution/tools/java-version-checker/build.gradle b/distribution/tools/java-version-checker/build.gradle index 0a47d0652e465..3d4ec5aced29c 100644 --- a/distribution/tools/java-version-checker/build.gradle +++ b/distribution/tools/java-version-checker/build.gradle @@ -1,30 +1,10 @@ apply plugin: 'elasticsearch.build' -sourceSets { - unsupportedJdkVersionEntrypoint -} - -tasks.named(sourceSets.unsupportedJdkVersionEntrypoint.compileJavaTaskName).configure { - targetCompatibility = JavaVersion.VERSION_1_8 -} - - -tasks.named("jar") { - manifest { - attributes("Multi-Release": "true") - } - - FileCollection mainOutput = sourceSets.main.output; - from(sourceSets.unsupportedJdkVersionEntrypoint.output) - eachFile { details -> - if (details.path.equals("org/elasticsearch/tools/java_version_checker/JavaVersionChecker.class") && - mainOutput.asFileTree.contains(details.file)) { - details.relativePath = details.relativePath.prepend("META-INF/versions/17") - } - } +compileJava { + options.release = 8 } // TODO revisit forbiddenApis issues -["javadoc", "forbiddenApisMain", "forbiddenApisUnsupportedJdkVersionEntrypoint"].each { +["javadoc", "forbiddenApisMain"].each { tasks.named(it).configure { enabled = false } } diff --git a/distribution/tools/java-version-checker/src/main/java/org/elasticsearch/tools/java_version_checker/JavaVersionChecker.java b/distribution/tools/java-version-checker/src/main/java/org/elasticsearch/tools/java_version_checker/JavaVersionChecker.java index 3b716709f644e..6d6d10b4d733b 100644 --- a/distribution/tools/java-version-checker/src/main/java/org/elasticsearch/tools/java_version_checker/JavaVersionChecker.java +++ b/distribution/tools/java-version-checker/src/main/java/org/elasticsearch/tools/java_version_checker/JavaVersionChecker.java @@ -1,17 +1,19 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.tools.java_version_checker; import java.util.Arrays; +import java.util.Locale; /** - * Java 17 compatible main which just exits without error. + * Java 8 compatible main to check the runtime version */ final class JavaVersionChecker { @@ -22,5 +24,27 @@ public static void main(final String[] args) { if (args.length != 0) { throw new IllegalArgumentException("expected zero arguments but was " + Arrays.toString(args)); } + + final int MIN_VERSION = 17; + final int version; + String versionString = System.getProperty("java.specification.version"); + if (versionString.equals("1.8")) { + version = 8; + } else { + version = Integer.parseInt(versionString); + } + if (version >= MIN_VERSION) { + return; + } + + final String message = String.format( + Locale.ROOT, + "The minimum required Java version is %d; your Java version %d from [%s] does not meet that requirement.", + MIN_VERSION, + version, + System.getProperty("java.home") + ); + System.err.println(message); + System.exit(1); } } diff --git a/distribution/tools/java-version-checker/src/unsupportedJdkVersionEntrypoint/java/org/elasticsearch/tools/java_version_checker/JavaVersionChecker.java b/distribution/tools/java-version-checker/src/unsupportedJdkVersionEntrypoint/java/org/elasticsearch/tools/java_version_checker/JavaVersionChecker.java deleted file mode 100644 index f3a47c7b5ad43..0000000000000 --- a/distribution/tools/java-version-checker/src/unsupportedJdkVersionEntrypoint/java/org/elasticsearch/tools/java_version_checker/JavaVersionChecker.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.tools.java_version_checker; - -import java.util.Arrays; -import java.util.Locale; - -/** - * Java 7 compatible main which exits with an error. - */ -final class JavaVersionChecker { - - private JavaVersionChecker() {} - - public static void main(final String[] args) { - // no leniency! - if (args.length != 0) { - throw new IllegalArgumentException("expected zero arguments but was " + Arrays.toString(args)); - } - final String message = String.format( - Locale.ROOT, - "The minimum required Java version is 17; your Java version %s from [%s] does not meet that requirement.", - System.getProperty("java.specification.version"), - System.getProperty("java.home") - ); - System.err.println(message); - System.exit(1); - } -} diff --git a/distribution/tools/keystore-cli/build.gradle b/distribution/tools/keystore-cli/build.gradle index 71c6724248340..0140cd9d8eedf 100644 --- a/distribution/tools/keystore-cli/build.gradle +++ b/distribution/tools/keystore-cli/build.gradle @@ -1,16 +1,17 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.build' dependencies { compileOnly project(":server") - compileOnly project(":libs:elasticsearch-cli") + compileOnly project(":libs:cli") testImplementation project(":test:framework") testImplementation "com.google.jimfs:jimfs:${versions.jimfs}" testRuntimeOnly "com.google.guava:guava:${versions.jimfs_guava}" diff --git a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/AddFileKeyStoreCommand.java b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/AddFileKeyStoreCommand.java index e378747e583db..cc662bd747575 100644 --- a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/AddFileKeyStoreCommand.java +++ b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/AddFileKeyStoreCommand.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli.keystore; diff --git a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/AddStringKeyStoreCommand.java b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/AddStringKeyStoreCommand.java index 0ad35c5512259..c01c18418858a 100644 --- a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/AddStringKeyStoreCommand.java +++ b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/AddStringKeyStoreCommand.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli.keystore; diff --git a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/BaseKeyStoreCommand.java b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/BaseKeyStoreCommand.java index 74ab33a12efcf..0380018d36cff 100644 --- a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/BaseKeyStoreCommand.java +++ b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/BaseKeyStoreCommand.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli.keystore; diff --git a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/ChangeKeyStorePasswordCommand.java b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/ChangeKeyStorePasswordCommand.java index 47b5631ae5d21..4dca3d538263a 100644 --- a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/ChangeKeyStorePasswordCommand.java +++ b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/ChangeKeyStorePasswordCommand.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli.keystore; diff --git a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/CreateKeyStoreCommand.java b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/CreateKeyStoreCommand.java index e615ef76db007..a922c92f5f44b 100644 --- a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/CreateKeyStoreCommand.java +++ b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/CreateKeyStoreCommand.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli.keystore; diff --git a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/HasPasswordKeyStoreCommand.java b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/HasPasswordKeyStoreCommand.java index 4a0502a19ee49..0428d5dcf7df8 100644 --- a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/HasPasswordKeyStoreCommand.java +++ b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/HasPasswordKeyStoreCommand.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli.keystore; diff --git a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/KeyStoreCli.java b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/KeyStoreCli.java index 99778a4750e07..70a607b3a35eb 100644 --- a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/KeyStoreCli.java +++ b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/KeyStoreCli.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli.keystore; diff --git a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/KeyStoreCliProvider.java b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/KeyStoreCliProvider.java index 5d64bc21015ac..b20e92bb634f2 100644 --- a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/KeyStoreCliProvider.java +++ b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/KeyStoreCliProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli.keystore; diff --git a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/ListKeyStoreCommand.java b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/ListKeyStoreCommand.java index 37de590c3b58f..2e36dff9b84fc 100644 --- a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/ListKeyStoreCommand.java +++ b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/ListKeyStoreCommand.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli.keystore; diff --git a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/RemoveSettingKeyStoreCommand.java b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/RemoveSettingKeyStoreCommand.java index a50f4580b370c..8a973c6d67f7d 100644 --- a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/RemoveSettingKeyStoreCommand.java +++ b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/RemoveSettingKeyStoreCommand.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli.keystore; diff --git a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/ShowKeyStoreCommand.java b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/ShowKeyStoreCommand.java index b20041ca6ccc5..ae7090aa0b08c 100644 --- a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/ShowKeyStoreCommand.java +++ b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/ShowKeyStoreCommand.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli.keystore; diff --git a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/UpgradeKeyStoreCommand.java b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/UpgradeKeyStoreCommand.java index 16579c25eab9d..b7061d6153b80 100644 --- a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/UpgradeKeyStoreCommand.java +++ b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/UpgradeKeyStoreCommand.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli.keystore; diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/AddFileKeyStoreCommandTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/AddFileKeyStoreCommandTests.java index 3c83a3215d7b0..edd70e4e52f55 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/AddFileKeyStoreCommandTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/AddFileKeyStoreCommandTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli.keystore; diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/AddStringKeyStoreCommandTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/AddStringKeyStoreCommandTests.java index a1a15e478dd18..3de18e094104f 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/AddStringKeyStoreCommandTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/AddStringKeyStoreCommandTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli.keystore; diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/BootstrapTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/BootstrapTests.java index d4405db422976..0fc76943f9d05 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/BootstrapTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/BootstrapTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli.keystore; diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/ChangeKeyStorePasswordCommandTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/ChangeKeyStorePasswordCommandTests.java index bb13f2744b8e1..45667175b7754 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/ChangeKeyStorePasswordCommandTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/ChangeKeyStorePasswordCommandTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli.keystore; diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/CreateKeyStoreCommandTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/CreateKeyStoreCommandTests.java index 1a708f0a137fa..72a83a48b6344 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/CreateKeyStoreCommandTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/CreateKeyStoreCommandTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli.keystore; diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/HasPasswordKeyStoreCommandTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/HasPasswordKeyStoreCommandTests.java index 3036cc20ce52e..322dede028572 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/HasPasswordKeyStoreCommandTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/HasPasswordKeyStoreCommandTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli.keystore; diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreCommandTestCase.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreCommandTestCase.java index 5dcf290c0e12a..80edce4a20796 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreCommandTestCase.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreCommandTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli.keystore; diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreWrapperTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreWrapperTests.java index 3004494262e6b..38bb7d592f7c0 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreWrapperTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreWrapperTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli.keystore; diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/ListKeyStoreCommandTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/ListKeyStoreCommandTests.java index 1f4f7c8a266e0..fff812824bcd3 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/ListKeyStoreCommandTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/ListKeyStoreCommandTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli.keystore; diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/RemoveSettingKeyStoreCommandTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/RemoveSettingKeyStoreCommandTests.java index 6643530e84d04..f0dce32cd9573 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/RemoveSettingKeyStoreCommandTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/RemoveSettingKeyStoreCommandTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli.keystore; diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/ShowKeyStoreCommandTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/ShowKeyStoreCommandTests.java index bb9a533c354ef..14bb98ebb563e 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/ShowKeyStoreCommandTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/ShowKeyStoreCommandTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli.keystore; diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/UpgradeKeyStoreCommandTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/UpgradeKeyStoreCommandTests.java index 979b118a887e5..bb533f32c7ac2 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/UpgradeKeyStoreCommandTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/UpgradeKeyStoreCommandTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli.keystore; diff --git a/distribution/tools/plugin-cli/build.gradle b/distribution/tools/plugin-cli/build.gradle index c0d2dc0bdb5c7..dc2bcd96b8d9f 100644 --- a/distribution/tools/plugin-cli/build.gradle +++ b/distribution/tools/plugin-cli/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.OS @@ -20,15 +21,15 @@ tasks.named("dependencyLicenses").configure { dependencies { compileOnly project(":server") - compileOnly project(":libs:elasticsearch-cli") - implementation project(":libs:elasticsearch-plugin-api") - implementation project(":libs:elasticsearch-plugin-scanner") + compileOnly project(":libs:cli") + implementation project(":libs:plugin-api") + implementation project(":libs:plugin-scanner") // TODO: asm is picked up from the plugin scanner, we should consolidate so it is not defined twice - implementation 'org.ow2.asm:asm:9.7' - implementation 'org.ow2.asm:asm-tree:9.7' + implementation 'org.ow2.asm:asm:9.7.1' + implementation 'org.ow2.asm:asm-tree:9.7.1' api "org.bouncycastle:bcpg-fips:1.0.7.1" - api "org.bouncycastle:bc-fips:1.0.2.4" + api "org.bouncycastle:bc-fips:1.0.2.5" testImplementation project(":test:framework") testImplementation "com.google.jimfs:jimfs:${versions.jimfs}" testRuntimeOnly "com.google.guava:guava:${versions.jimfs_guava}" diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java index c7bee4a6c172d..d443cf5e1e181 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.cli; diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginCommand.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginCommand.java index 2d6c557179c36..4b8aacccae3ee 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginCommand.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginCommand.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.cli; diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallablePlugin.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallablePlugin.java index 27845baab9126..ff8d1de03aa6e 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallablePlugin.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallablePlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.cli; diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ListPluginsCommand.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ListPluginsCommand.java index 1038dc9bef05c..fc578c81b24c9 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ListPluginsCommand.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ListPluginsCommand.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.cli; diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginCli.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginCli.java index 63d4478b3b52a..9c97b8a991f19 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginCli.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginCli.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.cli; diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginCliProvider.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginCliProvider.java index de87d1a5d0869..caad516174a99 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginCliProvider.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginCliProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.cli; diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginSecurity.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginSecurity.java index 6e762ffd43d3e..376c797d68899 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginSecurity.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginSecurity.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.cli; diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginSyncException.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginSyncException.java index 72e28efc4faab..77129c15803cc 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginSyncException.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginSyncException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.cli; diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginsConfig.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginsConfig.java index 168e5ba3806f3..63d87c9db5b0e 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginsConfig.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginsConfig.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.cli; diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ProgressInputStream.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ProgressInputStream.java index c162c27a896fc..f024575b359c5 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ProgressInputStream.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ProgressInputStream.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.cli; diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ProxyUtils.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ProxyUtils.java index 55e9f2ebc9e84..a870fced21e21 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ProxyUtils.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ProxyUtils.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.cli; diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/RemovePluginAction.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/RemovePluginAction.java index 4714ef202b258..a8f9e746a24e1 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/RemovePluginAction.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/RemovePluginAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.cli; diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/RemovePluginCommand.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/RemovePluginCommand.java index de1c7a7849f5b..db4671d9001ea 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/RemovePluginCommand.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/RemovePluginCommand.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.cli; diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/SyncPluginsAction.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/SyncPluginsAction.java index 75b29fae945eb..d6d0619422770 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/SyncPluginsAction.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/SyncPluginsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.cli; @@ -24,6 +25,7 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Comparator; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Locale; @@ -59,7 +61,7 @@ public SyncPluginsAction(Terminal terminal, Environment env) { * @throws UserException if a plugins config file is found. */ public static void ensureNoConfigFile(Environment env) throws UserException { - final Path pluginsConfig = env.configFile().resolve("elasticsearch-plugins.yml"); + final Path pluginsConfig = env.configFile().resolve(ELASTICSEARCH_PLUGINS_YML); if (Files.exists(pluginsConfig)) { throw new UserException( ExitCodes.USAGE, @@ -206,9 +208,8 @@ private List getPluginsToUpgrade( Optional cachedPluginsConfig, List existingPlugins ) { - final Map cachedPluginIdToLocation = cachedPluginsConfig.map( - config -> config.getPlugins().stream().collect(Collectors.toMap(InstallablePlugin::getId, InstallablePlugin::getLocation)) - ).orElse(Map.of()); + final Map cachedPluginIdToLocation = new HashMap<>(); + cachedPluginsConfig.ifPresent(config -> config.getPlugins().forEach(p -> cachedPluginIdToLocation.put(p.getId(), p.getLocation()))); return pluginsToMaybeUpgrade.stream().filter(eachPlugin -> { final String eachPluginId = eachPlugin.getId(); diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/SyncPluginsCliProvider.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/SyncPluginsCliProvider.java index ac6c0b704ca5b..88b24ab9ae614 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/SyncPluginsCliProvider.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/SyncPluginsCliProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.cli; diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java index 3dc7af07d4d83..d638534943ecd 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.cli; diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ListPluginsCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ListPluginsCommandTests.java index b225bc441794a..0064b8c4bc513 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ListPluginsCommandTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ListPluginsCommandTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.cli; diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/MockInstallPluginCommand.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/MockInstallPluginCommand.java index 17f17624f4bb0..d8fcf01c97950 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/MockInstallPluginCommand.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/MockInstallPluginCommand.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.cli; diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/MockRemovePluginCommand.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/MockRemovePluginCommand.java index 3b0028fbba5ba..bdddab8b8945d 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/MockRemovePluginCommand.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/MockRemovePluginCommand.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.cli; diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/PluginsConfigTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/PluginsConfigTests.java index eb34023e370dc..dda618e9baf0f 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/PluginsConfigTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/PluginsConfigTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.cli; diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ProgressInputStreamTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ProgressInputStreamTests.java index 2184332470f12..cde54230fc7b3 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ProgressInputStreamTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ProgressInputStreamTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.cli; diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ProxyMatcher.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ProxyMatcher.java index 532b75e9af113..fa9de2bc99614 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ProxyMatcher.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ProxyMatcher.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.cli; diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ProxyUtilsTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ProxyUtilsTests.java index a05289362374e..eaefd17b1f07b 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ProxyUtilsTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ProxyUtilsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.cli; diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/RemovePluginActionTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/RemovePluginActionTests.java index 73e89fc948029..aabdd4aaceb9e 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/RemovePluginActionTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/RemovePluginActionTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.cli; diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/SyncPluginsActionTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/SyncPluginsActionTests.java index 9802b4039bb7b..2d2336428a0a5 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/SyncPluginsActionTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/SyncPluginsActionTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.cli; @@ -156,6 +157,22 @@ public void test_getPluginChanges_withOfficialPluginToUpgrade_returnsPluginToUpg assertThat(pluginChanges.upgrade.get(0).getId(), equalTo("analysis-icu")); } + /** + * Check that when there is an official plugin in the config file and in the cached config, then we + * calculate that the plugin does not need to be upgraded. + */ + public void test_getPluginChanges_withOfficialPluginCachedConfigAndNoChanges_returnsNoChanges() throws Exception { + createPlugin("analysis-icu"); + config.setPlugins(List.of(new InstallablePlugin("analysis-icu"))); + + final PluginsConfig cachedConfig = new PluginsConfig(); + cachedConfig.setPlugins(List.of(new InstallablePlugin("analysis-icu"))); + + final PluginChanges pluginChanges = action.getPluginChanges(config, Optional.of(cachedConfig)); + + assertThat(pluginChanges.isEmpty(), is(true)); + } + /** * Check that if an unofficial plugins' location has not changed in the cached config, then we * calculate that the plugin does not need to be upgraded. diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/test_model/ExtensibleInterface.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/test_model/ExtensibleInterface.java index 517c57a036801..c24133b1bfd53 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/test_model/ExtensibleInterface.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/test_model/ExtensibleInterface.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.cli.test_model; diff --git a/distribution/tools/server-cli/build.gradle b/distribution/tools/server-cli/build.gradle index 623f9d40cd49e..299d511ba5dbe 100644 --- a/distribution/tools/server-cli/build.gradle +++ b/distribution/tools/server-cli/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.internal.precommit.CheckForbiddenApisTask @@ -11,7 +12,7 @@ apply plugin: 'elasticsearch.build' dependencies { compileOnly project(":server") - compileOnly project(":libs:elasticsearch-cli") + compileOnly project(":libs:cli") testImplementation project(":test:framework") } diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java index 67f0b571092ff..c3b9768946767 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.server.cli; diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/DefaultSystemMemoryInfo.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/DefaultSystemMemoryInfo.java index 516d1c257d795..c0b999d0926a4 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/DefaultSystemMemoryInfo.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/DefaultSystemMemoryInfo.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.server.cli; diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ErrorPumpThread.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ErrorPumpThread.java index 94c7653a08e0e..2e1d9ea3f2fe2 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ErrorPumpThread.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ErrorPumpThread.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.server.cli; diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmErgonomics.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmErgonomics.java index ec39c0fc89ac2..1160589e43966 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmErgonomics.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmErgonomics.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.server.cli; diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOption.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOption.java index 60cbcb86c02b9..0fa3e9463fc48 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOption.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOption.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.server.cli; diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java index 0bfa0f211807d..d2ac1549efd1a 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.server.cli; diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/KeyStoreLoader.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/KeyStoreLoader.java index 6741e95e98dc5..9430cb598cf02 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/KeyStoreLoader.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/KeyStoreLoader.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.server.cli; diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/KeystorePasswordTerminal.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/KeystorePasswordTerminal.java index 0fddf76caff59..85bbc79a6a29d 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/KeystorePasswordTerminal.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/KeystorePasswordTerminal.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.server.cli; diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/MachineDependentHeap.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/MachineDependentHeap.java index 693aa781a54b0..26fd7294ed557 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/MachineDependentHeap.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/MachineDependentHeap.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.server.cli; diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/OverridableSystemMemoryInfo.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/OverridableSystemMemoryInfo.java index 245b252bdedb4..24dcac6013604 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/OverridableSystemMemoryInfo.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/OverridableSystemMemoryInfo.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.server.cli; diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ProcessUtil.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ProcessUtil.java index fb5f7bae34844..9564bdd8aa557 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ProcessUtil.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ProcessUtil.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.server.cli; diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SecureSettingsLoader.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SecureSettingsLoader.java index 98fc47f4e64bc..75fc44e23698f 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SecureSettingsLoader.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SecureSettingsLoader.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.server.cli; diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java index bea7fbb7f63e8..22b62972befe4 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.server.cli; diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCliProvider.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCliProvider.java index ae9b46a39df51..7a9ef63b5b8bf 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCliProvider.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCliProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.server.cli; diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcess.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcess.java index 35b5d93b39933..89377b5b3f8bb 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcess.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcess.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.server.cli; diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessBuilder.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessBuilder.java index fcc290ebe9e72..293c4af3270b9 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessBuilder.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.server.cli; diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessUtils.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessUtils.java index ebbc68b1be90b..47d750f75612c 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessUtils.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessUtils.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.server.cli; diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java index 2d707f150cc8b..5f4beb0e7c9a0 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.server.cli; @@ -12,9 +13,11 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.UpdateForV9; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import java.util.stream.Stream; final class SystemJvmOptions { @@ -22,8 +25,8 @@ final class SystemJvmOptions { static List systemJvmOptions(Settings nodeSettings, final Map sysprops) { String distroType = sysprops.get("es.distribution.type"); boolean isHotspot = sysprops.getOrDefault("sun.management.compiler", "").contains("HotSpot"); - - return Stream.concat( + boolean useEntitlements = Boolean.parseBoolean(sysprops.getOrDefault("es.entitlements.enabled", "false")); + return Stream.of( Stream.of( /* * Cache ttl in seconds for positive DNS lookups noting that this overrides the JDK security property @@ -35,8 +38,6 @@ static List systemJvmOptions(Settings nodeSettings, final Map systemJvmOptions(Settings nodeSettings, final Map= 23 ? "CLDR" : "SPI,COMPAT"; } /* @@ -96,42 +99,42 @@ private static String getLocaleProviders() { * that cgroup statistics are available for the container this process * will run in. */ - private static String maybeOverrideDockerCgroup(String distroType) { + private static Stream maybeOverrideDockerCgroup(String distroType) { if ("docker".equals(distroType)) { - return "-Des.cgroups.hierarchy.override=/"; + return Stream.of("-Des.cgroups.hierarchy.override=/"); } - return ""; + return Stream.empty(); } - private static String setReplayFile(String distroType, boolean isHotspot) { + private static Stream maybeSetReplayFile(String distroType, boolean isHotspot) { if (isHotspot == false) { // the replay file option is only guaranteed for hotspot vms - return ""; + return Stream.empty(); } String replayDir = "logs"; if ("rpm".equals(distroType) || "deb".equals(distroType)) { replayDir = "/var/log/elasticsearch"; } - return "-XX:ReplayDataFile=" + replayDir + "/replay_pid%p.log"; + return Stream.of("-XX:ReplayDataFile=" + replayDir + "/replay_pid%p.log"); } /* * node.processors determines thread pool sizes for Elasticsearch. When it * is set, we need to also tell the JVM to respect a different value */ - private static String maybeSetActiveProcessorCount(Settings nodeSettings) { + private static Stream maybeSetActiveProcessorCount(Settings nodeSettings) { if (EsExecutors.NODE_PROCESSORS_SETTING.exists(nodeSettings)) { int allocated = EsExecutors.allocatedProcessors(nodeSettings); - return "-XX:ActiveProcessorCount=" + allocated; + return Stream.of("-XX:ActiveProcessorCount=" + allocated); } - return ""; + return Stream.empty(); } - private static String maybeEnableNativeAccess() { + private static Stream maybeEnableNativeAccess() { if (Runtime.version().feature() >= 21) { - return "--enable-native-access=org.elasticsearch.nativeaccess,org.apache.lucene.core"; + return Stream.of("--enable-native-access=org.elasticsearch.nativeaccess,org.apache.lucene.core"); } - return ""; + return Stream.empty(); } /* @@ -144,4 +147,41 @@ private static Stream maybeWorkaroundG1Bug() { } return Stream.of(); } + + private static Stream maybeAllowSecurityManager() { + // Will become conditional on useEntitlements once entitlements can run without SM + return Stream.of("-Djava.security.manager=allow"); + } + + private static Stream maybeAttachEntitlementAgent(boolean useEntitlements) { + if (useEntitlements == false) { + return Stream.empty(); + } + + Path dir = Path.of("lib", "entitlement-bridge"); + if (Files.exists(dir) == false) { + throw new IllegalStateException("Directory for entitlement bridge jar does not exist: " + dir); + } + String bridgeJar; + try (var s = Files.list(dir)) { + var candidates = s.limit(2).toList(); + if (candidates.size() != 1) { + throw new IllegalStateException("Expected one jar in " + dir + "; found " + candidates.size()); + } + bridgeJar = candidates.get(0).toString(); + } catch (IOException e) { + throw new IllegalStateException("Failed to list entitlement jars in: " + dir, e); + } + // We instrument classes in these modules to call the bridge. Because the bridge gets patched + // into java.base, we must export the bridge from java.base to these modules. + String modulesContainingEntitlementInstrumentation = "java.logging"; + return Stream.of( + "-Des.entitlements.enabled=true", + "-XX:+EnableDynamicAgentLoading", + "-Djdk.attach.allowAttachSelf=true", + "--patch-module=java.base=" + bridgeJar, + "--add-exports=java.base/org.elasticsearch.entitlement.bridge=org.elasticsearch.entitlement," + + modulesContainingEntitlementInstrumentation + ); + } } diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemMemoryInfo.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemMemoryInfo.java index 6133c0e937f42..78431628b6612 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemMemoryInfo.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemMemoryInfo.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.server.cli; diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java index e8a8d3ee8df77..a7ba8eb11fbcc 100644 --- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java +++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.server.cli; diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmErgonomicsTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmErgonomicsTests.java index a8a2ddbe14aab..900eab4de1bf2 100644 --- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmErgonomicsTests.java +++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmErgonomicsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.server.cli; diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java index fc889f036a795..b8f4641f9b757 100644 --- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java +++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.server.cli; diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/MachineDependentHeapTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/MachineDependentHeapTests.java index aeb0b98e0be29..e2dc9f32647e0 100644 --- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/MachineDependentHeapTests.java +++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/MachineDependentHeapTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.server.cli; diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/OverridableSystemMemoryInfoTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/OverridableSystemMemoryInfoTests.java index 2db6d36749db5..fb0a3fc630c7b 100644 --- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/OverridableSystemMemoryInfoTests.java +++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/OverridableSystemMemoryInfoTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.server.cli; diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java index e603790051c0c..bacc89548c9a1 100644 --- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java +++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.server.cli; diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessTests.java index dc36485fb77ab..d4c05937f35b7 100644 --- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessTests.java +++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.server.cli; diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessUtilsTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessUtilsTests.java index 8cd1b63e41b03..1ecc33b9420ee 100644 --- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessUtilsTests.java +++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessUtilsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.server.cli; diff --git a/distribution/tools/windows-service-cli/build.gradle b/distribution/tools/windows-service-cli/build.gradle index 77da0d407a40d..dcfaf244b7eec 100644 --- a/distribution/tools/windows-service-cli/build.gradle +++ b/distribution/tools/windows-service-cli/build.gradle @@ -2,7 +2,7 @@ apply plugin: 'elasticsearch.build' dependencies { compileOnly project(":server") - compileOnly project(":libs:elasticsearch-cli") + compileOnly project(":libs:cli") compileOnly project(":distribution:tools:server-cli") testImplementation project(":test:framework") diff --git a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/ProcrunCommand.java b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/ProcrunCommand.java index e32e34fb24400..49552400a2c93 100644 --- a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/ProcrunCommand.java +++ b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/ProcrunCommand.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.windows.service; diff --git a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceCli.java b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceCli.java index 6f9a7e6b2169f..65a525bddbd2b 100644 --- a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceCli.java +++ b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceCli.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.windows.service; diff --git a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceCliProvider.java b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceCliProvider.java index 1b1f8d08c9805..e61bff432c03e 100644 --- a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceCliProvider.java +++ b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceCliProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.windows.service; diff --git a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemon.java b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemon.java index 66ae78470c55d..66ee712fcce95 100644 --- a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemon.java +++ b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemon.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.windows.service; diff --git a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemonProvider.java b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemonProvider.java index a07883dcffcec..f13424858055f 100644 --- a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemonProvider.java +++ b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemonProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.windows.service; diff --git a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceInstallCommand.java b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceInstallCommand.java index 2df889be4a681..75f508ad21499 100644 --- a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceInstallCommand.java +++ b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceInstallCommand.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.windows.service; diff --git a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceManagerCommand.java b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceManagerCommand.java index dc9906feb6e3e..97df7aabbd716 100644 --- a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceManagerCommand.java +++ b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceManagerCommand.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.windows.service; diff --git a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceRemoveCommand.java b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceRemoveCommand.java index 60dd0e2a85220..973b4f71cae99 100644 --- a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceRemoveCommand.java +++ b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceRemoveCommand.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.windows.service; diff --git a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceStartCommand.java b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceStartCommand.java index 7f958beecf588..af1722ab789e6 100644 --- a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceStartCommand.java +++ b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceStartCommand.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.windows.service; diff --git a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceStopCommand.java b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceStopCommand.java index 6d281fcfa8e97..b2fd9e78f35e6 100644 --- a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceStopCommand.java +++ b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceStopCommand.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.windows.service; diff --git a/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/ProcrunCommandTests.java b/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/ProcrunCommandTests.java index 8f44eaa80f23a..c4f746df16652 100644 --- a/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/ProcrunCommandTests.java +++ b/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/ProcrunCommandTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.windows.service; diff --git a/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceCliTestCase.java b/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceCliTestCase.java index 808173005b96f..41b9f75b785c4 100644 --- a/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceCliTestCase.java +++ b/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceCliTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.windows.service; diff --git a/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceInstallCommandTests.java b/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceInstallCommandTests.java index 5103d39c7311f..4d3d6400c55c4 100644 --- a/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceInstallCommandTests.java +++ b/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceInstallCommandTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.windows.service; diff --git a/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceManagerCommandTests.java b/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceManagerCommandTests.java index 1699dd3f78316..20b9812e465b4 100644 --- a/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceManagerCommandTests.java +++ b/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceManagerCommandTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.windows.service; diff --git a/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceRemoveCommandTests.java b/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceRemoveCommandTests.java index d0e72e9de5c66..8b0853e2897b2 100644 --- a/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceRemoveCommandTests.java +++ b/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceRemoveCommandTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.windows.service; diff --git a/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceStartCommandTests.java b/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceStartCommandTests.java index 502008d22422f..0b6b45548c816 100644 --- a/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceStartCommandTests.java +++ b/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceStartCommandTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.windows.service; diff --git a/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceStopCommandTests.java b/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceStopCommandTests.java index a36e090bd7ac4..d06a5695daa31 100644 --- a/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceStopCommandTests.java +++ b/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceStopCommandTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.windows.service; diff --git a/docs/Versions.asciidoc b/docs/Versions.asciidoc index fb99ef498df17..7cb61b159b6fe 100644 --- a/docs/Versions.asciidoc +++ b/docs/Versions.asciidoc @@ -1,14 +1,15 @@ include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[] -:lucene_version: 9.11.1 -:lucene_version_path: 9_11_1 +:lucene_version: 9.12.1 +:lucene_version_path: 9_12_1 :jdk: 11.0.2 :jdk_major: 11 :build_type: tar :docker-repo: docker.elastic.co/elasticsearch/elasticsearch :docker-image: {docker-repo}:{version} +:docker-wolfi-image: {docker-repo}-wolfi:{version} :kib-docker-repo: docker.elastic.co/kibana/kibana :kib-docker-image: {kib-docker-repo}:{version} :plugin_url: https://artifacts.elastic.co/downloads/elasticsearch-plugins diff --git a/docs/build.gradle b/docs/build.gradle index 99453b840b0d2..43b61ea97c089 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -1,14 +1,14 @@ import org.elasticsearch.gradle.Version -import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.internal.doc.DocSnippetTask import static org.elasticsearch.gradle.testclusters.TestDistribution.DEFAULT /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.docs-test' @@ -28,7 +28,7 @@ ext.docsFileTree = fileTree(projectDir) { // These files simply don't pass yet. We should figure out how to fix them. exclude 'reference/watcher/reference/actions.asciidoc' exclude 'reference/rest-api/security/ssl.asciidoc' - if (BuildParams.inFipsJvm) { + if (buildParams.inFipsJvm) { // We don't support this component in FIPS 140 exclude 'reference/ingest/processors/attachment.asciidoc' // We can't conditionally control output, this would be missing the ingest-attachment component @@ -37,7 +37,7 @@ ext.docsFileTree = fileTree(projectDir) { } tasks.named("yamlRestTest") { - if (BuildParams.isSnapshotBuild() == false) { + if (buildParams.isSnapshotBuild() == false) { // LOOKUP is not available in snapshots systemProperty 'tests.rest.blacklist', [ "reference/esql/processing-commands/lookup/esql-lookup-example" @@ -82,7 +82,7 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach { setting 'xpack.license.self_generated.type', 'trial' setting 'indices.lifecycle.history_index_enabled', 'false' keystorePassword 'keystore-password' - if (BuildParams.isSnapshotBuild() == false) { + if (buildParams.isSnapshotBuild() == false) { requiresFeature 'es.failure_store_feature_flag_enabled', new Version(8, 12, 0) } } @@ -130,8 +130,9 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach { setting 'xpack.security.enabled', 'true' setting 'xpack.security.authc.api_key.enabled', 'true' setting 'xpack.security.authc.token.enabled', 'true' - // disable the ILM history for doc tests to avoid potential lingering tasks that'd cause test flakiness + // disable the ILM and SLM history for doc tests to avoid potential lingering tasks that'd cause test flakiness setting 'indices.lifecycle.history_index_enabled', 'false' + setting 'slm.history_index_enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.authc.realms.file.file.order', '0' setting 'xpack.security.authc.realms.native.native.order', '1' @@ -169,7 +170,7 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach { return } // Do not install ingest-attachment in a FIPS 140 JVM as this is not supported - if (subproj.path.startsWith(':modules:ingest-attachment') && BuildParams.inFipsJvm) { + if (subproj.path.startsWith(':modules:ingest-attachment') && buildParams.inFipsJvm) { return } plugin subproj.path @@ -177,8 +178,9 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach { } tasks.named("yamlRestTest").configure { + def repoFolder = "${layout.buildDirectory.asFile.get()}/cluster/shared/repo" doFirst { - delete("${buildDir}/cluster/shared/repo") + delete(repoFolder) } } diff --git a/docs/changelog/104125.yaml b/docs/changelog/104125.yaml new file mode 100644 index 0000000000000..e5c5ea6a3f1cd --- /dev/null +++ b/docs/changelog/104125.yaml @@ -0,0 +1,18 @@ +pr: 104125 +summary: Disable machine learning on macOS x86_64 +area: Machine Learning +type: breaking +issues: [] +breaking: + title: Disable machine learning on macOS x86_64 + area: Packaging + details: The machine learning plugin is permanently disabled on macOS x86_64. + For the last three years Apple has been selling hardware based on the arm64 + architecture, and support will increasingly focus on this architecture in + the future. Changes to upstream dependencies of Elastic's machine learning + functionality have made it unviable for Elastic to continue to build machine + learning on macOS x86_64. + impact: To continue to use machine learning functionality on macOS please switch to + an arm64 machine (Apple silicon). Alternatively, it will still be possible to run + Elasticsearch with machine learning enabled in a Docker container on macOS x86_64. + notable: false diff --git a/docs/changelog/106520.yaml b/docs/changelog/106520.yaml deleted file mode 100644 index c3fe69a4c3dbd..0000000000000 --- a/docs/changelog/106520.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 106520 -summary: Updated the transport CA name in Security Auto-Configuration. -area: Security -type: bug -issues: - - 106455 diff --git a/docs/changelog/107047.yaml b/docs/changelog/107047.yaml deleted file mode 100644 index 89caed6f55074..0000000000000 --- a/docs/changelog/107047.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 107047 -summary: "Search/Mapping: KnnVectorQueryBuilder support for allowUnmappedFields" -area: Search -type: bug -issues: - - 106846 diff --git a/docs/changelog/109017.yaml b/docs/changelog/109017.yaml deleted file mode 100644 index 80bcdd6fc0e25..0000000000000 --- a/docs/changelog/109017.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 109017 -summary: "ESQL: Add `MV_PSERIES_WEIGHTED_SUM` for score calculations used by security\ - \ solution" -area: ES|QL -type: "feature" -issues: [ ] diff --git a/docs/changelog/109193.yaml b/docs/changelog/109193.yaml deleted file mode 100644 index 5cc664eaee2cd..0000000000000 --- a/docs/changelog/109193.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 109193 -summary: "[ES|QL] explicit cast a string literal to `date_period` and `time_duration`\ - \ in arithmetic operations" -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/109414.yaml b/docs/changelog/109414.yaml deleted file mode 100644 index 81b7541bde35b..0000000000000 --- a/docs/changelog/109414.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 109414 -summary: Don't fail retention lease sync actions due to capacity constraints -area: CRUD -type: bug -issues: - - 105926 diff --git a/docs/changelog/109583.yaml b/docs/changelog/109583.yaml deleted file mode 100644 index 84757e307b4fb..0000000000000 --- a/docs/changelog/109583.yaml +++ /dev/null @@ -1,29 +0,0 @@ -pr: 109583 -summary: "ESQL: INLINESTATS" -area: ES|QL -type: feature -issues: - - 107589 -highlight: - title: "ESQL: INLINESTATS" - body: |- - This adds the `INLINESTATS` command to ESQL which performs a STATS and - then enriches the results into the output stream. So, this query: - - [source,esql] - ---- - FROM test - | INLINESTATS m=MAX(a * b) BY b - | WHERE m == a * b - | SORT a DESC, b DESC - | LIMIT 3 - ---- - - Produces output like: - - | a | b | m | - | --- | --- | ----- | - | 99 | 999 | 98901 | - | 99 | 998 | 98802 | - | 99 | 997 | 98703 | - notable: true diff --git a/docs/changelog/109667.yaml b/docs/changelog/109667.yaml deleted file mode 100644 index 782a1b1cf6c9b..0000000000000 --- a/docs/changelog/109667.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109667 -summary: Inference autoscaling -area: Machine Learning -type: feature -issues: [] diff --git a/docs/changelog/109684.yaml b/docs/changelog/109684.yaml deleted file mode 100644 index 156f568290cf5..0000000000000 --- a/docs/changelog/109684.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109684 -summary: Avoid `ModelAssignment` deadlock -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/110021.yaml b/docs/changelog/110021.yaml deleted file mode 100644 index 51878b960dfd0..0000000000000 --- a/docs/changelog/110021.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 110021 -summary: "[ES|QL] validate `mv_sort` order" -area: ES|QL -type: bug -issues: - - 109910 diff --git a/docs/changelog/110116.yaml b/docs/changelog/110116.yaml deleted file mode 100644 index 9c309b8b80311..0000000000000 --- a/docs/changelog/110116.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110116 -summary: "[ESQL] Make query wrapped by `SingleValueQuery` cacheable" -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/110216.yaml b/docs/changelog/110216.yaml deleted file mode 100644 index 00ab20b230e2c..0000000000000 --- a/docs/changelog/110216.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110216 -summary: Register SLM run before snapshotting to save stats -area: ILM+SLM -type: enhancement -issues: [] diff --git a/docs/changelog/110237.yaml b/docs/changelog/110237.yaml deleted file mode 100644 index 076855385376c..0000000000000 --- a/docs/changelog/110237.yaml +++ /dev/null @@ -1,7 +0,0 @@ -pr: 110237 -summary: Optimize the loop processing of URL decoding -area: Infra/REST API -type: enhancement -issues: - - 110235 - diff --git a/docs/changelog/110399.yaml b/docs/changelog/110399.yaml deleted file mode 100644 index 9e04e2656809e..0000000000000 --- a/docs/changelog/110399.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 110399 -summary: "[Inference API] Prevent inference endpoints from being deleted if they are\ - \ referenced by semantic text" -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/110427.yaml b/docs/changelog/110427.yaml deleted file mode 100644 index ba8a1246e90e4..0000000000000 --- a/docs/changelog/110427.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 110427 -summary: "[Inference API] Remove unused Cohere rerank service settings fields in a\ - \ BWC way" -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/110520.yaml b/docs/changelog/110520.yaml deleted file mode 100644 index fba4b84e2279e..0000000000000 --- a/docs/changelog/110520.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110520 -summary: Add protection for OOM during aggregations partial reduction -area: Aggregations -type: enhancement -issues: [] diff --git a/docs/changelog/110524.yaml b/docs/changelog/110524.yaml deleted file mode 100644 index 6274c99b09998..0000000000000 --- a/docs/changelog/110524.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110524 -summary: Introduce mode `subobjects=auto` for objects -area: Mapping -type: enhancement -issues: [] diff --git a/docs/changelog/110527.yaml b/docs/changelog/110527.yaml deleted file mode 100644 index 3ab19ecaaaa76..0000000000000 --- a/docs/changelog/110527.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110527 -summary: "ESQL: Add boolean support to Max and Min aggs" -area: ES|QL -type: feature -issues: [] diff --git a/docs/changelog/110554.yaml b/docs/changelog/110554.yaml deleted file mode 100644 index 8c0b896a4c979..0000000000000 --- a/docs/changelog/110554.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110554 -summary: Fix `MapperBuilderContext#isDataStream` when used in dynamic mappers -area: "Mapping" -type: bug -issues: [] diff --git a/docs/changelog/110574.yaml b/docs/changelog/110574.yaml deleted file mode 100644 index 1840838500151..0000000000000 --- a/docs/changelog/110574.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 110574 -summary: "ES|QL: better validation for GROK patterns" -area: ES|QL -type: bug -issues: - - 110533 diff --git a/docs/changelog/110578.yaml b/docs/changelog/110578.yaml deleted file mode 100644 index 5d48171e4f328..0000000000000 --- a/docs/changelog/110578.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110578 -summary: Add `size_in_bytes` to enrich cache stats -area: Ingest Node -type: enhancement -issues: [] diff --git a/docs/changelog/110593.yaml b/docs/changelog/110593.yaml deleted file mode 100644 index 21a5d426ceb46..0000000000000 --- a/docs/changelog/110593.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 110593 -summary: "[ES|QL] add tests for stats by constant" -area: ES|QL -type: bug -issues: - - 105383 diff --git a/docs/changelog/110603.yaml b/docs/changelog/110603.yaml deleted file mode 100644 index 4ba19985853df..0000000000000 --- a/docs/changelog/110603.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 110603 -summary: Stop iterating over all fields to extract @timestamp value -area: TSDB -type: enhancement -issues: - - 92297 diff --git a/docs/changelog/110606.yaml b/docs/changelog/110606.yaml deleted file mode 100644 index d4ab5234289c4..0000000000000 --- a/docs/changelog/110606.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110606 -summary: Adding mapping validation to the simulate ingest API -area: Ingest Node -type: enhancement -issues: [] diff --git a/docs/changelog/110630.yaml b/docs/changelog/110630.yaml deleted file mode 100644 index 9bf78e1209753..0000000000000 --- a/docs/changelog/110630.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110630 -summary: Telemetry for inference adaptive allocations -area: Machine Learning -type: feature -issues: [] diff --git a/docs/changelog/110633.yaml b/docs/changelog/110633.yaml deleted file mode 100644 index d4d1dc68cdbcc..0000000000000 --- a/docs/changelog/110633.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110633 -summary: Add manage roles privilege -area: Authorization -type: enhancement -issues: [] diff --git a/docs/changelog/110669.yaml b/docs/changelog/110669.yaml deleted file mode 100644 index 301e756ca373c..0000000000000 --- a/docs/changelog/110669.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 110669 -summary: "[ES|QL] Use `RangeQuery` and String in `BinaryComparison` on datetime fields" -area: ES|QL -type: bug -issues: - - 107900 diff --git a/docs/changelog/110676.yaml b/docs/changelog/110676.yaml deleted file mode 100644 index efe7e0e55f18f..0000000000000 --- a/docs/changelog/110676.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110676 -summary: Allow querying `index_mode` -area: Mapping -type: enhancement -issues: [] diff --git a/docs/changelog/110677.yaml b/docs/changelog/110677.yaml deleted file mode 100644 index 72fe5129f3b9d..0000000000000 --- a/docs/changelog/110677.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110677 -summary: Add validation for synthetic source mode in logs mode indices -area: Logs -type: enhancement -issues: [] diff --git a/docs/changelog/110718.yaml b/docs/changelog/110718.yaml deleted file mode 100644 index 526083a8add0c..0000000000000 --- a/docs/changelog/110718.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110718 -summary: "ESQL: Add boolean support to TOP aggregation" -area: ES|QL -type: feature -issues: [] diff --git a/docs/changelog/110734.yaml b/docs/changelog/110734.yaml deleted file mode 100644 index d6dce144b89cd..0000000000000 --- a/docs/changelog/110734.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110734 -summary: Fix bug in ML serverless autoscaling which prevented trained model updates from triggering a scale up -area: Machine Learning -type: bug -issues: [ ] diff --git a/docs/changelog/110796.yaml b/docs/changelog/110796.yaml deleted file mode 100644 index a54a9a08bbd27..0000000000000 --- a/docs/changelog/110796.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110796 -summary: Remove needless forking to GENERIC in `TransportMultiSearchAction` -area: Search -type: bug -issues: [] diff --git a/docs/changelog/110816.yaml b/docs/changelog/110816.yaml deleted file mode 100644 index bf707376ec9ea..0000000000000 --- a/docs/changelog/110816.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 110816 -summary: GET _cluster/settings with include_defaults returns the expected fallback value if defined in elasticsearch.yml -area: Infra/Settings -type: bug -issues: - - 110815 diff --git a/docs/changelog/110829.yaml b/docs/changelog/110829.yaml deleted file mode 100644 index 365a14436ec89..0000000000000 --- a/docs/changelog/110829.yaml +++ /dev/null @@ -1,10 +0,0 @@ -pr: 110829 -summary: deprecate `edge_ngram` side parameter -area: Analysis -type: deprecation -issues: [] -deprecation: - title: deprecate `edge_ngram` side parameter - area: Analysis - details: edge_ngram will no longer accept the side parameter. - impact: Users will need to update any usage of edge_ngram token filter that utilizes `side`. If the `back` value was used, they can achieve the same behavior by using the `reverse` token filter. diff --git a/docs/changelog/110833.yaml b/docs/changelog/110833.yaml deleted file mode 100644 index 008fc489ed731..0000000000000 --- a/docs/changelog/110833.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110833 -summary: Make empty string searches be consistent with case (in)sensitivity -area: Search -type: bug -issues: [] diff --git a/docs/changelog/110846.yaml b/docs/changelog/110846.yaml deleted file mode 100644 index 56cc65e83648c..0000000000000 --- a/docs/changelog/110846.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110846 -summary: Fix MLTQuery handling of custom term frequencies -area: Ranking -type: bug -issues: [] diff --git a/docs/changelog/110847.yaml b/docs/changelog/110847.yaml deleted file mode 100644 index 214adc97ac7cb..0000000000000 --- a/docs/changelog/110847.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110847 -summary: SLM Interval based scheduling -area: ILM+SLM -type: feature -issues: [] diff --git a/docs/changelog/110860.yaml b/docs/changelog/110860.yaml deleted file mode 100644 index 5649ca4c88362..0000000000000 --- a/docs/changelog/110860.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110860 -summary: Speedup `CanMatchPreFilterSearchPhase` constructor -area: Search -type: bug -issues: [] diff --git a/docs/changelog/110879.yaml b/docs/changelog/110879.yaml deleted file mode 100644 index d114c6c2aa472..0000000000000 --- a/docs/changelog/110879.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110879 -summary: Add EXP ES|QL function -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/110901.yaml b/docs/changelog/110901.yaml deleted file mode 100644 index 599cb7ce9ec98..0000000000000 --- a/docs/changelog/110901.yaml +++ /dev/null @@ -1,15 +0,0 @@ -pr: 110901 -summary: Set lenient to true by default when using updateable synonyms -area: Analysis -type: breaking -issues: [] -breaking: - title: Set lenient to true by default when using updateable synonyms - area: Analysis - details: | - When a `synonym` or `synonym_graph` token filter is configured with `updateable: true`, the default `lenient` - value will now be `true`. - impact: | - `synonym` or `synonym_graph` token filters configured with `updateable: true` will ignore invalid synonyms by - default. This prevents shard initialization errors on invalid synonyms. - notable: true diff --git a/docs/changelog/110921.yaml b/docs/changelog/110921.yaml deleted file mode 100644 index 28cd569404945..0000000000000 --- a/docs/changelog/110921.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110921 -summary: "ESQL: Support IP fields in MAX and MIN aggregations" -area: ES|QL -type: feature -issues: [] diff --git a/docs/changelog/110928.yaml b/docs/changelog/110928.yaml deleted file mode 100644 index dcb2df6e6cca9..0000000000000 --- a/docs/changelog/110928.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110928 -summary: Dense vector field types updatable for int4 -area: Vector Search -type: enhancement -issues: [] diff --git a/docs/changelog/110951.yaml b/docs/changelog/110951.yaml deleted file mode 100644 index ec8bc9cae6347..0000000000000 --- a/docs/changelog/110951.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110951 -summary: Allow task canceling of validate API calls -area: Transform -type: bug -issues: [] diff --git a/docs/changelog/110971.yaml b/docs/changelog/110971.yaml deleted file mode 100644 index 3579f77dc0d1d..0000000000000 --- a/docs/changelog/110971.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110971 -summary: "Search in ES|QL: Add MATCH operator" -area: ES|QL -type: feature -issues: [] diff --git a/docs/changelog/110974.yaml b/docs/changelog/110974.yaml deleted file mode 100644 index c9e8c9b78675e..0000000000000 --- a/docs/changelog/110974.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110974 -summary: Add custom rule parameters to force time shift -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/110986.yaml b/docs/changelog/110986.yaml deleted file mode 100644 index 4e320b19c9578..0000000000000 --- a/docs/changelog/110986.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 110986 -summary: Fix unnecessary mustache template evaluation -area: Ingest Node -type: enhancement -issues: - - 110191 diff --git a/docs/changelog/110993.yaml b/docs/changelog/110993.yaml deleted file mode 100644 index 9eb653a09e3a4..0000000000000 --- a/docs/changelog/110993.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 110993 -summary: Add link to Max Shards Per Node exception message -area: Distributed -type: enhancement -issues: [] diff --git a/docs/changelog/111015.yaml b/docs/changelog/111015.yaml deleted file mode 100644 index 3cc363c8bbf6b..0000000000000 --- a/docs/changelog/111015.yaml +++ /dev/null @@ -1,15 +0,0 @@ -pr: 111015 -summary: Always allow rebalancing by default -area: Allocation -type: enhancement -issues: [] -highlight: - title: Always allow rebalancing by default - body: |- - In earlier versions of {es} the `cluster.routing.allocation.allow_rebalance` setting defaults to - `indices_all_active` which blocks all rebalancing moves while the cluster is in `yellow` or `red` health. This was - appropriate for the legacy allocator which might do too many rebalancing moves otherwise. Today's allocator has - better support for rebalancing a cluster that is not in `green` health, and expects to be able to rebalance some - shards away from over-full nodes to avoid allocating shards to undesirable locations in the first place. From - version 8.16 `allow_rebalance` setting defaults to `always` unless the legacy allocator is explicitly enabled. - notable: true diff --git a/docs/changelog/111064.yaml b/docs/changelog/111064.yaml deleted file mode 100644 index 848da842b090e..0000000000000 --- a/docs/changelog/111064.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111064 -summary: "ESQL: Fix Double operations returning infinite" -area: ES|QL -type: bug -issues: - - 111026 diff --git a/docs/changelog/111071.yaml b/docs/changelog/111071.yaml deleted file mode 100644 index 5e8ab53db3d03..0000000000000 --- a/docs/changelog/111071.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111071 -summary: Use native scalar scorer for int8_flat index -area: Vector Search -type: enhancement -issues: [] diff --git a/docs/changelog/111079.yaml b/docs/changelog/111079.yaml deleted file mode 100644 index aac22005f912d..0000000000000 --- a/docs/changelog/111079.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111079 -summary: PUT slm policy should only increase version if actually changed -area: ILM+SLM -type: enhancement -issues: [] diff --git a/docs/changelog/111091.yaml b/docs/changelog/111091.yaml deleted file mode 100644 index 8444681a14a48..0000000000000 --- a/docs/changelog/111091.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111091 -summary: "X-pack/plugin/otel: introduce x-pack-otel plugin" -area: Data streams -type: feature -issues: [] diff --git a/docs/changelog/111104.yaml b/docs/changelog/111104.yaml new file mode 100644 index 0000000000000..a7dffdd0be221 --- /dev/null +++ b/docs/changelog/111104.yaml @@ -0,0 +1,6 @@ +pr: 111104 +summary: "ESQL: Enable async get to support formatting" +area: ES|QL +type: feature +issues: + - 110926 diff --git a/docs/changelog/111105.yaml b/docs/changelog/111105.yaml deleted file mode 100644 index ed32bd1ef7fc3..0000000000000 --- a/docs/changelog/111105.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111105 -summary: "ESQL: TOP aggregation IP support" -area: ES|QL -type: feature -issues: [] diff --git a/docs/changelog/111118.yaml b/docs/changelog/111118.yaml deleted file mode 100644 index c9fe6cb443688..0000000000000 --- a/docs/changelog/111118.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111118 -summary: "[ES|QL] Simplify patterns for subfields" -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/111123.yaml b/docs/changelog/111123.yaml deleted file mode 100644 index 605b8607f4082..0000000000000 --- a/docs/changelog/111123.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111123 -summary: Add Lucene segment-level fields stats -area: Mapping -type: enhancement -issues: [] diff --git a/docs/changelog/111154.yaml b/docs/changelog/111154.yaml deleted file mode 100644 index 3297f5005a811..0000000000000 --- a/docs/changelog/111154.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111154 -summary: EIS integration -area: Inference -type: feature -issues: [] diff --git a/docs/changelog/111161.yaml b/docs/changelog/111161.yaml deleted file mode 100644 index c081d555ff1ee..0000000000000 --- a/docs/changelog/111161.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111161 -summary: Add support for templates when validating mappings in the simulate ingest - API -area: Ingest Node -type: enhancement -issues: [] diff --git a/docs/changelog/111181.yaml b/docs/changelog/111181.yaml deleted file mode 100644 index 7f9f5937b7652..0000000000000 --- a/docs/changelog/111181.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111181 -summary: "[Inference API] Add Alibaba Cloud AI Search Model support to Inference API" -area: Machine Learning -type: enhancement -issues: [ ] diff --git a/docs/changelog/111193.yaml b/docs/changelog/111193.yaml deleted file mode 100644 index 9e56facb60d3a..0000000000000 --- a/docs/changelog/111193.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111193 -summary: Fix cases of collections with one point -area: Geo -type: bug -issues: - - 110982 diff --git a/docs/changelog/111212.yaml b/docs/changelog/111212.yaml deleted file mode 100644 index 67d1513b3ff6f..0000000000000 --- a/docs/changelog/111212.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111212 -summary: Fix score count validation in reranker response -area: Ranking -type: bug -issues: - - 111202 diff --git a/docs/changelog/111215.yaml b/docs/changelog/111215.yaml deleted file mode 100644 index dc044c2283fc4..0000000000000 --- a/docs/changelog/111215.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111215 -summary: Make `SnapshotLifecycleStats` immutable so `SnapshotLifecycleMetadata.EMPTY` - isn't changed as side-effect -area: ILM+SLM -type: bug -issues: [] diff --git a/docs/changelog/111225.yaml b/docs/changelog/111225.yaml deleted file mode 100644 index bcd344847cfd2..0000000000000 --- a/docs/changelog/111225.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111225 -summary: Upgrade Azure SDK -area: Snapshot/Restore -type: upgrade -issues: [] diff --git a/docs/changelog/111226.yaml b/docs/changelog/111226.yaml deleted file mode 100644 index 1021a26fa789f..0000000000000 --- a/docs/changelog/111226.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111226 -summary: "ES|QL: add Telemetry API and track top functions" -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/111238.yaml b/docs/changelog/111238.yaml deleted file mode 100644 index b918b754ff595..0000000000000 --- a/docs/changelog/111238.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111238 -summary: Fix validation of TEXT fields with case insensitive comparison -area: EQL -type: bug -issues: - - 111235 diff --git a/docs/changelog/111245.yaml b/docs/changelog/111245.yaml deleted file mode 100644 index 384373d52cb20..0000000000000 --- a/docs/changelog/111245.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111245 -summary: Truncating watcher history if it is too large -area: Watcher -type: bug -issues: - - 94745 diff --git a/docs/changelog/111274.yaml b/docs/changelog/111274.yaml deleted file mode 100644 index e26bcc03ce118..0000000000000 --- a/docs/changelog/111274.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111274 -summary: Include account name in Azure settings exceptions -area: Snapshot/Restore -type: enhancement -issues: [] diff --git a/docs/changelog/111284.yaml b/docs/changelog/111284.yaml deleted file mode 100644 index f87649a134af6..0000000000000 --- a/docs/changelog/111284.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111284 -summary: Update `semantic_text` field to support indexing numeric and boolean data - types -area: Mapping -type: enhancement -issues: [] diff --git a/docs/changelog/111311.yaml b/docs/changelog/111311.yaml deleted file mode 100644 index 5786e11e885e2..0000000000000 --- a/docs/changelog/111311.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111311 -summary: Adding support for data streams with a match-all template -area: Data streams -type: bug -issues: - - 111204 diff --git a/docs/changelog/111315.yaml b/docs/changelog/111315.yaml deleted file mode 100644 index 0e2e56898b51c..0000000000000 --- a/docs/changelog/111315.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111315 -summary: Add link to flood-stage watermark exception message -area: Allocation -type: enhancement -issues: [] diff --git a/docs/changelog/111316.yaml b/docs/changelog/111316.yaml deleted file mode 100644 index 0d915cd1ec3ea..0000000000000 --- a/docs/changelog/111316.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111316 -summary: "[Service Account] Add `AutoOps` account" -area: Security -type: enhancement -issues: [] diff --git a/docs/changelog/111344.yaml b/docs/changelog/111344.yaml deleted file mode 100644 index 3d5988054749d..0000000000000 --- a/docs/changelog/111344.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111344 -summary: Add support for Azure Managed Identity -area: Snapshot/Restore -type: enhancement -issues: [] diff --git a/docs/changelog/111367.yaml b/docs/changelog/111367.yaml deleted file mode 100644 index 89e6c1d3b4da4..0000000000000 --- a/docs/changelog/111367.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111367 -summary: "ESQL: Add Values aggregation tests, fix `ConstantBytesRefBlock` memory handling" -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/111412.yaml b/docs/changelog/111412.yaml deleted file mode 100644 index 297fa77cd2664..0000000000000 --- a/docs/changelog/111412.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111412 -summary: Make enrich cache based on memory usage -area: Ingest Node -type: enhancement -issues: - - 106081 diff --git a/docs/changelog/111413.yaml b/docs/changelog/111413.yaml deleted file mode 100644 index 0eae45b17d0c4..0000000000000 --- a/docs/changelog/111413.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111413 -summary: "ESQL: Fix synthetic attribute pruning" -area: ES|QL -type: bug -issues: - - 105821 diff --git a/docs/changelog/111420.yaml b/docs/changelog/111420.yaml deleted file mode 100644 index 4e2640ac5762a..0000000000000 --- a/docs/changelog/111420.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111420 -summary: "[Query rules] Add `exclude` query rule type" -area: Relevance -type: feature -issues: [] diff --git a/docs/changelog/111437.yaml b/docs/changelog/111437.yaml deleted file mode 100644 index a50312ffdd1aa..0000000000000 --- a/docs/changelog/111437.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111437 -summary: "[ES|QL] Create `Range` in `PushFiltersToSource` for qualified pushable filters on the same field" -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/111445.yaml b/docs/changelog/111445.yaml deleted file mode 100644 index 9ba8e4371bd0c..0000000000000 --- a/docs/changelog/111445.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111445 -summary: Support booleans in routing path -area: TSDB -type: enhancement -issues: [] diff --git a/docs/changelog/111457.yaml b/docs/changelog/111457.yaml deleted file mode 100644 index f4ad4ee53eb0a..0000000000000 --- a/docs/changelog/111457.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111457 -summary: Add support for boolean dimensions -area: TSDB -type: enhancement -issues: - - 111338 diff --git a/docs/changelog/111490.yaml b/docs/changelog/111490.yaml deleted file mode 100644 index b67c16189cc62..0000000000000 --- a/docs/changelog/111490.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111490 -summary: Temporarily return both `modelId` and `inferenceId` for GET /_inference until we migrate clients to only `inferenceId` -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/111501.yaml b/docs/changelog/111501.yaml deleted file mode 100644 index a424142376e52..0000000000000 --- a/docs/changelog/111501.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111501 -summary: "[ES|QL] Combine Disjunctive CIDRMatch" -area: ES|QL -type: enhancement -issues: - - 105143 diff --git a/docs/changelog/111516.yaml b/docs/changelog/111516.yaml deleted file mode 100644 index 96e8bd843f750..0000000000000 --- a/docs/changelog/111516.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111516 -summary: Adding support for `allow_partial_search_results` in PIT -area: Search -type: enhancement -issues: [] diff --git a/docs/changelog/111519.yaml b/docs/changelog/111519.yaml deleted file mode 100644 index 8cc62fb8ed903..0000000000000 --- a/docs/changelog/111519.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111519 -summary: "ESQL: Don't mutate the `BoolQueryBuilder` in plan" -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/111523.yaml b/docs/changelog/111523.yaml deleted file mode 100644 index 202d16c5a426d..0000000000000 --- a/docs/changelog/111523.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111523 -summary: Search coordinator uses `event.ingested` in cluster state to do rewrites -area: Search -type: enhancement -issues: [] diff --git a/docs/changelog/111535.yaml b/docs/changelog/111535.yaml deleted file mode 100644 index 4beebbf28d4e1..0000000000000 --- a/docs/changelog/111535.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111535 -summary: Fix remote cluster credential secure settings reload -area: Authorization -type: bug -issues: [] diff --git a/docs/changelog/111544.yaml b/docs/changelog/111544.yaml deleted file mode 100644 index d4c46f485e664..0000000000000 --- a/docs/changelog/111544.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111544 -summary: "ESQL: Strings support for MAX and MIN aggregations" -area: ES|QL -type: feature -issues: [] diff --git a/docs/changelog/111548.yaml b/docs/changelog/111548.yaml deleted file mode 100644 index ca9e5ae622894..0000000000000 --- a/docs/changelog/111548.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111548 -summary: Json parsing exceptions should not cause 500 errors -area: Infra/Core -type: bug -issues: - - 111542 diff --git a/docs/changelog/111552.yaml b/docs/changelog/111552.yaml deleted file mode 100644 index d9991788d4fa9..0000000000000 --- a/docs/changelog/111552.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111552 -summary: Siem ea 9521 improve test -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/111576.yaml b/docs/changelog/111576.yaml deleted file mode 100644 index 6d3c331f4bbd5..0000000000000 --- a/docs/changelog/111576.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111576 -summary: Execute shard snapshot tasks in shard-id order -area: Snapshot/Restore -type: enhancement -issues: - - 108739 diff --git a/docs/changelog/111600.yaml b/docs/changelog/111600.yaml deleted file mode 100644 index 0c1e01e1c2e23..0000000000000 --- a/docs/changelog/111600.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111600 -summary: Make ecs@mappings work with OTel attributes -area: Data streams -type: enhancement -issues: [] diff --git a/docs/changelog/111624.yaml b/docs/changelog/111624.yaml deleted file mode 100644 index 7b04b244ef7a7..0000000000000 --- a/docs/changelog/111624.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111624 -summary: Extend logging for dropped warning headers -area: Infra/Core -type: enhancement -issues: - - 90527 diff --git a/docs/changelog/111644.yaml b/docs/changelog/111644.yaml deleted file mode 100644 index 3705d697c95e3..0000000000000 --- a/docs/changelog/111644.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111644 -summary: Force using the last centroid during merging -area: Aggregations -type: bug -issues: - - 111065 diff --git a/docs/changelog/111655.yaml b/docs/changelog/111655.yaml deleted file mode 100644 index 077714d15a712..0000000000000 --- a/docs/changelog/111655.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111655 -summary: Migrate Inference to `ChunkedToXContent` -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/111683.yaml b/docs/changelog/111683.yaml deleted file mode 100644 index cbb2e5ad71ddc..0000000000000 --- a/docs/changelog/111683.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111683 -summary: Only emit product origin in deprecation log if present -area: Infra/Logging -type: bug -issues: - - 81757 diff --git a/docs/changelog/111689.yaml b/docs/changelog/111689.yaml deleted file mode 100644 index ccb3d4d4f87c5..0000000000000 --- a/docs/changelog/111689.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111689 -summary: Add nanos support to `ZonedDateTime` serialization -area: Infra/Core -type: enhancement -issues: - - 68292 diff --git a/docs/changelog/111690.yaml b/docs/changelog/111690.yaml deleted file mode 100644 index 36e715744ad88..0000000000000 --- a/docs/changelog/111690.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111690 -summary: "ESQL: Support INLINESTATS grouped on expressions" -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/111740.yaml b/docs/changelog/111740.yaml deleted file mode 100644 index 48b7ee200e45e..0000000000000 --- a/docs/changelog/111740.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111740 -summary: Fix Start Trial API output acknowledgement header for features -area: License -type: bug -issues: - - 111739 diff --git a/docs/changelog/111749.yaml b/docs/changelog/111749.yaml deleted file mode 100644 index 77e0c65005dd6..0000000000000 --- a/docs/changelog/111749.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111749 -summary: "ESQL: Added `mv_percentile` function" -area: ES|QL -type: feature -issues: - - 111591 diff --git a/docs/changelog/111779.yaml b/docs/changelog/111779.yaml deleted file mode 100644 index 52c635490e1e4..0000000000000 --- a/docs/changelog/111779.yaml +++ /dev/null @@ -1,7 +0,0 @@ -pr: 111779 -summary: "ESQL: Fix serialization during `can_match`" -area: ES|QL -type: bug -issues: - - 111701 - - 111726 diff --git a/docs/changelog/111797.yaml b/docs/changelog/111797.yaml deleted file mode 100644 index 00b793a19d9c3..0000000000000 --- a/docs/changelog/111797.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111797 -summary: "ESQL: fix for missing indices error message" -area: ES|QL -type: bug -issues: - - 111712 diff --git a/docs/changelog/111809.yaml b/docs/changelog/111809.yaml deleted file mode 100644 index 5a2f220e3a697..0000000000000 --- a/docs/changelog/111809.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111809 -summary: Add Field caps support for Semantic Text -area: Mapping -type: enhancement -issues: [] diff --git a/docs/changelog/111818.yaml b/docs/changelog/111818.yaml deleted file mode 100644 index c3a632861aae6..0000000000000 --- a/docs/changelog/111818.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111818 -summary: Add tier preference to security index settings allowlist -area: Security -type: enhancement -issues: [] diff --git a/docs/changelog/111840.yaml b/docs/changelog/111840.yaml deleted file mode 100644 index c40a9e2aef621..0000000000000 --- a/docs/changelog/111840.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111840 -summary: "ESQL: Add async ID and `is_running` headers to ESQL async query" -area: ES|QL -type: feature -issues: [] diff --git a/docs/changelog/111855.yaml b/docs/changelog/111855.yaml deleted file mode 100644 index 3f15e9c20135a..0000000000000 --- a/docs/changelog/111855.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111855 -summary: "ESQL: Profile more timing information" -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/111874.yaml b/docs/changelog/111874.yaml deleted file mode 100644 index 26ec90aa6cd4c..0000000000000 --- a/docs/changelog/111874.yaml +++ /dev/null @@ -1,8 +0,0 @@ -pr: 111874 -summary: "ESQL: BUCKET: allow numerical spans as whole numbers" -area: ES|QL -type: enhancement -issues: - - 104646 - - 109340 - - 105375 diff --git a/docs/changelog/111879.yaml b/docs/changelog/111879.yaml deleted file mode 100644 index b8c2111e1d286..0000000000000 --- a/docs/changelog/111879.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111879 -summary: "ESQL: Have BUCKET generate friendlier intervals" -area: ES|QL -type: enhancement -issues: - - 110916 diff --git a/docs/changelog/111915.yaml b/docs/changelog/111915.yaml deleted file mode 100644 index f64c45b82d10c..0000000000000 --- a/docs/changelog/111915.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111915 -summary: Fix DLS & FLS sometimes being enforced when it is disabled -area: Authorization -type: bug -issues: - - 94709 diff --git a/docs/changelog/111917.yaml b/docs/changelog/111917.yaml deleted file mode 100644 index 0dc760d76a698..0000000000000 --- a/docs/changelog/111917.yaml +++ /dev/null @@ -1,7 +0,0 @@ -pr: 111917 -summary: "[ES|QL] Cast mixed numeric types to a common numeric type for Coalesce and\ - \ In at Analyzer" -area: ES|QL -type: enhancement -issues: - - 111486 diff --git a/docs/changelog/111932.yaml b/docs/changelog/111932.yaml deleted file mode 100644 index ce840ecebcff0..0000000000000 --- a/docs/changelog/111932.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111932 -summary: Fix union-types where one index is missing the field -area: ES|QL -type: bug -issues: - - 111912 diff --git a/docs/changelog/111937.yaml b/docs/changelog/111937.yaml deleted file mode 100644 index 7d856e29d54c5..0000000000000 --- a/docs/changelog/111937.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111937 -summary: Handle `BigInteger` in xcontent copy -area: Infra/Core -type: bug -issues: - - 111812 diff --git a/docs/changelog/111948.yaml b/docs/changelog/111948.yaml deleted file mode 100644 index a3a592abaf1ca..0000000000000 --- a/docs/changelog/111948.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111948 -summary: Upgrade xcontent to Jackson 2.17.0 -area: Infra/Core -type: upgrade -issues: [] diff --git a/docs/changelog/111950.yaml b/docs/changelog/111950.yaml deleted file mode 100644 index 3f23c17d8e652..0000000000000 --- a/docs/changelog/111950.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111950 -summary: "[ES|QL] Name parameter with leading underscore" -area: ES|QL -type: enhancement -issues: - - 111821 diff --git a/docs/changelog/111955.yaml b/docs/changelog/111955.yaml deleted file mode 100644 index ebc518203b7cc..0000000000000 --- a/docs/changelog/111955.yaml +++ /dev/null @@ -1,7 +0,0 @@ -pr: 111955 -summary: Clean up dangling S3 multipart uploads -area: Snapshot/Restore -type: enhancement -issues: - - 101169 - - 44971 diff --git a/docs/changelog/111968.yaml b/docs/changelog/111968.yaml deleted file mode 100644 index 9d758c76369e9..0000000000000 --- a/docs/changelog/111968.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111968 -summary: "ESQL: don't lose the original casting error message" -area: ES|QL -type: bug -issues: - - 111967 diff --git a/docs/changelog/111969.yaml b/docs/changelog/111969.yaml deleted file mode 100644 index 2d276850c4988..0000000000000 --- a/docs/changelog/111969.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111969 -summary: "[Profiling] add `container.id` field to event index template" -area: Application -type: enhancement -issues: [] diff --git a/docs/changelog/111972.yaml b/docs/changelog/111972.yaml deleted file mode 100644 index a5bfcd5b0882e..0000000000000 --- a/docs/changelog/111972.yaml +++ /dev/null @@ -1,17 +0,0 @@ -pr: 111972 -summary: Introduce global retention in data stream lifecycle. -area: Data streams -type: feature -issues: [] -highlight: - title: Add global retention in data stream lifecycle - body: |- - Data stream lifecycle now supports configuring retention on a cluster level, - namely global retention. Global retention \nallows us to configure two different - retentions: - - - `data_streams.lifecycle.retention.default` is applied to all data streams managed - by the data stream lifecycle that do not have retention defined on the data stream level. - - `data_streams.lifecycle.retention.max` is applied to all data streams managed by the - data stream lifecycle and it allows any data stream \ndata to be deleted after the `max_retention` has passed. - notable: true diff --git a/docs/changelog/111981.yaml b/docs/changelog/111981.yaml deleted file mode 100644 index 13b8fe4b7e38d..0000000000000 --- a/docs/changelog/111981.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111981 -summary: Allow fields with dots in sparse vector field mapper -area: Mapping -type: enhancement -issues: - - 109118 diff --git a/docs/changelog/112019.yaml b/docs/changelog/112019.yaml deleted file mode 100644 index 7afb207864ed7..0000000000000 --- a/docs/changelog/112019.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112019 -summary: Display effective retention in the relevant data stream APIs -area: Data streams -type: enhancement -issues: [] diff --git a/docs/changelog/112024.yaml b/docs/changelog/112024.yaml deleted file mode 100644 index e426693fba964..0000000000000 --- a/docs/changelog/112024.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112024 -summary: (API) Cluster Health report `unassigned_primary_shards` -area: Health -type: enhancement -issues: [] diff --git a/docs/changelog/112026.yaml b/docs/changelog/112026.yaml deleted file mode 100644 index fedf001923ab4..0000000000000 --- a/docs/changelog/112026.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112026 -summary: Create `StreamingHttpResultPublisher` -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/112055.yaml b/docs/changelog/112055.yaml deleted file mode 100644 index cdf15b3b37468..0000000000000 --- a/docs/changelog/112055.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 112055 -summary: "ESQL: `mv_median_absolute_deviation` function" -area: ES|QL -type: feature -issues: - - 111590 diff --git a/docs/changelog/112058.yaml b/docs/changelog/112058.yaml deleted file mode 100644 index e974b3413582e..0000000000000 --- a/docs/changelog/112058.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112058 -summary: Fix RRF validation for `rank_constant` < 1 -area: Ranking -type: bug -issues: [] diff --git a/docs/changelog/112066.yaml b/docs/changelog/112066.yaml deleted file mode 100644 index 5dd846766bc8e..0000000000000 --- a/docs/changelog/112066.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 112066 -summary: Do not treat replica as unassigned if primary recently created and unassigned - time is below a threshold -area: Health -type: enhancement -issues: [] diff --git a/docs/changelog/112100.yaml b/docs/changelog/112100.yaml deleted file mode 100644 index 9135edecb4d77..0000000000000 --- a/docs/changelog/112100.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112100 -summary: Exclude internal data streams from global retention -area: Data streams -type: bug -issues: [] diff --git a/docs/changelog/112123.yaml b/docs/changelog/112123.yaml deleted file mode 100644 index 0c0d7ac44cd17..0000000000000 --- a/docs/changelog/112123.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112123 -summary: SLM interval schedule followup - add back `getFieldName` style getters -area: ILM+SLM -type: enhancement -issues: [] diff --git a/docs/changelog/112126.yaml b/docs/changelog/112126.yaml deleted file mode 100644 index f6a7aeb893a5e..0000000000000 --- a/docs/changelog/112126.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112126 -summary: Add support for spatial relationships in point field mapper -area: Geo -type: enhancement -issues: [] diff --git a/docs/changelog/112133.yaml b/docs/changelog/112133.yaml deleted file mode 100644 index 11109402b7373..0000000000000 --- a/docs/changelog/112133.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112133 -summary: Add telemetry for repository usage -area: Snapshot/Restore -type: enhancement -issues: [] diff --git a/docs/changelog/112151.yaml b/docs/changelog/112151.yaml deleted file mode 100644 index f5cbfd8da07c2..0000000000000 --- a/docs/changelog/112151.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112151 -summary: Store original source for keywords using a normalizer -area: Logs -type: enhancement -issues: [] diff --git a/docs/changelog/112199.yaml b/docs/changelog/112199.yaml deleted file mode 100644 index eb22f215f9828..0000000000000 --- a/docs/changelog/112199.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112199 -summary: Support docvalues only query in shape field -area: Geo -type: enhancement -issues: [] diff --git a/docs/changelog/112200.yaml b/docs/changelog/112200.yaml deleted file mode 100644 index 0c2c3d71e3ddf..0000000000000 --- a/docs/changelog/112200.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 112200 -summary: "ES|QL: better validation of GROK patterns" -area: ES|QL -type: bug -issues: - - 112111 diff --git a/docs/changelog/112210.yaml b/docs/changelog/112210.yaml deleted file mode 100644 index 6483b8b01315c..0000000000000 --- a/docs/changelog/112210.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112210 -summary: Expose global retention settings via data stream lifecycle API -area: Data streams -type: enhancement -issues: [] diff --git a/docs/changelog/112214.yaml b/docs/changelog/112214.yaml deleted file mode 100644 index 430f95a72bb3f..0000000000000 --- a/docs/changelog/112214.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112214 -summary: '`ByteArrayStreamInput:` Return -1 when there are no more bytes to read' -area: Infra/Core -type: bug -issues: [] diff --git a/docs/changelog/112218.yaml b/docs/changelog/112218.yaml deleted file mode 100644 index c426dd7ade4ed..0000000000000 --- a/docs/changelog/112218.yaml +++ /dev/null @@ -1,9 +0,0 @@ -pr: 112218 -summary: "ESQL: Fix a bug in `MV_PERCENTILE`" -area: ES|QL -type: bug -issues: - - 112193 - - 112180 - - 112187 - - 112188 diff --git a/docs/changelog/112262.yaml b/docs/changelog/112262.yaml deleted file mode 100644 index fe23c14c79c9e..0000000000000 --- a/docs/changelog/112262.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 112262 -summary: Check for disabling own user in Put User API -area: Authentication -type: bug -issues: - - 90205 diff --git a/docs/changelog/112263.yaml b/docs/changelog/112263.yaml deleted file mode 100644 index 2d1321f327673..0000000000000 --- a/docs/changelog/112263.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 112263 -summary: Fix `TokenService` always appearing used in Feature Usage -area: License -type: bug -issues: - - 61956 diff --git a/docs/changelog/112270.yaml b/docs/changelog/112270.yaml deleted file mode 100644 index 1e6b9c7fc9290..0000000000000 --- a/docs/changelog/112270.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112270 -summary: Support sparse embedding models in the elasticsearch inference service -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/112273.yaml b/docs/changelog/112273.yaml deleted file mode 100644 index 3182a1884a145..0000000000000 --- a/docs/changelog/112273.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111181 -summary: "[Inference API] Add Docs for AlibabaCloud AI Search Support for the Inference API" -area: Machine Learning -type: enhancement -issues: [ ] diff --git a/docs/changelog/112277.yaml b/docs/changelog/112277.yaml deleted file mode 100644 index eac474555999a..0000000000000 --- a/docs/changelog/112277.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112277 -summary: Upgrade `repository-azure` dependencies -area: Snapshot/Restore -type: upgrade -issues: [] diff --git a/docs/changelog/112282.yaml b/docs/changelog/112282.yaml deleted file mode 100644 index beea119b06aef..0000000000000 --- a/docs/changelog/112282.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 112282 -summary: Adds example plugin for custom ingest processor -area: Ingest Node -type: enhancement -issues: - - 111539 diff --git a/docs/changelog/112294.yaml b/docs/changelog/112294.yaml deleted file mode 100644 index 71ce9eeef584c..0000000000000 --- a/docs/changelog/112294.yaml +++ /dev/null @@ -1,8 +0,0 @@ -pr: 112294 -summary: "Use fallback synthetic source for `copy_to` and doc_values: false cases" -area: Mapping -type: enhancement -issues: - - 110753 - - 110038 - - 109546 diff --git a/docs/changelog/112303.yaml b/docs/changelog/112303.yaml deleted file mode 100644 index a363e621e4c48..0000000000000 --- a/docs/changelog/112303.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112303 -summary: Add 'verbose' flag retrieving `maximum_timestamp` for get data stream API -area: Data streams -type: enhancement -issues: [] diff --git a/docs/changelog/112320.yaml b/docs/changelog/112320.yaml deleted file mode 100644 index d35a08dfa4e91..0000000000000 --- a/docs/changelog/112320.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112320 -summary: Upgrade xcontent to Jackson 2.17.2 -area: Infra/Core -type: upgrade -issues: [] diff --git a/docs/changelog/112330.yaml b/docs/changelog/112330.yaml deleted file mode 100644 index 498698f5175ba..0000000000000 --- a/docs/changelog/112330.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112330 -summary: Add links to network disconnect troubleshooting -area: Network -type: enhancement -issues: [] diff --git a/docs/changelog/112337.yaml b/docs/changelog/112337.yaml deleted file mode 100644 index f7d667e23cfe9..0000000000000 --- a/docs/changelog/112337.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112337 -summary: Add workaround for missing shard gen blob -area: Snapshot/Restore -type: enhancement -issues: [] diff --git a/docs/changelog/112341.yaml b/docs/changelog/112341.yaml deleted file mode 100644 index 8f44b53ad9998..0000000000000 --- a/docs/changelog/112341.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112341 -summary: Fix DLS using runtime fields and synthetic source -area: Authorization -type: bug -issues: [] diff --git a/docs/changelog/112345.yaml b/docs/changelog/112345.yaml deleted file mode 100644 index b922fe3754cbb..0000000000000 --- a/docs/changelog/112345.yaml +++ /dev/null @@ -1,8 +0,0 @@ -pr: 112345 -summary: Allow dimension fields to have multiple values in standard and logsdb index - mode -area: Mapping -type: enhancement -issues: - - 112232 - - 112239 diff --git a/docs/changelog/112348.yaml b/docs/changelog/112348.yaml deleted file mode 100644 index 84110a7cd4f1b..0000000000000 --- a/docs/changelog/112348.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 112348 -summary: Introduce repository integrity verification API -area: Snapshot/Restore -type: enhancement -issues: - - 52622 diff --git a/docs/changelog/112350.yaml b/docs/changelog/112350.yaml deleted file mode 100644 index 994cd3a65c633..0000000000000 --- a/docs/changelog/112350.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112350 -summary: "[ESQL] Add `SPACE` function" -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/112369.yaml b/docs/changelog/112369.yaml deleted file mode 100644 index fb1c4775f7a12..0000000000000 --- a/docs/changelog/112369.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112369 -summary: Register Task while Streaming -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/112397.yaml b/docs/changelog/112397.yaml deleted file mode 100644 index e67478ec69b1c..0000000000000 --- a/docs/changelog/112397.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112397 -summary: Control storing array source with index setting -area: Mapping -type: enhancement -issues: [] diff --git a/docs/changelog/112400.yaml b/docs/changelog/112400.yaml deleted file mode 100644 index 6d622e5fb5248..0000000000000 --- a/docs/changelog/112400.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112400 -summary: Make sure file accesses in `DnRoleMapper` are done in stack frames with permissions -area: Infra/Core -type: bug -issues: [] diff --git a/docs/changelog/112401.yaml b/docs/changelog/112401.yaml deleted file mode 100644 index 65e9e76ac25f6..0000000000000 --- a/docs/changelog/112401.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 112401 -summary: "ESQL: Fix CASE when conditions are multivalued" -area: ES|QL -type: bug -issues: - - 112359 diff --git a/docs/changelog/112409.yaml b/docs/changelog/112409.yaml deleted file mode 100644 index bad94b9f5f2be..0000000000000 --- a/docs/changelog/112409.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 112409 -summary: Include reason when no nodes are found -area: "Transform" -type: bug -issues: - - 112404 diff --git a/docs/changelog/112412.yaml b/docs/changelog/112412.yaml deleted file mode 100644 index fda53ebd1ade0..0000000000000 --- a/docs/changelog/112412.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112412 -summary: Expose `HexFormat` in Painless -area: Infra/Scripting -type: enhancement -issues: [] diff --git a/docs/changelog/112431.yaml b/docs/changelog/112431.yaml deleted file mode 100644 index b8c1197bdc7ef..0000000000000 --- a/docs/changelog/112431.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 112431 -summary: "Async search: Add ID and \"is running\" http headers" -area: Search -type: feature -issues: - - 109576 diff --git a/docs/changelog/112440.yaml b/docs/changelog/112440.yaml deleted file mode 100644 index f208474fa2686..0000000000000 --- a/docs/changelog/112440.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112440 -summary: "logs-apm.error-*: define log.level field as keyword" -area: Data streams -type: bug -issues: [] diff --git a/docs/changelog/112444.yaml b/docs/changelog/112444.yaml deleted file mode 100644 index bfa4fd693f0e0..0000000000000 --- a/docs/changelog/112444.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 112444 -summary: Full coverage of ECS by ecs@mappings when `date_detection` is disabled -area: Mapping -type: bug -issues: - - 112398 diff --git a/docs/changelog/112451.yaml b/docs/changelog/112451.yaml deleted file mode 100644 index aa852cf5e2a1a..0000000000000 --- a/docs/changelog/112451.yaml +++ /dev/null @@ -1,29 +0,0 @@ -pr: 112451 -summary: Update data stream lifecycle telemetry to track global retention -area: Data streams -type: breaking -issues: [] -breaking: - title: Update data stream lifecycle telemetry to track global retention - area: REST API - details: |- - In this release we introduced global retention settings that fulfil the following criteria: - - - a data stream managed by the data stream lifecycle, - - a data stream that is not an internal data stream. - - As a result, we defined different types of retention: - - - **data retention**: the retention configured on data stream level by the data stream user or owner - - **default global retention:** the retention configured by an admin on a cluster level and applied to any - data stream that doesn't have data retention and fulfils the criteria. - - **max global retention:** the retention configured by an admin to guard against having long retention periods. - Any data stream that fulfills the criteria will adhere to the data retention unless it exceeds the max retention, - in which case the max global retention applies. - - **effective retention:** the retention that applies on the data stream that fulfill the criteria at a given moment - in time. It takes into consideration all the retention above and resolves it to the retention that will take effect. - - Considering the above changes, having a field named `retention` in the usage API was confusing. For this reason, we - renamed it to `data_retention` and added telemetry about the other configurations too. - impact: Users that use the field `data_lifecycle.retention` should use the `data_lifecycle.data_retention` - notable: false diff --git a/docs/changelog/112489.yaml b/docs/changelog/112489.yaml deleted file mode 100644 index ebc84927b0e76..0000000000000 --- a/docs/changelog/112489.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 112489 -summary: "ES|QL: better validation for RLIKE patterns" -area: ES|QL -type: bug -issues: - - 112485 diff --git a/docs/changelog/112508.yaml b/docs/changelog/112508.yaml deleted file mode 100644 index 3945ebd226ac4..0000000000000 --- a/docs/changelog/112508.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112508 -summary: "[ML] Create Inference API will no longer return model_id and now only return inference_id" -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/112519.yaml b/docs/changelog/112519.yaml deleted file mode 100644 index aa8a942ef0f58..0000000000000 --- a/docs/changelog/112519.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112519 -summary: Lower the memory footprint when creating `DelayedBucket` -area: Aggregations -type: enhancement -issues: [] diff --git a/docs/changelog/112547.yaml b/docs/changelog/112547.yaml deleted file mode 100644 index 7f42f2a82976e..0000000000000 --- a/docs/changelog/112547.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112547 -summary: Remove reduce and `reduceContext` from `DelayedBucket` -area: Aggregations -type: enhancement -issues: [] diff --git a/docs/changelog/112574.yaml b/docs/changelog/112574.yaml deleted file mode 100644 index 3111697a8b97f..0000000000000 --- a/docs/changelog/112574.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112574 -summary: Add privileges required for CDR misconfiguration features to work on AWS SecurityHub integration -area: Authorization -type: enhancement -issues: [] diff --git a/docs/changelog/112581.yaml b/docs/changelog/112581.yaml deleted file mode 100644 index 489b4780c06fb..0000000000000 --- a/docs/changelog/112581.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112581 -summary: Fix missing header in `put_geoip_database` JSON spec -area: Ingest Node -type: bug -issues: [] diff --git a/docs/changelog/112610.yaml b/docs/changelog/112610.yaml deleted file mode 100644 index 3d67a80a8f0b3..0000000000000 --- a/docs/changelog/112610.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 112610 -summary: Support widening of numeric types in union-types -area: ES|QL -type: bug -issues: - - 111277 diff --git a/docs/changelog/112612.yaml b/docs/changelog/112612.yaml deleted file mode 100644 index d6037e34ff171..0000000000000 --- a/docs/changelog/112612.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112612 -summary: Set `replica_unassigned_buffer_time` in constructor -area: Health -type: bug -issues: [] diff --git a/docs/changelog/112649.yaml b/docs/changelog/112649.yaml deleted file mode 100644 index e3cf1e8e34881..0000000000000 --- a/docs/changelog/112649.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112649 -summary: Allowlist `tracestate` header on remote server port -area: Security -type: bug -issues: [] diff --git a/docs/changelog/112687.yaml b/docs/changelog/112687.yaml deleted file mode 100644 index dd079e1b700c4..0000000000000 --- a/docs/changelog/112687.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112687 -summary: Add `TaskManager` to `pluginServices` -area: Infra/Metrics -type: enhancement -issues: [] diff --git a/docs/changelog/112703.yaml b/docs/changelog/112703.yaml deleted file mode 100644 index a428e8c4e2339..0000000000000 --- a/docs/changelog/112703.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112703 -summary: JSON parse failures should be 4xx codes -area: Infra/Core -type: bug -issues: [] diff --git a/docs/changelog/112713.yaml b/docs/changelog/112713.yaml deleted file mode 100644 index 1ccf451b13f82..0000000000000 --- a/docs/changelog/112713.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112713 -summary: Fix encoding of dynamic arrays in ignored source -area: Logs -type: bug -issues: [] diff --git a/docs/changelog/112720.yaml b/docs/changelog/112720.yaml deleted file mode 100644 index a44ea5a699520..0000000000000 --- a/docs/changelog/112720.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112720 -summary: Fix NPE in `dense_vector` stats -area: Vector Search -type: bug -issues: [] diff --git a/docs/changelog/113120.yaml b/docs/changelog/113120.yaml new file mode 100644 index 0000000000000..801167d61c19c --- /dev/null +++ b/docs/changelog/113120.yaml @@ -0,0 +1,5 @@ +pr: 113120 +summary: ESQL - enabling scoring with METADATA `_score` +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/113827.yaml b/docs/changelog/113827.yaml new file mode 100644 index 0000000000000..2c05f3eeb5d6a --- /dev/null +++ b/docs/changelog/113827.yaml @@ -0,0 +1,5 @@ +pr: 113827 +summary: Add Optional Source Filtering to Source Loaders +area: Mapping +type: enhancement +issues: [] diff --git a/docs/changelog/114317.yaml b/docs/changelog/114317.yaml new file mode 100644 index 0000000000000..9c73fe513e197 --- /dev/null +++ b/docs/changelog/114317.yaml @@ -0,0 +1,5 @@ +pr: 114317 +summary: "ESQL: CATEGORIZE as a `BlockHash`" +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/114445.yaml b/docs/changelog/114445.yaml new file mode 100644 index 0000000000000..afbc080d1e0b9 --- /dev/null +++ b/docs/changelog/114445.yaml @@ -0,0 +1,6 @@ +pr: 114445 +summary: Wrap jackson exception on malformed json string +area: Infra/Core +type: bug +issues: + - 114142 diff --git a/docs/changelog/114618.yaml b/docs/changelog/114618.yaml new file mode 100644 index 0000000000000..ada402fe35742 --- /dev/null +++ b/docs/changelog/114618.yaml @@ -0,0 +1,5 @@ +pr: 114618 +summary: Add a new index setting to skip recovery source when synthetic source is enabled +area: Logs +type: enhancement +issues: [] diff --git a/docs/changelog/114914.yaml b/docs/changelog/114914.yaml new file mode 100644 index 0000000000000..bad13e26682dc --- /dev/null +++ b/docs/changelog/114914.yaml @@ -0,0 +1,5 @@ +pr: 114914 +summary: Adding chunking settings to `IbmWatsonxService` +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/115020.yaml b/docs/changelog/115020.yaml new file mode 100644 index 0000000000000..2b0aefafea507 --- /dev/null +++ b/docs/changelog/115020.yaml @@ -0,0 +1,5 @@ +pr: 115020 +summary: Adding endpoint creation validation for all task types to remaining services +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/115314.yaml b/docs/changelog/115314.yaml new file mode 100644 index 0000000000000..76ac12d58fcf3 --- /dev/null +++ b/docs/changelog/115314.yaml @@ -0,0 +1,5 @@ +pr: 115314 +summary: Only aggregations require at least one shard request +area: Search +type: enhancement +issues: [] diff --git a/docs/changelog/115616.yaml b/docs/changelog/115616.yaml new file mode 100644 index 0000000000000..4fb4dc18538de --- /dev/null +++ b/docs/changelog/115616.yaml @@ -0,0 +1,6 @@ +pr: 115616 +summary: Fix double lookup failure on ESQL +area: ES|QL +type: bug +issues: + - 111398 diff --git a/docs/changelog/115836.yaml b/docs/changelog/115836.yaml new file mode 100644 index 0000000000000..f6da638f1feff --- /dev/null +++ b/docs/changelog/115836.yaml @@ -0,0 +1,5 @@ +pr: 115836 +summary: Catch and handle disconnect exceptions in search +area: Search +type: bug +issues: [] diff --git a/docs/changelog/115876.yaml b/docs/changelog/115876.yaml new file mode 100644 index 0000000000000..29b34b8b250fb --- /dev/null +++ b/docs/changelog/115876.yaml @@ -0,0 +1,5 @@ +pr: 115876 +summary: Inference duration and error metrics +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/116060.yaml b/docs/changelog/116060.yaml new file mode 100644 index 0000000000000..b067677ed41d9 --- /dev/null +++ b/docs/changelog/116060.yaml @@ -0,0 +1,6 @@ +pr: 116060 +summary: Fix leak in `DfsQueryPhase` and introduce search disconnect stress test +area: Search +type: bug +issues: + - 115056 diff --git a/docs/changelog/116115.yaml b/docs/changelog/116115.yaml new file mode 100644 index 0000000000000..33e1735c20ca4 --- /dev/null +++ b/docs/changelog/116115.yaml @@ -0,0 +1,5 @@ +pr: 116115 +summary: Allow http unsafe buffers by default +area: Network +type: enhancement +issues: [] diff --git a/docs/changelog/116388.yaml b/docs/changelog/116388.yaml new file mode 100644 index 0000000000000..59cdafc9ec337 --- /dev/null +++ b/docs/changelog/116388.yaml @@ -0,0 +1,5 @@ +pr: 116388 +summary: Add support for partial shard results +area: EQL +type: enhancement +issues: [] diff --git a/docs/changelog/116531.yaml b/docs/changelog/116531.yaml new file mode 100644 index 0000000000000..908bbff487973 --- /dev/null +++ b/docs/changelog/116531.yaml @@ -0,0 +1,5 @@ +pr: 116531 +summary: "Add a standard deviation aggregating function: STD_DEV" +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/116663.yaml b/docs/changelog/116663.yaml new file mode 100644 index 0000000000000..40bcdea29bc31 --- /dev/null +++ b/docs/changelog/116663.yaml @@ -0,0 +1,5 @@ +pr: 116663 +summary: KNN vector rescoring for quantized vectors +area: Vector Search +type: feature +issues: [] diff --git a/docs/changelog/116687.yaml b/docs/changelog/116687.yaml new file mode 100644 index 0000000000000..f8c7f86eff04a --- /dev/null +++ b/docs/changelog/116687.yaml @@ -0,0 +1,5 @@ +pr: 116687 +summary: Add LogsDB option to route on sort fields +area: Logs +type: enhancement +issues: [] diff --git a/docs/changelog/116755.yaml b/docs/changelog/116755.yaml new file mode 100644 index 0000000000000..3aa5ec8580b59 --- /dev/null +++ b/docs/changelog/116755.yaml @@ -0,0 +1,5 @@ +pr: 116755 +summary: Smarter field caps with subscribable listener +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/116765.yaml b/docs/changelog/116765.yaml new file mode 100644 index 0000000000000..ec2357c17acaf --- /dev/null +++ b/docs/changelog/116765.yaml @@ -0,0 +1,5 @@ +pr: 116765 +summary: Metrics for incremental bulk splits +area: Distributed +type: enhancement +issues: [] diff --git a/docs/changelog/116964.yaml b/docs/changelog/116964.yaml new file mode 100644 index 0000000000000..2e3ecd06fa098 --- /dev/null +++ b/docs/changelog/116964.yaml @@ -0,0 +1,6 @@ +pr: 116964 +summary: "Support ST_ENVELOPE and related (ST_XMIN, ST_XMAX, ST_YMIN, ST_YMAX) functions" +area: ES|QL +type: feature +issues: + - 104875 diff --git a/docs/changelog/116996.yaml b/docs/changelog/116996.yaml new file mode 100644 index 0000000000000..59f59355131bf --- /dev/null +++ b/docs/changelog/116996.yaml @@ -0,0 +1,5 @@ +pr: 116996 +summary: Initial work on `ReindexDatastreamIndexAction` +area: Data streams +type: enhancement +issues: [] diff --git a/docs/changelog/117199.yaml b/docs/changelog/117199.yaml new file mode 100644 index 0000000000000..b685e98b61f6b --- /dev/null +++ b/docs/changelog/117199.yaml @@ -0,0 +1,5 @@ +pr: 117199 +summary: Speed up bit compared with floats or bytes script operations +area: Vector Search +type: enhancement +issues: [] diff --git a/docs/changelog/117201.yaml b/docs/changelog/117201.yaml new file mode 100644 index 0000000000000..f8a2be35c70a3 --- /dev/null +++ b/docs/changelog/117201.yaml @@ -0,0 +1,6 @@ +pr: 117201 +summary: "Use `field_caps` native nested fields filtering" +area: ES|QL +type: bug +issues: + - 117054 diff --git a/docs/changelog/117214.yaml b/docs/changelog/117214.yaml new file mode 100644 index 0000000000000..ba74197eb7634 --- /dev/null +++ b/docs/changelog/117214.yaml @@ -0,0 +1,5 @@ +pr: 117214 +summary: Returning ignored fields in the simulate ingest API +area: Ingest Node +type: enhancement +issues: [] diff --git a/docs/changelog/117229.yaml b/docs/changelog/117229.yaml new file mode 100644 index 0000000000000..f1b859c03e4fa --- /dev/null +++ b/docs/changelog/117229.yaml @@ -0,0 +1,6 @@ +pr: 117229 +summary: "In this pr, a 400 error is returned when _source / _seq_no / _feature /\ + \ _nested_path / _field_names is requested, rather a 5xx" +area: Search +type: bug +issues: [] diff --git a/docs/changelog/117243.yaml b/docs/changelog/117243.yaml new file mode 100644 index 0000000000000..f871d476bd0ec --- /dev/null +++ b/docs/changelog/117243.yaml @@ -0,0 +1,5 @@ +pr: 117243 +summary: Bump major version for feature migration system indices +area: Infra/Core +type: upgrade +issues: [] diff --git a/docs/changelog/117246.yaml b/docs/changelog/117246.yaml new file mode 100644 index 0000000000000..29c4464855967 --- /dev/null +++ b/docs/changelog/117246.yaml @@ -0,0 +1,5 @@ +pr: 117246 +summary: LOOKUP JOIN using field-caps for field mapping +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/117265.yaml b/docs/changelog/117265.yaml new file mode 100644 index 0000000000000..ec6605155538d --- /dev/null +++ b/docs/changelog/117265.yaml @@ -0,0 +1,5 @@ +pr: 117265 +summary: Async search responses have CCS metadata while searches are running +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/117287.yaml b/docs/changelog/117287.yaml new file mode 100644 index 0000000000000..08da9dd8087b2 --- /dev/null +++ b/docs/changelog/117287.yaml @@ -0,0 +1,5 @@ +pr: 117287 +summary: Fixing bug setting index when parsing Google Vertex AI results +area: Machine Learning +type: bug +issues: [] diff --git a/docs/changelog/117303.yaml b/docs/changelog/117303.yaml new file mode 100644 index 0000000000000..71d134f2cd077 --- /dev/null +++ b/docs/changelog/117303.yaml @@ -0,0 +1,5 @@ +pr: 117303 +summary: Remove HTTP content copies +area: Network +type: enhancement +issues: [] diff --git a/docs/changelog/117359.yaml b/docs/changelog/117359.yaml new file mode 100644 index 0000000000000..87d2d828ace54 --- /dev/null +++ b/docs/changelog/117359.yaml @@ -0,0 +1,5 @@ +pr: 117359 +summary: Term query for ES|QL +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/117451.yaml b/docs/changelog/117451.yaml new file mode 100644 index 0000000000000..bda0ca59e0953 --- /dev/null +++ b/docs/changelog/117451.yaml @@ -0,0 +1,6 @@ +pr: 117451 +summary: ST_EXTENT aggregation +area: ES|QL +type: feature +issues: + - 104659 diff --git a/docs/changelog/117469.yaml b/docs/changelog/117469.yaml new file mode 100644 index 0000000000000..cfb14f78cb578 --- /dev/null +++ b/docs/changelog/117469.yaml @@ -0,0 +1,6 @@ +pr: 117469 +summary: Handle exceptions in query phase can match +area: Search +type: bug +issues: + - 104994 diff --git a/docs/changelog/117555.yaml b/docs/changelog/117555.yaml new file mode 100644 index 0000000000000..7891ab6d93a64 --- /dev/null +++ b/docs/changelog/117555.yaml @@ -0,0 +1,5 @@ +pr: 117555 +summary: Expand type compatibility for match function and operator +area: ES|QL +type: feature +issues: [] diff --git a/docs/changelog/117581.yaml b/docs/changelog/117581.yaml new file mode 100644 index 0000000000000..b88017f45e9c9 --- /dev/null +++ b/docs/changelog/117581.yaml @@ -0,0 +1,5 @@ +pr: 117581 +summary: Make reserved built-in roles queryable +area: Authorization +type: enhancement +issues: [] diff --git a/docs/changelog/117589.yaml b/docs/changelog/117589.yaml new file mode 100644 index 0000000000000..e6880fd9477b5 --- /dev/null +++ b/docs/changelog/117589.yaml @@ -0,0 +1,5 @@ +pr: 117589 +summary: "Add Inference Unified API for chat completions for OpenAI" +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/117606.yaml b/docs/changelog/117606.yaml new file mode 100644 index 0000000000000..ea61099a1a6b4 --- /dev/null +++ b/docs/changelog/117606.yaml @@ -0,0 +1,5 @@ +pr: 117606 +summary: Remove deprecated sort from reindex operation within dataframe analytics procedure +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/117643.yaml b/docs/changelog/117643.yaml new file mode 100644 index 0000000000000..9105749377d2c --- /dev/null +++ b/docs/changelog/117643.yaml @@ -0,0 +1,6 @@ +pr: 117643 +summary: Drop null columns in text formats +area: ES|QL +type: bug +issues: + - 116848 diff --git a/docs/changelog/117655.yaml b/docs/changelog/117655.yaml new file mode 100644 index 0000000000000..f2afd3570f104 --- /dev/null +++ b/docs/changelog/117655.yaml @@ -0,0 +1,5 @@ +pr: 117655 +summary: Add nulls support to Categorize +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/117696.yaml b/docs/changelog/117696.yaml new file mode 100644 index 0000000000000..ee3ec064ce9ae --- /dev/null +++ b/docs/changelog/117696.yaml @@ -0,0 +1,5 @@ +pr: 117696 +summary: "[8.x] Add version prefix to Inference Service API path" +area: Inference +type: enhancement +issues: [] diff --git a/docs/changelog/117701.yaml b/docs/changelog/117701.yaml new file mode 100644 index 0000000000000..5a72bdeb143e6 --- /dev/null +++ b/docs/changelog/117701.yaml @@ -0,0 +1,6 @@ +pr: 117701 +summary: Watcher history index has too many indexed fields - +area: Watcher +type: bug +issues: + - 71479 diff --git a/docs/changelog/117729.yaml b/docs/changelog/117729.yaml new file mode 100644 index 0000000000000..e37df59796b8d --- /dev/null +++ b/docs/changelog/117729.yaml @@ -0,0 +1,5 @@ +pr: 117729 +summary: "[8.x] fix/SearchStatesIt_failures" +area: Search +type: bug +issues: [] diff --git a/docs/changelog/117731.yaml b/docs/changelog/117731.yaml new file mode 100644 index 0000000000000..f69cd5bf31100 --- /dev/null +++ b/docs/changelog/117731.yaml @@ -0,0 +1,5 @@ +pr: 117731 +summary: Add cluster level reduction +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/117747.yaml b/docs/changelog/117747.yaml new file mode 100644 index 0000000000000..fe659b42785bc --- /dev/null +++ b/docs/changelog/117747.yaml @@ -0,0 +1,5 @@ +pr: 117747 +summary: Backport scoring support in ES|QL to 8.x branch +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/117748.yaml b/docs/changelog/117748.yaml new file mode 100644 index 0000000000000..615adbae07ad7 --- /dev/null +++ b/docs/changelog/117748.yaml @@ -0,0 +1,6 @@ +pr: 117748 +summary: Add IMDSv2 support to `repository-s3` +area: Snapshot/Restore +type: enhancement +issues: + - 105135 diff --git a/docs/changelog/117750.yaml b/docs/changelog/117750.yaml new file mode 100644 index 0000000000000..3ba3f1693f4df --- /dev/null +++ b/docs/changelog/117750.yaml @@ -0,0 +1,6 @@ +pr: 117750 +summary: '`CrossClusterIT` `testCancel` failure' +area: Search +type: bug +issues: + - 108061 diff --git a/docs/changelog/117839.yaml b/docs/changelog/117839.yaml new file mode 100644 index 0000000000000..98c97b5078c02 --- /dev/null +++ b/docs/changelog/117839.yaml @@ -0,0 +1,5 @@ +pr: 117839 +summary: Add match support for `semantic_text` fields +area: "Search" +type: enhancement +issues: [] diff --git a/docs/changelog/117840.yaml b/docs/changelog/117840.yaml new file mode 100644 index 0000000000000..e1f469643af42 --- /dev/null +++ b/docs/changelog/117840.yaml @@ -0,0 +1,5 @@ +pr: 117840 +summary: Fix timeout ingesting an empty string into a `semantic_text` field +area: Machine Learning +type: bug +issues: [] diff --git a/docs/changelog/117851.yaml b/docs/changelog/117851.yaml new file mode 100644 index 0000000000000..21888cd6fb80f --- /dev/null +++ b/docs/changelog/117851.yaml @@ -0,0 +1,5 @@ +pr: 117851 +summary: Addition of `tier_preference`, `creation_date` and `version` fields in Elasticsearch monitoring template +area: Monitoring +type: enhancement +issues: [] diff --git a/docs/changelog/117858.yaml b/docs/changelog/117858.yaml new file mode 100644 index 0000000000000..70f12dc40027f --- /dev/null +++ b/docs/changelog/117858.yaml @@ -0,0 +1,5 @@ +pr: 117858 +summary: Create upgrade mode +area: Transform +type: enhancement +issues: [] diff --git a/docs/changelog/117898.yaml b/docs/changelog/117898.yaml new file mode 100644 index 0000000000000..c60061abc49ff --- /dev/null +++ b/docs/changelog/117898.yaml @@ -0,0 +1,5 @@ +pr: 117898 +summary: Limit size of query +area: ES|QL +type: bug +issues: [] diff --git a/docs/changelog/117917.yaml b/docs/changelog/117917.yaml new file mode 100644 index 0000000000000..b6dc90f6b903d --- /dev/null +++ b/docs/changelog/117917.yaml @@ -0,0 +1,5 @@ +pr: 117917 +summary: Add option to store `sparse_vector` outside `_source` +area: Mapping +type: feature +issues: [] diff --git a/docs/changelog/117939.yaml b/docs/changelog/117939.yaml new file mode 100644 index 0000000000000..d41111f099f97 --- /dev/null +++ b/docs/changelog/117939.yaml @@ -0,0 +1,5 @@ +pr: 117939 +summary: Adding default endpoint for Elastic Rerank +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/117949.yaml b/docs/changelog/117949.yaml new file mode 100644 index 0000000000000..b67f36a224094 --- /dev/null +++ b/docs/changelog/117949.yaml @@ -0,0 +1,5 @@ +pr: 117949 +summary: Move `SlowLogFieldProvider` instantiation to node construction +area: Infra/Logging +type: bug +issues: [] diff --git a/docs/changelog/117989.yaml b/docs/changelog/117989.yaml new file mode 100644 index 0000000000000..e4967141b3ebd --- /dev/null +++ b/docs/changelog/117989.yaml @@ -0,0 +1,5 @@ +pr: 117989 +summary: ESQL Add esql hash function +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/117994.yaml b/docs/changelog/117994.yaml new file mode 100644 index 0000000000000..603f2d855a11a --- /dev/null +++ b/docs/changelog/117994.yaml @@ -0,0 +1,5 @@ +pr: 117994 +summary: Even better(er) binary quantization +area: Vector Search +type: enhancement +issues: [] diff --git a/docs/changelog/118016.yaml b/docs/changelog/118016.yaml new file mode 100644 index 0000000000000..7ee78b901b199 --- /dev/null +++ b/docs/changelog/118016.yaml @@ -0,0 +1,6 @@ +pr: 118016 +summary: Propagate status codes from shard failures appropriately +area: Search +type: enhancement +issues: + - 118482 diff --git a/docs/changelog/118027.yaml b/docs/changelog/118027.yaml new file mode 100644 index 0000000000000..161c156b56a65 --- /dev/null +++ b/docs/changelog/118027.yaml @@ -0,0 +1,6 @@ +pr: 118027 +summary: Esql compare nanos and millis +area: ES|QL +type: enhancement +issues: + - 116281 diff --git a/docs/changelog/118035.yaml b/docs/changelog/118035.yaml new file mode 100644 index 0000000000000..fdeaa184723b9 --- /dev/null +++ b/docs/changelog/118035.yaml @@ -0,0 +1,6 @@ +pr: 118035 +summary: Include hidden indices in `DeprecationInfoAction` +area: Indices APIs +type: bug +issues: + - 118020 diff --git a/docs/changelog/118064.yaml b/docs/changelog/118064.yaml new file mode 100644 index 0000000000000..7d12f365bf142 --- /dev/null +++ b/docs/changelog/118064.yaml @@ -0,0 +1,5 @@ +pr: 118064 +summary: Add Highlighter for Semantic Text Fields +area: Highlighting +type: feature +issues: [] diff --git a/docs/changelog/118094.yaml b/docs/changelog/118094.yaml new file mode 100644 index 0000000000000..a8866543fa7d2 --- /dev/null +++ b/docs/changelog/118094.yaml @@ -0,0 +1,5 @@ +pr: 118094 +summary: Update ASM 9.7 -> 9.7.1 to support JDK 24 +area: Infra/Core +type: upgrade +issues: [] diff --git a/docs/changelog/118102.yaml b/docs/changelog/118102.yaml new file mode 100644 index 0000000000000..e5ec32cdddbec --- /dev/null +++ b/docs/changelog/118102.yaml @@ -0,0 +1,5 @@ +pr: 118102 +summary: "ESQL: Enterprise license enforcement for CCS" +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/118114.yaml b/docs/changelog/118114.yaml new file mode 100644 index 0000000000000..1b7532d5df981 --- /dev/null +++ b/docs/changelog/118114.yaml @@ -0,0 +1,5 @@ +pr: 118114 +summary: Enable physical plan verification +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/118135.yaml b/docs/changelog/118135.yaml new file mode 100644 index 0000000000000..e7e3e737cbfe2 --- /dev/null +++ b/docs/changelog/118135.yaml @@ -0,0 +1,5 @@ +pr: 118135 +summary: Backport Term query for ES|QL to 8.x +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/118143.yaml b/docs/changelog/118143.yaml new file mode 100644 index 0000000000000..4dcbf4b4b6c2c --- /dev/null +++ b/docs/changelog/118143.yaml @@ -0,0 +1,5 @@ +pr: 118143 +summary: Infrastructure for assuming cluster features in the next major version +area: "Infra/Core" +type: feature +issues: [] diff --git a/docs/changelog/118166.yaml b/docs/changelog/118166.yaml new file mode 100644 index 0000000000000..99e3fcafd5805 --- /dev/null +++ b/docs/changelog/118166.yaml @@ -0,0 +1,5 @@ +pr: 118166 +summary: Update minimum supported snapshot version for Machine Learning jobs to 8.3.0 +area: Machine Learning +type: upgrade +issues: [] diff --git a/docs/changelog/118173.yaml b/docs/changelog/118173.yaml new file mode 100644 index 0000000000000..a3c9054674ba5 --- /dev/null +++ b/docs/changelog/118173.yaml @@ -0,0 +1,5 @@ +pr: 118173 +summary: ES|QL categorize with multiple groupings +area: Machine Learning +type: feature +issues: [] diff --git a/docs/changelog/118177.yaml b/docs/changelog/118177.yaml new file mode 100644 index 0000000000000..5201fec3db306 --- /dev/null +++ b/docs/changelog/118177.yaml @@ -0,0 +1,6 @@ +pr: 118177 +summary: Fixing bedrock event executor terminated cache issue +area: Machine Learning +type: bug +issues: + - 117916 diff --git a/docs/changelog/118194.yaml b/docs/changelog/118194.yaml new file mode 100644 index 0000000000000..0e5eca55d597c --- /dev/null +++ b/docs/changelog/118194.yaml @@ -0,0 +1,5 @@ +pr: 118194 +summary: Retry on `ClusterBlockException` on transform destination index +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/118266.yaml b/docs/changelog/118266.yaml new file mode 100644 index 0000000000000..1b14b12b973c5 --- /dev/null +++ b/docs/changelog/118266.yaml @@ -0,0 +1,5 @@ +pr: 118266 +summary: Prevent data nodes from sending stack traces to coordinator when `error_trace=false` +area: Search +type: enhancement +issues: [] diff --git a/docs/changelog/118267.yaml b/docs/changelog/118267.yaml new file mode 100644 index 0000000000000..3e3920caeb0f9 --- /dev/null +++ b/docs/changelog/118267.yaml @@ -0,0 +1,5 @@ +pr: 118267 +summary: Adding get migration reindex status +area: Data streams +type: enhancement +issues: [] diff --git a/docs/changelog/118291.yaml b/docs/changelog/118291.yaml new file mode 100644 index 0000000000000..8001b3972e876 --- /dev/null +++ b/docs/changelog/118291.yaml @@ -0,0 +1,5 @@ +pr: 118291 +summary: Adding a migration reindex cancel API +area: Data streams +type: enhancement +issues: [] diff --git a/docs/changelog/118298.yaml b/docs/changelog/118298.yaml new file mode 100644 index 0000000000000..794add47bf0db --- /dev/null +++ b/docs/changelog/118298.yaml @@ -0,0 +1,10 @@ +pr: 118298 +summary: Enhancement/v7 critical deprecation logging +area: Infra/REST API +type: deprecation +issues: [] +deprecation: + title: V_7 deprecation logging set to critical + area: REST API + details: This changes the V_7 API deprecation logging level to CRITICAL. + impact: Any usage of deprecated V_7 API features will now be logged at the CRITICAL level. This does not change functionality. diff --git a/docs/changelog/118300.yaml b/docs/changelog/118300.yaml new file mode 100644 index 0000000000000..e11f5bfd73b2e --- /dev/null +++ b/docs/changelog/118300.yaml @@ -0,0 +1,5 @@ +pr: 118300 +summary: Upgrade to Lucene 9.12.1 +area: Search +type: upgrade +issues: [] diff --git a/docs/changelog/118353.yaml b/docs/changelog/118353.yaml new file mode 100644 index 0000000000000..7be62a4a60c7e --- /dev/null +++ b/docs/changelog/118353.yaml @@ -0,0 +1,5 @@ +pr: 118353 +summary: Epoch Millis Rounding Down and Not Up 2 +area: Infra/Core +type: bug +issues: [] diff --git a/docs/changelog/118369.yaml b/docs/changelog/118369.yaml new file mode 100644 index 0000000000000..6d2382712f57b --- /dev/null +++ b/docs/changelog/118369.yaml @@ -0,0 +1,5 @@ +pr: 118369 +summary: "[8.x] Update sparse text embeddings API route for Inference Service" +area: Inference +type: enhancement +issues: [] diff --git a/docs/changelog/118375.yaml b/docs/changelog/118375.yaml new file mode 100644 index 0000000000000..bad2751aeaa50 --- /dev/null +++ b/docs/changelog/118375.yaml @@ -0,0 +1,5 @@ +pr: 118375 +summary: Check for presence of error object when validating streaming responses from integrations in the inference API +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/118410.yaml b/docs/changelog/118410.yaml new file mode 100644 index 0000000000000..ccc7f71ee2e1c --- /dev/null +++ b/docs/changelog/118410.yaml @@ -0,0 +1,5 @@ +pr: 118410 +summary: Push down filter passed lookup join +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/118435.yaml b/docs/changelog/118435.yaml new file mode 100644 index 0000000000000..8bccbeb54698d --- /dev/null +++ b/docs/changelog/118435.yaml @@ -0,0 +1,6 @@ +pr: 118435 +summary: '`_score` should not be a reserved attribute in ES|QL' +area: ES|QL +type: enhancement +issues: + - 118460 diff --git a/docs/changelog/118454.yaml b/docs/changelog/118454.yaml new file mode 100644 index 0000000000000..9a19ede64d705 --- /dev/null +++ b/docs/changelog/118454.yaml @@ -0,0 +1,5 @@ +pr: 118454 +summary: Fix RLIKE folding with (unsupported) case insensitive pattern +area: ES|QL +type: bug +issues: [] diff --git a/docs/changelog/118474.yaml b/docs/changelog/118474.yaml new file mode 100644 index 0000000000000..1b0c6942eb323 --- /dev/null +++ b/docs/changelog/118474.yaml @@ -0,0 +1,6 @@ +pr: 118474 +summary: Esql bucket function for date nanos +area: ES|QL +type: enhancement +issues: + - 118031 diff --git a/docs/changelog/118516.yaml b/docs/changelog/118516.yaml new file mode 100644 index 0000000000000..8a618a6d6cfd7 --- /dev/null +++ b/docs/changelog/118516.yaml @@ -0,0 +1,6 @@ +pr: 118435 +summary: Fix moving function linear weighted avg +area: Aggregations +type: bug +issues: + - 113751 diff --git a/docs/changelog/118544.yaml b/docs/changelog/118544.yaml new file mode 100644 index 0000000000000..d59783c4e6194 --- /dev/null +++ b/docs/changelog/118544.yaml @@ -0,0 +1,5 @@ +pr: 118544 +summary: ESQL - Remove restrictions for disjunctions in full text functions +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/118585.yaml b/docs/changelog/118585.yaml new file mode 100644 index 0000000000000..4caa5efabbd33 --- /dev/null +++ b/docs/changelog/118585.yaml @@ -0,0 +1,7 @@ +pr: 118585 +summary: Add a generic `rescorer` retriever based on the search request's rescore + functionality +area: Ranking +type: feature +issues: + - 118327 diff --git a/docs/changelog/118603.yaml b/docs/changelog/118603.yaml new file mode 100644 index 0000000000000..d61619adfa5f6 --- /dev/null +++ b/docs/changelog/118603.yaml @@ -0,0 +1,6 @@ +pr: 118603 +summary: Allow DATE_PARSE to read the timezones +area: ES|QL +type: bug +issues: + - 117680 diff --git a/docs/changelog/118617.yaml b/docs/changelog/118617.yaml new file mode 100644 index 0000000000000..a8793a114e913 --- /dev/null +++ b/docs/changelog/118617.yaml @@ -0,0 +1,5 @@ +pr: 118617 +summary: Add support for `sparse_vector` queries against `semantic_text` fields +area: "Search" +type: enhancement +issues: [] diff --git a/docs/changelog/118674.yaml b/docs/changelog/118674.yaml new file mode 100644 index 0000000000000..eeb90a3b38f66 --- /dev/null +++ b/docs/changelog/118674.yaml @@ -0,0 +1,5 @@ +pr: 118674 +summary: Ignore failures from renormalizing buckets in read-only index +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/118681.yaml b/docs/changelog/118681.yaml new file mode 100644 index 0000000000000..a186c05e6cd7d --- /dev/null +++ b/docs/changelog/118681.yaml @@ -0,0 +1,6 @@ +pr: 118681 +summary: '`ConnectTransportException` returns retryable BAD_GATEWAY' +area: Network +type: enhancement +issues: + - 118320 diff --git a/docs/changelog/118697.yaml b/docs/changelog/118697.yaml new file mode 100644 index 0000000000000..6e24e6ae4b47f --- /dev/null +++ b/docs/changelog/118697.yaml @@ -0,0 +1,6 @@ +pr: 118697 +summary: Esql implicit casting for date nanos +area: ES|QL +type: enhancement +issues: + - 118476 diff --git a/docs/changelog/118704.yaml b/docs/changelog/118704.yaml new file mode 100644 index 0000000000000..c89735664f25e --- /dev/null +++ b/docs/changelog/118704.yaml @@ -0,0 +1,6 @@ +pr: 118704 +summary: Avoid updating settings version in `MetadataMigrateToDataStreamService` when + settings have not changed +area: Data streams +type: bug +issues: [] diff --git a/docs/changelog/118757.yaml b/docs/changelog/118757.yaml new file mode 100644 index 0000000000000..956e220f21aeb --- /dev/null +++ b/docs/changelog/118757.yaml @@ -0,0 +1,5 @@ +pr: 118757 +summary: Improve handling of nested fields in index reader wrappers +area: Authorization +type: enhancement +issues: [] diff --git a/docs/changelog/118774.yaml b/docs/changelog/118774.yaml new file mode 100644 index 0000000000000..cbd1ca82d1c59 --- /dev/null +++ b/docs/changelog/118774.yaml @@ -0,0 +1,5 @@ +pr: 118774 +summary: Apply default k for knn query eagerly +area: Vector Search +type: bug +issues: [] diff --git a/docs/changelog/118802.yaml b/docs/changelog/118802.yaml new file mode 100644 index 0000000000000..600c4b6a1e203 --- /dev/null +++ b/docs/changelog/118802.yaml @@ -0,0 +1,5 @@ +pr: 118802 +summary: ST_EXTENT_AGG optimize envelope extraction from doc-values for cartesian_shape +area: "ES|QL" +type: enhancement +issues: [] diff --git a/docs/changelog/118804.yaml b/docs/changelog/118804.yaml new file mode 100644 index 0000000000000..1548367a5485f --- /dev/null +++ b/docs/changelog/118804.yaml @@ -0,0 +1,15 @@ +pr: 118804 +summary: Add new experimental `rank_vectors` mapping for late-interaction second order + ranking +area: Vector Search +type: feature +issues: [] +highlight: + title: Add new experimental `rank_vectors` mapping for late-interaction second order + ranking + body: + Late-interaction models are powerful rerankers. While their size and overall + cost doesn't lend itself for HNSW indexing, utilizing them as second order reranking + can provide excellent boosts in relevance. The new `rank_vectors` mapping allows for rescoring + over new and novel multi-vector late-interaction models like ColBERT or ColPali. + notable: true diff --git a/docs/changelog/118816.yaml b/docs/changelog/118816.yaml new file mode 100644 index 0000000000000..f1c1eac90dbcf --- /dev/null +++ b/docs/changelog/118816.yaml @@ -0,0 +1,6 @@ +pr: 118816 +summary: Support flattened field with downsampling +area: Downsampling +type: bug +issues: + - 116319 diff --git a/docs/changelog/118823.yaml b/docs/changelog/118823.yaml new file mode 100644 index 0000000000000..b1afe1c873c17 --- /dev/null +++ b/docs/changelog/118823.yaml @@ -0,0 +1,5 @@ +pr: 118823 +summary: Fix attribute set equals +area: ES|QL +type: bug +issues: [] diff --git a/docs/changelog/118858.yaml b/docs/changelog/118858.yaml new file mode 100644 index 0000000000000..a2161df1c84c7 --- /dev/null +++ b/docs/changelog/118858.yaml @@ -0,0 +1,5 @@ +pr: 118858 +summary: Lookup join on multiple join fields not yet supported +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/118870.yaml b/docs/changelog/118870.yaml new file mode 100644 index 0000000000000..ce3692d5454ae --- /dev/null +++ b/docs/changelog/118870.yaml @@ -0,0 +1,6 @@ +pr: 118870 +summary: Rewrite TO_UPPER/TO_LOWER comparisons +area: ES|QL +type: enhancement +issues: + - 118304 diff --git a/docs/changelog/118890.yaml b/docs/changelog/118890.yaml new file mode 100644 index 0000000000000..d3fc17157f130 --- /dev/null +++ b/docs/changelog/118890.yaml @@ -0,0 +1,5 @@ +pr: 118890 +summary: Add action to create index from a source index +area: Data streams +type: enhancement +issues: [] diff --git a/docs/changelog/118919.yaml b/docs/changelog/118919.yaml new file mode 100644 index 0000000000000..832fd86fe08ba --- /dev/null +++ b/docs/changelog/118919.yaml @@ -0,0 +1,5 @@ +pr: 118919 +summary: Remove unsupported timeout from rest-api-spec license API +area: License +type: bug +issues: [] diff --git a/docs/changelog/118921.yaml b/docs/changelog/118921.yaml new file mode 100644 index 0000000000000..bd341616d8a14 --- /dev/null +++ b/docs/changelog/118921.yaml @@ -0,0 +1,5 @@ +pr: 118921 +summary: Add missing timeouts to rest-api-spec shutdown APIs +area: Infra/Node Lifecycle +type: bug +issues: [] diff --git a/docs/changelog/118931.yaml b/docs/changelog/118931.yaml new file mode 100644 index 0000000000000..81e9b3cb16521 --- /dev/null +++ b/docs/changelog/118931.yaml @@ -0,0 +1,6 @@ +pr: 118931 +summary: Add a `LicenseAware` interface for licensed Nodes +area: ES|QL +type: enhancement +issues: + - 117405 diff --git a/docs/changelog/118954.yaml b/docs/changelog/118954.yaml new file mode 100644 index 0000000000000..ab2f2cda5c11e --- /dev/null +++ b/docs/changelog/118954.yaml @@ -0,0 +1,5 @@ +pr: 118954 +summary: Add missing parameter to `xpack.info` rest-api-spec +area: Infra/REST API +type: bug +issues: [] diff --git a/docs/changelog/118958.yaml b/docs/changelog/118958.yaml new file mode 100644 index 0000000000000..fb0fd6388ab61 --- /dev/null +++ b/docs/changelog/118958.yaml @@ -0,0 +1,5 @@ +pr: 118958 +summary: Add missing timeouts to rest-api-spec SLM APIs +area: ILM+SLM +type: bug +issues: [] diff --git a/docs/changelog/118959.yaml b/docs/changelog/118959.yaml new file mode 100644 index 0000000000000..95a9c146ae672 --- /dev/null +++ b/docs/changelog/118959.yaml @@ -0,0 +1,5 @@ +pr: 118959 +summary: Allow kibana_system user to manage .reindexed-v8-internal.alerts indices +area: Authorization +type: enhancement +issues: [] diff --git a/docs/changelog/118999.yaml b/docs/changelog/118999.yaml new file mode 100644 index 0000000000000..0188cebbd7685 --- /dev/null +++ b/docs/changelog/118999.yaml @@ -0,0 +1,6 @@ +pr: 118999 +summary: Fix loss of context in the inference API for streaming APIs +area: Machine Learning +type: bug +issues: + - 119000 diff --git a/docs/changelog/119011.yaml b/docs/changelog/119011.yaml new file mode 100644 index 0000000000000..0dbb683ceb2fd --- /dev/null +++ b/docs/changelog/119011.yaml @@ -0,0 +1,5 @@ +pr: 119011 +summary: "Add support for knn vector queries on `semantic_text` fields" +area: Search +type: enhancement +issues: [] diff --git a/docs/changelog/119054.yaml b/docs/changelog/119054.yaml new file mode 100644 index 0000000000000..720f2e0ab02ed --- /dev/null +++ b/docs/changelog/119054.yaml @@ -0,0 +1,6 @@ +pr: 119054 +summary: "[Security Solution] allows `kibana_system` user to manage .reindexed-v8-*\ + \ Security Solution indices" +area: Authorization +type: enhancement +issues: [] diff --git a/docs/changelog/119097.yaml b/docs/changelog/119097.yaml new file mode 100644 index 0000000000000..711ddfc0edfcb --- /dev/null +++ b/docs/changelog/119097.yaml @@ -0,0 +1,6 @@ +pr: 119097 +summary: "[8.x] Update data stream deprecations warnings to new format and filter\ + \ sea…" +area: Data streams +type: enhancement +issues: [] diff --git a/docs/changelog/119131.yaml b/docs/changelog/119131.yaml new file mode 100644 index 0000000000000..2628b6184f90d --- /dev/null +++ b/docs/changelog/119131.yaml @@ -0,0 +1,5 @@ +pr: 119131 +summary: Expose BwC enrich cache setting in plugin +area: Ingest Node +type: bug +issues: [] diff --git a/docs/changelog/119134.yaml b/docs/changelog/119134.yaml new file mode 100644 index 0000000000000..c4aefac91c701 --- /dev/null +++ b/docs/changelog/119134.yaml @@ -0,0 +1,6 @@ +pr: 119134 +summary: Handle `index.mapping.ignore_malformed` in downsampling +area: Downsampling +type: bug +issues: + - 119075 diff --git a/docs/changelog/119233.yaml b/docs/changelog/119233.yaml new file mode 100644 index 0000000000000..ef89c011ce4f6 --- /dev/null +++ b/docs/changelog/119233.yaml @@ -0,0 +1,5 @@ +pr: 119233 +summary: Fixing `GetDatabaseConfigurationAction` response serialization +area: Ingest Node +type: bug +issues: [] diff --git a/docs/changelog/119265.yaml b/docs/changelog/119265.yaml new file mode 100644 index 0000000000000..296106b9c01c6 --- /dev/null +++ b/docs/changelog/119265.yaml @@ -0,0 +1,6 @@ +pr: 119265 +summary: Fix `AbstractShapeGeometryFieldMapperTests` +area: "ES|QL" +type: bug +issues: + - 119201 diff --git a/docs/changelog/119291.yaml b/docs/changelog/119291.yaml new file mode 100644 index 0000000000000..89a6b6118049c --- /dev/null +++ b/docs/changelog/119291.yaml @@ -0,0 +1,5 @@ +pr: 119291 +summary: Register mustache size limit setting +area: Infra/Scripting +type: bug +issues: [] diff --git a/docs/changelog/119296.yaml b/docs/changelog/119296.yaml new file mode 100644 index 0000000000000..f1a92cffb5131 --- /dev/null +++ b/docs/changelog/119296.yaml @@ -0,0 +1,6 @@ +pr: 119296 +summary: Fix writing for LOOKUP status +area: ES|QL +type: bug +issues: + - 119086 diff --git a/docs/changelog/119348.yaml b/docs/changelog/119348.yaml new file mode 100644 index 0000000000000..05d2d7499ba03 --- /dev/null +++ b/docs/changelog/119348.yaml @@ -0,0 +1,5 @@ +pr: 119348 +summary: Auto-migrate `max_page_search_size` +area: Transform +type: enhancement +issues: [] diff --git a/docs/changelog/119447.yaml b/docs/changelog/119447.yaml new file mode 100644 index 0000000000000..e5a39bbc81fc8 --- /dev/null +++ b/docs/changelog/119447.yaml @@ -0,0 +1,5 @@ +pr: 119447 +summary: "[8.x] Add missing timeouts to rest-api-spec SLM APIs" +area: ILM+SLM +type: bug +issues: [] diff --git a/docs/changelog/119474.yaml b/docs/changelog/119474.yaml new file mode 100644 index 0000000000000..e37561277d220 --- /dev/null +++ b/docs/changelog/119474.yaml @@ -0,0 +1,5 @@ +pr: 119474 +summary: "Add ES|QL cross-cluster query telemetry collection" +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/119476.yaml b/docs/changelog/119476.yaml new file mode 100644 index 0000000000000..c275e6965d4a1 --- /dev/null +++ b/docs/changelog/119476.yaml @@ -0,0 +1,6 @@ +pr: 119476 +summary: Fix TopN row size estimate +area: ES|QL +type: bug +issues: + - 106956 diff --git a/docs/changelog/119495.yaml b/docs/changelog/119495.yaml new file mode 100644 index 0000000000000..b3e8f7e79d984 --- /dev/null +++ b/docs/changelog/119495.yaml @@ -0,0 +1,5 @@ +pr: 119495 +summary: Add mapping for `event_name` for OTel logs +area: Data streams +type: enhancement +issues: [] diff --git a/docs/changelog/119504.yaml b/docs/changelog/119504.yaml new file mode 100644 index 0000000000000..f63e422face10 --- /dev/null +++ b/docs/changelog/119504.yaml @@ -0,0 +1,5 @@ +pr: 119504 +summary: Optimized index sorting for OTel logs +area: Data streams +type: enhancement +issues: [] diff --git a/docs/changelog/119516.yaml b/docs/changelog/119516.yaml new file mode 100644 index 0000000000000..06dd5168a0823 --- /dev/null +++ b/docs/changelog/119516.yaml @@ -0,0 +1,5 @@ +pr: 119516 +summary: "Fix: do not let `_resolve/cluster` hang if remote is unresponsive" +area: Search +type: bug +issues: [] diff --git a/docs/changelog/119601.yaml b/docs/changelog/119601.yaml new file mode 100644 index 0000000000000..3570588a5c690 --- /dev/null +++ b/docs/changelog/119601.yaml @@ -0,0 +1,13 @@ +pr: 119601 +summary: "[8.x] Add new experimental `rank_vectors` mapping for late-interaction second\ + \ order ranking" +area: Vector Search +type: feature +issues: [] +highlight: + title: "[8.x] Add new experimental `rank_vectors` mapping for late-interaction second\ + \ order ranking" + body: |- + Backports the following commits to 8.x: - Add new experimental + rank_vectors mapping for late-interaction second order ranking (#118804) + notable: true diff --git a/docs/changelog/119621.yaml b/docs/changelog/119621.yaml new file mode 100644 index 0000000000000..66c679572cfc0 --- /dev/null +++ b/docs/changelog/119621.yaml @@ -0,0 +1,5 @@ +pr: 119621 +summary: Enable node-level reduction by default +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/119637.yaml b/docs/changelog/119637.yaml new file mode 100644 index 0000000000000..c2fd6dc51f068 --- /dev/null +++ b/docs/changelog/119637.yaml @@ -0,0 +1,5 @@ +pr: 119637 +summary: Fix spike detection for short spikes at the tail of the data +area: Machine Learning +type: bug +issues: [] diff --git a/docs/internal/DistributedArchitectureGuide.md b/docs/internal/DistributedArchitectureGuide.md index 732e2e7be46fa..0114be68b9be2 100644 --- a/docs/internal/DistributedArchitectureGuide.md +++ b/docs/internal/DistributedArchitectureGuide.md @@ -252,7 +252,7 @@ changes. The cloud service will add more resources to the cluster based on Elast Elasticsearch by itself cannot automatically scale. Autoscaling recommendations are tailored for the user [based on user defined policies][], composed of data -roles (hot, frozen, etc) and [deciders][]. There's a public [webinar on autoscaling][], as well as the +roles (hot, frozen, etc.) and [deciders][]. There's a public [webinar on autoscaling][], as well as the public [Autoscaling APIs] docs. Autoscaling's current implementation is based primary on storage requirements, as well as memory capacity @@ -332,7 +332,7 @@ problems in the cluster. It uses [an algorithm defined here][]. Some examples ar [an algorithm defined here]: https://github.com/elastic/elasticsearch/blob/v8.13.2/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java#L158-L176 The `ProactiveStorageDeciderService` maintains a forecast window that [defaults to 30 minutes][]. It only -runs on data streams (ILM, rollover, etc), not regular indexes. It looks at past [index changes][] that +runs on data streams (ILM, rollover, etc.), not regular indexes. It looks at past [index changes][] that took place within the forecast window to [predict][] resources that will be needed shortly. [defaults to 30 minutes]: https://github.com/elastic/elasticsearch/blob/v8.13.2/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderService.java#L32 diff --git a/docs/internal/Versioning.md b/docs/internal/Versioning.md new file mode 100644 index 0000000000000..f0f730f618259 --- /dev/null +++ b/docs/internal/Versioning.md @@ -0,0 +1,297 @@ +Versioning Elasticsearch +======================== + +Elasticsearch is a complicated product, and is run in many different scenarios. +A single version number is not sufficient to cover the whole of the product, +instead we need different concepts to provide versioning capabilities +for different aspects of Elasticsearch, depending on their scope, updatability, +responsiveness, and maintenance. + +## Release version + +This is the version number used for published releases of Elasticsearch, +and the Elastic stack. This takes the form _major.minor.patch_, +with a corresponding version id. + +Uses of this version number should be avoided, as it does not apply to +some scenarios, and use of release version will break Elasticsearch nodes. + +The release version is accessible in code through `Build.current().version()`, +but it **should not** be assumed that this is a semantic version number, +it could be any arbitrary string. + +## Transport protocol + +The transport protocol is used to send binary data between Elasticsearch nodes; +`TransportVersion` is the version number used for this protocol. +This version number is negotiated between each pair of nodes in the cluster +on first connection, and is set as the lower of the highest transport version +understood by each node. +This version is then accessible through the `getTransportVersion` method +on `StreamInput` and `StreamOutput`, so serialization code can read/write +objects in a form that will be understood by the other node. + +Every change to the transport protocol is represented by a new transport version, +higher than all previous transport versions, which then becomes the highest version +recognized by that build of Elasticsearch. The version ids are stored +as constants in the `TransportVersions` class. +Each id has a standard pattern `M_NNN_SS_P`, where: +* `M` is the major version +* `NNN` is an incrementing id +* `SS` is used in subsidiary repos amending the default transport protocol +* `P` is used for patches and backports + +When you make a change to the serialization form of any object, +you need to create a new sequential constant in `TransportVersions`, +introduced in the same PR that adds the change, that increments +the `NNN` component from the previous highest version, +with other components set to zero. +For example, if the previous version number is `8_413_00_1`, +the next version number should be `8_414_00_0`. + +Once you have defined your constant, you then need to use it +in serialization code. If the transport version is at or above the new id, +the modified protocol should be used: + + str = in.readString(); + bool = in.readBoolean(); + if (in.getTransportVersion().onOrAfter(TransportVersions.NEW_CONSTANT)) { + num = in.readVInt(); + } + +If a transport version change needs to be reverted, a **new** version constant +should be added representing the revert, and the version id checks +adjusted appropriately to only use the modified protocol between the version id +the change was added, and the new version id used for the revert (exclusive). +The `between` method can be used for this. + +Once a transport change with a new version has been merged into main or a release branch, +it **must not** be modified - this is so the meaning of that specific +transport version does not change. + +_Elastic developers_ - please see corresponding documentation for Serverless +on creating transport versions for Serverless changes. + +### Collapsing transport versions + +As each change adds a new constant, the list of constants in `TransportVersions` +will keep growing. However, once there has been an official release of Elasticsearch, +that includes that change, that specific transport version is no longer needed, +apart from constants that happen to be used for release builds. +As part of managing transport versions, consecutive transport versions can be +periodically collapsed together into those that are only used for release builds. +This task is normally performed by Core/Infra on a semi-regular basis, +usually after each new minor release, to collapse the transport versions +for the previous minor release. An example of such an operation can be found +[here](https://github.com/elastic/elasticsearch/pull/104937). + +### Minimum compatibility versions + +The transport version used between two nodes is determined by the initial handshake +(see `TransportHandshaker`, where the two nodes swap their highest known transport version). +The lowest transport version that is compatible with the current node +is determined by `TransportVersions.MINIMUM_COMPATIBLE`, +and the node is prevented from joining the cluster if it is below that version. +This constant should be updated manually on a major release. + +The minimum version that can be used for CCS is determined by +`TransportVersions.MINIMUM_CCS_VERSION`, but this is not actively checked +before queries are performed. Only if a query cannot be serialized at that +version is an action rejected. This constant is updated automatically +as part of performing a release. + +### Mapping to release versions + +For releases that do use a version number, it can be confusing to encounter +a log or exception message that references an arbitrary transport version, +where you don't know which release version that corresponds to. This is where +the `.toReleaseVersion()` method comes in. It uses metadata stored in a csv file +(`TransportVersions.csv`) to map from the transport version id to the corresponding +release version. For any transport versions it encounters without a direct map, +it performs a best guess based on the information it has. The csv file +is updated automatically as part of performing a release. + +In releases that do not have a release version number, that method becomes +a no-op. + +### Managing patches and backports + +Backporting transport version changes to previous releases +should only be done if absolutely necessary, as it is very easy to get wrong +and break the release in a way that is very hard to recover from. + +If we consider the version number as an incrementing line, what we are doing is +grafting a change that takes effect at a certain point in the line, +to additionally take effect in a fixed window earlier in the line. + +To take an example, using indicative version numbers, when the latest +transport version is 52, we decide we need to backport a change done in +transport version 50 to transport version 45. We use the `P` version id component +to create version 45.1 with the backported change. +This change will apply for version ids 45.1 to 45.9 (should they exist in the future). + +The serialization code in the backport needs to use the backported protocol +for all version numbers 45.1 to 45.9. The `TransportVersion.isPatchFrom` method +can be used to easily determine if this is the case: `streamVersion.isPatchFrom(45.1)`. +However, the `onOrAfter` also does what is needed on patch branches. + +The serialization code in version 53 then needs to additionally check +version numbers 45.1-45.9 to use the backported protocol, also using the `isPatchFrom` method. + +As an example, [this transport change](https://github.com/elastic/elasticsearch/pull/107862) +was backported from 8.15 to [8.14.0](https://github.com/elastic/elasticsearch/pull/108251) +and [8.13.4](https://github.com/elastic/elasticsearch/pull/108250) at the same time +(8.14 was a build candidate at the time). + +The 8.13 PR has: + + if (transportVersion.onOrAfter(8.13_backport_id)) + +The 8.14 PR has: + + if (transportVersion.isPatchFrom(8.13_backport_id) + || transportVersion.onOrAfter(8.14_backport_id)) + +The 8.15 PR has: + + if (transportVersion.isPatchFrom(8.13_backport_id) + || transportVersion.isPatchFrom(8.14_backport_id) + || transportVersion.onOrAfter(8.15_transport_id)) + +In particular, if you are backporting a change to a patch release, +you also need to make sure that any subsequent released version on any branch +also has that change, and knows about the patch backport ids and what they mean. + +## Index version + +Index version is a single incrementing version number for the index data format, +metadata, and associated mappings. It is declared the same way as the +transport version - with the pattern `M_NNN_SS_P`, for the major version, version id, +subsidiary version id, and patch number respectively. + +Index version is stored in index metadata when an index is created, +and it is used to determine the storage format and what functionality that index supports. +The index version does not change once an index is created. + +In the same way as transport versions, when a change is needed to the index +data format or metadata, or new mapping types are added, create a new version constant +below the last one, incrementing the `NNN` version component. + +Unlike transport version, version constants cannot be collapsed together, +as an index keeps its creation version id once it is created. +Fortunately, new index versions are only created once a month or so, +so we don’t have a large list of index versions that need managing. + +Similar to transport version, index version has a `toReleaseVersion` to map +onto release versions, in appropriate situations. + +## Cluster Features + +Cluster features are identifiers, published by a node in cluster state, +indicating they support a particular top-level operation or set of functionality. +They are used for internal checks within Elasticsearch, and for gating tests +on certain functionality. For example, to check all nodes have upgraded +to a certain point before running a large migration operation to a new data format. +Cluster features should not be referenced by anything outside the Elasticsearch codebase. + +Cluster features are indicative of top-level functionality introduced to +Elasticsearch - e.g. a new transport endpoint, or new operations. + +It is also used to check nodes can join a cluster - once all nodes in a cluster +support a particular feature, no nodes can then join the cluster that do not +support that feature. This is to ensure that once a feature is supported +by a cluster, it will then always be supported in the future. + +To declare a new cluster feature, add an implementation of the `FeatureSpecification` SPI, +suitably registered (or use an existing one for your code area), and add the feature +as a constant to be returned by getFeatures. To then check whether all nodes +in the cluster support that feature, use the method `clusterHasFeature` on `FeatureService`. +It is only possible to check whether all nodes in the cluster have a feature; +individual node checks should not be done. + +Once a cluster feature is declared and deployed, it cannot be modified or removed, +else new nodes will not be able to join existing clusters. +If functionality represented by a cluster feature needs to be removed, +a new cluster feature should be added indicating that functionality is no longer +supported, and the code modified accordingly (bearing in mind additional BwC constraints). + +The cluster features infrastructure is only designed to support a few hundred features +per major release, and once features are added to a cluster they can not be removed. +Cluster features should therefore be used sparingly. +Adding too many cluster features risks increasing cluster instability. + +When we release a new major version N, we limit our backwards compatibility +to the highest minor of the previous major N-1. Therefore, any cluster formed +with the new major version is guaranteed to have all features introduced during +releases of major N-1. All such features can be deemed to be met by the cluster, +and the features themselves can be removed from cluster state over time, +and the feature checks removed from the code of major version N. + +### Testing + +Tests often want to check if a certain feature is implemented / available on all nodes, +particularly BwC or mixed cluster test. + +Rather than introducing a production feature just for a test condition, +this can be done by adding a _test feature_ in an implementation of +`FeatureSpecification.getTestFeatures`. These features will only be set +on clusters running as part of an integration test. Even so, cluster features +should be used sparingly if possible; Capabilities is generally a better +option for test conditions. + +In Java Rest tests, checking cluster features can be done using +`ESRestTestCase.clusterHasFeature(feature)` + +In YAML Rest tests, conditions can be defined in the `requires` or `skip` sections +that use cluster features; see [here](https://github.com/elastic/elasticsearch/blob/main/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/README.asciidoc#skipping-tests) for more information. + +To aid with backwards compatibility tests, the test framework adds synthetic features +for each previously released Elasticsearch version, of the form `gte_v{VERSION}` +(for example `gte_v8.14.2`). +This can be used to add conditions based on previous releases. It _cannot_ be used +to check the current snapshot version; real features or capabilities should be +used instead. + +## Capabilities + +The Capabilities API is a REST API for external clients to check the capabilities +of an Elasticsearch cluster. As it is dynamically calculated for every query, +it is not limited in size or usage. + +A capabilities query can be used to query for 3 things: +* Is this endpoint supported for this HTTP method? +* Are these parameters of this endpoint supported? +* Are these capabilities (arbitrary string ids) of this endpoint supported? + +The API will return with a simple true/false, indicating if all specified aspects +of the endpoint are supported by all nodes in the cluster. +If any aspect is not supported by any one node, the API returns `false`. + +The API can also return `supported: null` (indicating unknown) +if there was a problem communicating with one or more nodes in the cluster. + +All registered endpoints automatically work with the endpoint existence check. +To add support for parameter and feature capability queries to your REST endpoint, +implement the `supportedQueryParameters` and `supportedCapabilities` methods in your rest handler. + +To perform a capability query, perform a REST call to the `_capabilities` API, +with parameters `method`, `path`, `parameters`, `capabilities`. +The call will query every node in the cluster, and return `{supported: true}` +if all nodes support that specific combination of method, path, query parameters, +and endpoint capabilities. If any single aspect is not supported, +the query will return `{supported: false}`. If there are any problems +communicating with nodes in the cluster, the response will be `{supported: null}` +indicating support or lack thereof cannot currently be determined. +Capabilities can be checked using the clusterHasCapability method in ESRestTestCase. + +Similar to cluster features, YAML tests can have skip and requires conditions +specified with capabilities like the following: + + - requires: + capabilities: + - method: GET + path: /_endpoint + parameters: [param1, param2] + capabilities: [cap1, cap2] + +method: GET is the default, and does not need to be explicitly specified. diff --git a/docs/plugins/analysis-icu.asciidoc b/docs/plugins/analysis-icu.asciidoc index f6ca6ceae7ea4..da7efd2843f50 100644 --- a/docs/plugins/analysis-icu.asciidoc +++ b/docs/plugins/analysis-icu.asciidoc @@ -380,7 +380,7 @@ GET /my-index-000001/_search <3> -------------------------- -<1> The `name` field uses the `standard` analyzer, and so support full text queries. +<1> The `name` field uses the `standard` analyzer, and so supports full text queries. <2> The `name.sort` field is an `icu_collation_keyword` field that will preserve the name as a single token doc values, and applies the German ``phonebook'' order. <3> An example query which searches the `name` field and sorts on the `name.sort` field. @@ -467,7 +467,7 @@ differences. `case_first`:: Possible values: `lower` or `upper`. Useful to control which case is sorted -first when case is not ignored for strength `tertiary`. The default depends on +first when the case is not ignored for strength `tertiary`. The default depends on the collation. `numeric`:: diff --git a/docs/plugins/analysis-kuromoji.asciidoc b/docs/plugins/analysis-kuromoji.asciidoc index b1d1d5a751057..217d88f361223 100644 --- a/docs/plugins/analysis-kuromoji.asciidoc +++ b/docs/plugins/analysis-kuromoji.asciidoc @@ -86,7 +86,7 @@ The `kuromoji_iteration_mark` normalizes Japanese horizontal iteration marks `normalize_kanji`:: - Indicates whether kanji iteration marks should be normalize. Defaults to `true`. + Indicates whether kanji iteration marks should be normalized. Defaults to `true`. `normalize_kana`:: @@ -133,6 +133,11 @@ unknown words. It can be set to: Whether punctuation should be discarded from the output. Defaults to `true`. +`lenient`:: + + Whether the `user_dictionary` should be deduplicated on the provided `text`. + False by default causing duplicates to generate an error. + `user_dictionary`:: + -- @@ -189,7 +194,7 @@ PUT kuromoji_sample + -- Additional expert user parameters `nbest_cost` and `nbest_examples` can be used -to include additional tokens that most likely according to the statistical model. +to include additional tokens that are most likely according to the statistical model. If both parameters are used, the largest number of both is applied. `nbest_cost`:: @@ -221,7 +226,8 @@ PUT kuromoji_sample "type": "kuromoji_tokenizer", "mode": "extended", "discard_punctuation": "false", - "user_dictionary": "userdict_ja.txt" + "user_dictionary": "userdict_ja.txt", + "lenient": "true" } }, "analyzer": { @@ -744,3 +750,39 @@ Which results in: ] } -------------------------------------------------- + +[[analysis-kuromoji-completion]] +==== `kuromoji_completion` token filter + +The `kuromoji_completion` token filter adds Japanese romanized tokens to the term attributes along with the original tokens (surface forms). + +[source,console] +-------------------------------------------------- +GET _analyze +{ + "analyzer": "kuromoji_completion", + "text": "寿司" <1> +} +-------------------------------------------------- + +<1> Returns `寿司`, `susi` (Kunrei-shiki) and `sushi` (Hepburn-shiki). + +The `kuromoji_completion` token filter accepts the following settings: + +`mode`:: ++ +-- + +The tokenization mode determines how the tokenizer handles compound and +unknown words. It can be set to: + +`index`:: + + Simple romanization. Expected to be used when indexing. + +`query`:: + + Input Method aware romanization. Expected to be used when querying. + +Defaults to `index`. +-- diff --git a/docs/plugins/analysis-nori.asciidoc b/docs/plugins/analysis-nori.asciidoc index 1a3153fa3bea5..ebb32b7580c0c 100644 --- a/docs/plugins/analysis-nori.asciidoc +++ b/docs/plugins/analysis-nori.asciidoc @@ -58,6 +58,11 @@ It can be set to: Whether punctuation should be discarded from the output. Defaults to `true`. +`lenient`:: + + Whether the `user_dictionary` should be deduplicated on the provided `text`. + False by default causing duplicates to generate an error. + `user_dictionary`:: + -- @@ -104,7 +109,8 @@ PUT nori_sample "type": "nori_tokenizer", "decompound_mode": "mixed", "discard_punctuation": "false", - "user_dictionary": "userdict_ko.txt" + "user_dictionary": "userdict_ko.txt", + "lenient": "true" } }, "analyzer": { @@ -299,7 +305,6 @@ Which responds with: } -------------------------------------------------- - [[analysis-nori-speech]] ==== `nori_part_of_speech` token filter @@ -447,7 +452,7 @@ Which responds with: The `nori_number` token filter normalizes Korean numbers to regular Arabic decimal numbers in half-width characters. -Korean numbers are often written using a combination of Hangul and Arabic numbers with various kinds punctuation. +Korean numbers are often written using a combination of Hangul and Arabic numbers with various kinds of punctuation. For example, 3.2천 means 3200. This filter does this kind of normalization and allows a search for 3200 to match 3.2천 in text, but can also be used to make range facets based on the normalized numbers and so on. @@ -470,7 +475,7 @@ The input is untokenized text and the result is the single term attribute emitte - 영영칠 -> 7 - 일영영영 -> 1000 - 삼천2백2십삼 -> 3223 -- 조육백만오천일 -> 1000006005001 +- 일조육백만오천일 -> 1000006005001 - 3.2천 -> 3200 - 1.2만345.67 -> 12345.67 - 4,647.100 -> 4647.1 diff --git a/docs/plugins/development/creating-classic-plugins.asciidoc b/docs/plugins/development/creating-classic-plugins.asciidoc index cc03ad51275fa..58dc00e496c2d 100644 --- a/docs/plugins/development/creating-classic-plugins.asciidoc +++ b/docs/plugins/development/creating-classic-plugins.asciidoc @@ -18,7 +18,7 @@ will refuse to start in the presence of plugins with the incorrect [discrete] ==== Classic plugin file structure -Classis plugins are ZIP files composed of JAR files and +Classic plugins are ZIP files composed of JAR files and <>, a Java properties file that describes the plugin. diff --git a/docs/plugins/development/creating-stable-plugins.asciidoc b/docs/plugins/development/creating-stable-plugins.asciidoc index c9a8a1f6c7e2a..9f98774b5a761 100644 --- a/docs/plugins/development/creating-stable-plugins.asciidoc +++ b/docs/plugins/development/creating-stable-plugins.asciidoc @@ -1,8 +1,8 @@ [[creating-stable-plugins]] === Creating text analysis plugins with the stable plugin API -Text analysis plugins provide {es} with custom {ref}/analysis.html[Lucene -analyzers, token filters, character filters, and tokenizers]. +Text analysis plugins provide {es} with custom {ref}/analysis.html[Lucene +analyzers, token filters, character filters, and tokenizers]. [discrete] ==== The stable plugin API @@ -10,7 +10,7 @@ analyzers, token filters, character filters, and tokenizers]. Text analysis plugins can be developed against the stable plugin API. This API consists of the following dependencies: -* `plugin-api` - an API used by plugin developers to implement custom {es} +* `plugin-api` - an API used by plugin developers to implement custom {es} plugins. * `plugin-analysis-api` - an API used by plugin developers to implement analysis plugins and integrate them into {es}. @@ -18,7 +18,7 @@ plugins and integrate them into {es}. core Lucene analysis interfaces like `Tokenizer`, `Analyzer`, and `TokenStream`. For new versions of {es} within the same major version, plugins built against -this API do not need to be recompiled. Future versions of the API will be +this API does not need to be recompiled. Future versions of the API will be backwards compatible and plugins are binary compatible with future versions of {es}. In other words, once you have a working artifact, you can re-use it when you upgrade {es} to a new bugfix or minor version. @@ -48,9 +48,9 @@ require code changes. Stable plugins are ZIP files composed of JAR files and two metadata files: -* `stable-plugin-descriptor.properties` - a Java properties file that describes +* `stable-plugin-descriptor.properties` - a Java properties file that describes the plugin. Refer to <>. -* `named_components.json` - a JSON file mapping interfaces to key-value pairs +* `named_components.json` - a JSON file mapping interfaces to key-value pairs of component names and implementation classes. Note that only JAR files at the root of the plugin are added to the classpath @@ -65,7 +65,7 @@ you use this plugin. However, you don't need Gradle to create plugins. The {es} Github repository contains {es-repo}tree/main/plugins/examples/stable-analysis[an example analysis plugin]. -The example `build.gradle` build script provides a good starting point for +The example `build.gradle` build script provides a good starting point for developing your own plugin. [discrete] @@ -77,29 +77,29 @@ Plugins are written in Java, so you need to install a Java Development Kit [discrete] ===== Step by step -. Create a directory for your project. +. Create a directory for your project. . Copy the example `build.gradle` build script to your project directory. Note that this build script uses the `elasticsearch.stable-esplugin` gradle plugin to build your plugin. . Edit the `build.gradle` build script: -** Add a definition for the `pluginApiVersion` and matching `luceneVersion` -variables to the top of the file. You can find these versions in the -`build-tools-internal/version.properties` file in the {es-repo}[Elasticsearch +** Add a definition for the `pluginApiVersion` and matching `luceneVersion` +variables to the top of the file. You can find these versions in the +`build-tools-internal/version.properties` file in the {es-repo}[Elasticsearch Github repository]. -** Edit the `name` and `description` in the `esplugin` section of the build -script. This will create the plugin descriptor file. If you're not using the -`elasticsearch.stable-esplugin` gradle plugin, refer to +** Edit the `name` and `description` in the `esplugin` section of the build +script. This will create the plugin descriptor file. If you're not using the +`elasticsearch.stable-esplugin` gradle plugin, refer to <> to create the file manually. ** Add module information. -** Ensure you have declared the following compile-time dependencies. These -dependencies are compile-time only because {es} will provide these libraries at +** Ensure you have declared the following compile-time dependencies. These +dependencies are compile-time only because {es} will provide these libraries at runtime. *** `org.elasticsearch.plugin:elasticsearch-plugin-api` *** `org.elasticsearch.plugin:elasticsearch-plugin-analysis-api` *** `org.apache.lucene:lucene-analysis-common` -** For unit testing, ensure these dependencies have also been added to the +** For unit testing, ensure these dependencies have also been added to the `build.gradle` script as `testImplementation` dependencies. -. Implement an interface from the analysis plugin API, annotating it with +. Implement an interface from the analysis plugin API, annotating it with `NamedComponent`. Refer to <> for an example. . You should now be able to assemble a plugin ZIP file by running: + @@ -107,22 +107,22 @@ runtime. ---- gradle bundlePlugin ---- -The resulting plugin ZIP file is written to the `build/distributions` +The resulting plugin ZIP file is written to the `build/distributions` directory. [discrete] ===== YAML REST tests -The Gradle `elasticsearch.yaml-rest-test` plugin enables testing of your -plugin using the {es-repo}blob/main/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/README.asciidoc[{es} yamlRestTest framework]. +The Gradle `elasticsearch.yaml-rest-test` plugin enables testing of your +plugin using the {es-repo}blob/main/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/README.asciidoc[{es} yamlRestTest framework]. These tests use a YAML-formatted domain language to issue REST requests against -an internal {es} cluster that has your plugin installed, and to check the -results of those requests. The structure of a YAML REST test directory is as +an internal {es} cluster that has your plugin installed, and to check the +results of those requests. The structure of a YAML REST test directory is as follows: -* A test suite class, defined under `src/yamlRestTest/java`. This class should +* A test suite class, defined under `src/yamlRestTest/java`. This class should extend `ESClientYamlSuiteTestCase`. -* The YAML tests themselves should be defined under +* The YAML tests themselves should be defined under `src/yamlRestTest/resources/test/`. [[plugin-descriptor-file-stable]] diff --git a/docs/plugins/discovery-azure-classic.asciidoc b/docs/plugins/discovery-azure-classic.asciidoc index aa710a2fe7ef9..b8d37f024172c 100644 --- a/docs/plugins/discovery-azure-classic.asciidoc +++ b/docs/plugins/discovery-azure-classic.asciidoc @@ -148,7 +148,7 @@ Before starting, you need to have: -- You should follow http://azure.microsoft.com/en-us/documentation/articles/linux-use-ssh-key/[this guide] to learn -how to create or use existing SSH keys. If you have already did it, you can skip the following. +how to create or use existing SSH keys. If you have already done it, you can skip the following. Here is a description on how to generate SSH keys using `openssl`: diff --git a/docs/plugins/discovery-gce.asciidoc b/docs/plugins/discovery-gce.asciidoc index 2e8cff21208e0..0a2629b7f094b 100644 --- a/docs/plugins/discovery-gce.asciidoc +++ b/docs/plugins/discovery-gce.asciidoc @@ -478,7 +478,7 @@ discovery: seed_providers: gce -------------------------------------------------- -Replaces `project_id` and `zone` with your settings. +Replace `project_id` and `zone` with your settings. To run test: diff --git a/docs/plugins/integrations.asciidoc b/docs/plugins/integrations.asciidoc index 71f237692ad35..aff4aed0becd2 100644 --- a/docs/plugins/integrations.asciidoc +++ b/docs/plugins/integrations.asciidoc @@ -91,7 +91,7 @@ Integrations are not plugins, but are external tools or modules that make it eas Elasticsearch Grails plugin. * https://hibernate.org/search/[Hibernate Search] - Integration with Hibernate ORM, from the Hibernate team. Automatic synchronization of write operations, yet exposes full Elasticsearch capabilities for queries. Can return either Elasticsearch native or re-map queries back into managed entities loaded within transaction from the reference database. + Integration with Hibernate ORM, from the Hibernate team. Automatic synchronization of write operations, yet exposes full Elasticsearch capabilities for queries. Can return either Elasticsearch native or re-map queries back into managed entities loaded within transactions from the reference database. * https://github.com/spring-projects/spring-data-elasticsearch[Spring Data Elasticsearch]: Spring Data implementation for Elasticsearch @@ -104,7 +104,7 @@ Integrations are not plugins, but are external tools or modules that make it eas * https://pulsar.apache.org/docs/en/io-elasticsearch[Apache Pulsar]: The Elasticsearch Sink Connector is used to pull messages from Pulsar topics - and persist the messages to a index. + and persist the messages to an index. * https://micronaut-projects.github.io/micronaut-elasticsearch/latest/guide/index.html[Micronaut Elasticsearch Integration]: Integration of Micronaut with Elasticsearch diff --git a/docs/plugins/mapper-annotated-text.asciidoc b/docs/plugins/mapper-annotated-text.asciidoc index afe8ba41da9b8..956b6bedffff1 100644 --- a/docs/plugins/mapper-annotated-text.asciidoc +++ b/docs/plugins/mapper-annotated-text.asciidoc @@ -143,7 +143,7 @@ broader positional queries e.g. finding mentions of a `Guitarist` near to `strat WARNING: Any use of `=` signs in annotation values eg `[Prince](person=Prince)` will cause the document to be rejected with a parse failure. In future we hope to have a use for -the equals signs so wil actively reject documents that contain this today. +the equals signs so will actively reject documents that contain this today. [[annotated-text-synthetic-source]] ===== Synthetic `_source` @@ -155,11 +155,6 @@ be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features. -`annotated_text` fields support {ref}/mapping-source-field.html#synthetic-source[synthetic `_source`] if they have -a {ref}/keyword.html#keyword-synthetic-source[`keyword`] sub-field that supports synthetic -`_source` or if the `annotated_text` field sets `store` to `true`. Either way, it may -not have {ref}/copy-to.html[`copy_to`]. - If using a sub-`keyword` field then the values are sorted in the same way as a `keyword` field's values are sorted. By default, that means sorted with duplicates removed. So: @@ -167,8 +162,16 @@ duplicates removed. So: ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "text": { "type": "annotated_text", @@ -215,8 +218,16 @@ are preserved. ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "text": { "type": "annotated_text", "store": true } } diff --git a/docs/plugins/store-smb.asciidoc b/docs/plugins/store-smb.asciidoc index 8557ef868010f..da803b4f42022 100644 --- a/docs/plugins/store-smb.asciidoc +++ b/docs/plugins/store-smb.asciidoc @@ -10,7 +10,7 @@ include::install_remove.asciidoc[] ==== Working around a bug in Windows SMB and Java on windows When using a shared file system based on the SMB protocol (like Azure File Service) to store indices, the way Lucene -open index segment files is with a write only flag. This is the _correct_ way to open the files, as they will only be +opens index segment files is with a write only flag. This is the _correct_ way to open the files, as they will only be used for writes and allows different FS implementations to optimize for it. Sadly, in windows with SMB, this disables the cache manager, causing writes to be slow. This has been described in https://issues.apache.org/jira/browse/LUCENE-6176[LUCENE-6176], but it affects each and every Java program out there!. @@ -44,7 +44,7 @@ This can be configured for all indices by adding this to the `elasticsearch.yml` index.store.type: smb_nio_fs ---- -Note that setting will be applied for newly created indices. +Note that settings will be applied for newly created indices. It can also be set on a per-index basis at index creation time: diff --git a/docs/reference/aggregations/bucket/histogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/histogram-aggregation.asciidoc index 12f7cbb310d74..c2ae23dccb7dc 100644 --- a/docs/reference/aggregations/bucket/histogram-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/histogram-aggregation.asciidoc @@ -165,7 +165,7 @@ Example: POST /sales/_search?size=0 { "query": { - "constant_score": { "filter": { "range": { "price": { "to": "500" } } } } + "constant_score": { "filter": { "range": { "price": { "lte": "500" } } } } }, "aggs": { "prices": { @@ -202,7 +202,7 @@ Example: POST /sales/_search?size=0 { "query": { - "constant_score": { "filter": { "range": { "price": { "to": "500" } } } } + "constant_score": { "filter": { "range": { "price": { "lte": "500" } } } } }, "aggs": { "prices": { diff --git a/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc b/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc index c4397748debfd..23a690b62372d 100644 --- a/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc @@ -77,8 +77,8 @@ percentiles: `[ 1, 5, 25, 50, 75, 95, 99 ]`. The response will look like this: As you can see, the aggregation will return a calculated value for each percentile in the default range. If we assume response times are in milliseconds, it is -immediately obvious that the webpage normally loads in 10-725ms, but occasionally -spikes to 945-985ms. +immediately obvious that the webpage normally loads in 10-720ms, but occasionally +spikes to 940-980ms. Often, administrators are only interested in outliers -- the extreme percentiles. We can specify just the percents we are interested in (requested percentiles diff --git a/docs/reference/aggregations/pipeline/percentiles-bucket-aggregation.asciidoc b/docs/reference/aggregations/pipeline/percentiles-bucket-aggregation.asciidoc index 658470c8d5a4e..d5bd868258081 100644 --- a/docs/reference/aggregations/pipeline/percentiles-bucket-aggregation.asciidoc +++ b/docs/reference/aggregations/pipeline/percentiles-bucket-aggregation.asciidoc @@ -127,10 +127,11 @@ And the following may be the response: ==== Percentiles_bucket implementation -The Percentile Bucket returns the nearest input data point that is not greater than the requested percentile; it does not -interpolate between data points. - The percentiles are calculated exactly and is not an approximation (unlike the Percentiles Metric). This means the implementation maintains an in-memory, sorted list of your data to compute the percentiles, before discarding the data. You may run into memory pressure issues if you attempt to calculate percentiles over many millions of data-points in a single `percentiles_bucket`. + +The Percentile Bucket returns the nearest input data point to the requested percentile, rounding indices toward +positive infinity; it does not interpolate between data points. For example, if there are eight data points and +you request the `50%th` percentile, it will return the `4th` item because `ROUND_UP(.50 * (8-1))` is `4`. diff --git a/docs/reference/alias.asciidoc b/docs/reference/alias.asciidoc index 9d784f530d63c..f676644c4ec48 100644 --- a/docs/reference/alias.asciidoc +++ b/docs/reference/alias.asciidoc @@ -407,3 +407,24 @@ POST _aliases } ---- // TEST[s/^/PUT my-index-2099.05.06-000001\n/] + +[discrete] +[[remove-index]] +=== Remove an index + +To remove an index, use the aliases API's `remove_index` action. + +[source,console] +---- +POST _aliases +{ + "actions": [ + { + "remove_index": { + "index": "my-index-2099.05.06-000001" + } + } + ] +} +---- +// TEST[s/^/PUT my-index-2099.05.06-000001\n/] diff --git a/docs/reference/analysis.asciidoc b/docs/reference/analysis.asciidoc index 72ab42d22b911..e8fbc3bd81b6d 100644 --- a/docs/reference/analysis.asciidoc +++ b/docs/reference/analysis.asciidoc @@ -9,8 +9,7 @@ -- _Text analysis_ is the process of converting unstructured text, like -the body of an email or a product description, into a structured format that's -optimized for search. +the body of an email or a product description, into a structured format that's <>. [discrete] [[when-to-configure-analysis]] diff --git a/docs/reference/analysis/tokenfilters/hyphenation-decompounder-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/hyphenation-decompounder-tokenfilter.asciidoc index eed66d81e9132..1bd36f801aa17 100644 --- a/docs/reference/analysis/tokenfilters/hyphenation-decompounder-tokenfilter.asciidoc +++ b/docs/reference/analysis/tokenfilters/hyphenation-decompounder-tokenfilter.asciidoc @@ -111,6 +111,18 @@ output. Defaults to `5`. (Optional, Boolean) If `true`, only include the longest matching subword. Defaults to `false`. +`no_sub_matches`:: +(Optional, Boolean) +If `true`, do not match sub tokens in tokens that are in the word list. +Defaults to `false`. + +`no_overlapping_matches`:: +(Optional, Boolean) +If `true`, do not allow overlapping tokens. +Defaults to `false`. + +Typically users will only want to include one of the three flags as enabling `no_overlapping_matches` is the most restrictive and `no_sub_matches` is more restrictive than `only_longest_match`. When enabling a more restrictive option the state of the less restrictive does not have any effect. + [[analysis-hyp-decomp-tokenfilter-customize]] ==== Customize and add to an analyzer diff --git a/docs/reference/analysis/tokenfilters/snowball-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/snowball-tokenfilter.asciidoc index 57e402988cc5a..d8300288c9f4b 100644 --- a/docs/reference/analysis/tokenfilters/snowball-tokenfilter.asciidoc +++ b/docs/reference/analysis/tokenfilters/snowball-tokenfilter.asciidoc @@ -11,6 +11,8 @@ values: `Arabic`, `Armenian`, `Basque`, `Catalan`, `Danish`, `Dutch`, `English`, `Lithuanian`, `Lovins`, `Norwegian`, `Porter`, `Portuguese`, `Romanian`, `Russian`, `Serbian`, `Spanish`, `Swedish`, `Turkish`. +deprecated:[8.16.0, `Kp` and `Lovins` support will be removed in a future version] + For example: [source,console] @@ -28,7 +30,7 @@ PUT /my-index-000001 "filter": { "my_snow": { "type": "snowball", - "language": "Lovins" + "language": "English" } } } diff --git a/docs/reference/analysis/tokenfilters/stemmer-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/stemmer-tokenfilter.asciidoc index 42ac594fca3bf..4cd088935af19 100644 --- a/docs/reference/analysis/tokenfilters/stemmer-tokenfilter.asciidoc +++ b/docs/reference/analysis/tokenfilters/stemmer-tokenfilter.asciidoc @@ -144,12 +144,12 @@ https://snowballstem.org/algorithms/danish/stemmer.html[*`danish`*] Dutch:: https://snowballstem.org/algorithms/dutch/stemmer.html[*`dutch`*], -https://snowballstem.org/algorithms/kraaij_pohlmann/stemmer.html[`dutch_kp`] +https://snowballstem.org/algorithms/kraaij_pohlmann/stemmer.html[`dutch_kp`] deprecated:[8.16.0, `dutch_kp` will be removed in a future version] English:: https://snowballstem.org/algorithms/porter/stemmer.html[*`english`*], https://ciir.cs.umass.edu/pubfiles/ir-35.pdf[`light_english`], -https://snowballstem.org/algorithms/lovins/stemmer.html[`lovins`], +https://snowballstem.org/algorithms/lovins/stemmer.html[`lovins`] deprecated:[8.16.0, `lovins` will be removed in a future version], https://www.researchgate.net/publication/220433848_How_effective_is_suffixing[`minimal_english`], https://snowballstem.org/algorithms/english/stemmer.html[`porter2`], {lucene-analysis-docs}/en/EnglishPossessiveFilter.html[`possessive_english`] diff --git a/docs/reference/analysis/tokenizers.asciidoc b/docs/reference/analysis/tokenizers.asciidoc index 38e4ebfcabc39..89928f07b5638 100644 --- a/docs/reference/analysis/tokenizers.asciidoc +++ b/docs/reference/analysis/tokenizers.asciidoc @@ -1,6 +1,14 @@ [[analysis-tokenizers]] == Tokenizer reference +.Difference between {es} tokenization and neural tokenization +[NOTE] +==== +{es}'s tokenization process produces linguistic tokens, optimized for search and retrieval. +This differs from neural tokenization in the context of machine learning and natural language processing. Neural tokenizers translate strings into smaller, subword tokens, which are encoded into vectors for consumptions by neural networks. +{es} does not have built-in neural tokenizers. +==== + A _tokenizer_ receives a stream of characters, breaks it up into individual _tokens_ (usually individual words), and outputs a stream of _tokens_. For instance, a <> tokenizer breaks diff --git a/docs/reference/api-conventions.asciidoc b/docs/reference/api-conventions.asciidoc index f8d925945401e..28933eb75050d 100644 --- a/docs/reference/api-conventions.asciidoc +++ b/docs/reference/api-conventions.asciidoc @@ -28,7 +28,7 @@ You can pass an `X-Opaque-Id` HTTP header to track the origin of a request in * Response of any request that includes the header * <<_identifying_running_tasks,Task management API>> response -* <<_identifying_search_slow_log_origin,Slow logs>> +* <> * <> For the deprecation logs, {es} also uses the `X-Opaque-Id` value to throttle @@ -52,7 +52,7 @@ safely generate a unique `traceparent` header for each request. If provided, {es} surfaces the header's `trace-id` value as `trace.id` in the: * <> -* <<_identifying_search_slow_log_origin,Slow logs>> +* <> * <> For example, the following `traceparent` value would produce the following diff --git a/docs/reference/autoscaling/apis/autoscaling-apis.asciidoc b/docs/reference/autoscaling/apis/autoscaling-apis.asciidoc index e4da2c45ee978..87de3818bfaf8 100644 --- a/docs/reference/autoscaling/apis/autoscaling-apis.asciidoc +++ b/docs/reference/autoscaling/apis/autoscaling-apis.asciidoc @@ -4,6 +4,13 @@ NOTE: {cloud-only} + +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-autoscaling[Autoscaling APIs]. +-- + You can use the following APIs to perform {cloud}/ec-autoscaling.html[autoscaling operations]. [discrete] diff --git a/docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc b/docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc index 190428485a003..349e40aab0540 100644 --- a/docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc +++ b/docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc @@ -7,6 +7,12 @@ NOTE: {cloud-only} +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-autoscaling[Autoscaling APIs]. +-- + Delete {cloud}/ec-autoscaling.html[autoscaling] policy. [[autoscaling-delete-autoscaling-policy-request]] diff --git a/docs/reference/autoscaling/apis/get-autoscaling-capacity.asciidoc b/docs/reference/autoscaling/apis/get-autoscaling-capacity.asciidoc index d635d8c8f7bd0..d45f7cbacc242 100644 --- a/docs/reference/autoscaling/apis/get-autoscaling-capacity.asciidoc +++ b/docs/reference/autoscaling/apis/get-autoscaling-capacity.asciidoc @@ -7,6 +7,12 @@ NOTE: {cloud-only} +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-autoscaling[Autoscaling APIs]. +-- + Get {cloud}/ec-autoscaling.html[autoscaling] capacity. [[autoscaling-get-autoscaling-capacity-request]] diff --git a/docs/reference/autoscaling/apis/get-autoscaling-policy.asciidoc b/docs/reference/autoscaling/apis/get-autoscaling-policy.asciidoc index 973eedcb361c9..9962b266fb662 100644 --- a/docs/reference/autoscaling/apis/get-autoscaling-policy.asciidoc +++ b/docs/reference/autoscaling/apis/get-autoscaling-policy.asciidoc @@ -7,6 +7,13 @@ NOTE: {cloud-only} +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-autoscaling[Autoscaling APIs]. +-- + + Get {cloud}/ec-autoscaling.html[autoscaling] policy. [[autoscaling-get-autoscaling-policy-request]] diff --git a/docs/reference/autoscaling/apis/put-autoscaling-policy.asciidoc b/docs/reference/autoscaling/apis/put-autoscaling-policy.asciidoc index e564f83411eb4..97c6a54fab03a 100644 --- a/docs/reference/autoscaling/apis/put-autoscaling-policy.asciidoc +++ b/docs/reference/autoscaling/apis/put-autoscaling-policy.asciidoc @@ -7,6 +7,12 @@ NOTE: {cloud-only} +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-autoscaling[Autoscaling APIs]. +-- + Creates or updates an {cloud}/ec-autoscaling.html[autoscaling] policy. [[autoscaling-put-autoscaling-policy-request]] diff --git a/docs/reference/behavioral-analytics/apis/delete-analytics-collection.asciidoc b/docs/reference/behavioral-analytics/apis/delete-analytics-collection.asciidoc index a6894a933b460..19c1b5437ef0c 100644 --- a/docs/reference/behavioral-analytics/apis/delete-analytics-collection.asciidoc +++ b/docs/reference/behavioral-analytics/apis/delete-analytics-collection.asciidoc @@ -8,6 +8,12 @@ beta::[] Delete Analytics Collection ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-analytics[Behavioral analytics APIs]. +-- + //// [source,console] ---- diff --git a/docs/reference/behavioral-analytics/apis/index.asciidoc b/docs/reference/behavioral-analytics/apis/index.asciidoc index 692d3374f89f5..6dc12599c2297 100644 --- a/docs/reference/behavioral-analytics/apis/index.asciidoc +++ b/docs/reference/behavioral-analytics/apis/index.asciidoc @@ -9,6 +9,12 @@ beta::[] --- +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-analytics[Behavioral analytics APIs]. +-- + Use the following APIs to manage tasks and resources related to <>: * <> diff --git a/docs/reference/behavioral-analytics/apis/list-analytics-collection.asciidoc b/docs/reference/behavioral-analytics/apis/list-analytics-collection.asciidoc index 14511a1258278..46ee8296f3ebe 100644 --- a/docs/reference/behavioral-analytics/apis/list-analytics-collection.asciidoc +++ b/docs/reference/behavioral-analytics/apis/list-analytics-collection.asciidoc @@ -8,6 +8,12 @@ beta::[] List Analytics Collections ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-analytics[Behavioral analytics APIs]. +-- + //// [source,console] ---- diff --git a/docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc b/docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc index f82717e22ed34..60985cd50d3d1 100644 --- a/docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc +++ b/docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc @@ -8,6 +8,12 @@ beta::[] Post Analytics Collection Event ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-analytics[Behavioral analytics APIs]. +-- + //// [source,console] ---- diff --git a/docs/reference/behavioral-analytics/apis/put-analytics-collection.asciidoc b/docs/reference/behavioral-analytics/apis/put-analytics-collection.asciidoc index cbbab2ae3e26c..412277afa2076 100644 --- a/docs/reference/behavioral-analytics/apis/put-analytics-collection.asciidoc +++ b/docs/reference/behavioral-analytics/apis/put-analytics-collection.asciidoc @@ -8,6 +8,12 @@ beta::[] Put Analytics Collection ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-analytics[Behavioral analytics APIs]. +-- + //// [source,console] ---- diff --git a/docs/reference/cat.asciidoc b/docs/reference/cat.asciidoc index 328970e1d9dc4..61ec9f7680f7f 100644 --- a/docs/reference/cat.asciidoc +++ b/docs/reference/cat.asciidoc @@ -1,6 +1,12 @@ [[cat]] == Compact and aligned text (CAT) APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs]. +-- + ["float",id="intro"] === Introduction diff --git a/docs/reference/cat/alias.asciidoc b/docs/reference/cat/alias.asciidoc index b6459eaa93b82..b5ab8ad937c1e 100644 --- a/docs/reference/cat/alias.asciidoc +++ b/docs/reference/cat/alias.asciidoc @@ -4,6 +4,12 @@ cat aliases ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs].. +-- + [IMPORTANT] ==== cat APIs are only intended for human consumption using the command line or the diff --git a/docs/reference/cat/allocation.asciidoc b/docs/reference/cat/allocation.asciidoc index 0891406d1be4b..a94c1f1959663 100644 --- a/docs/reference/cat/allocation.asciidoc +++ b/docs/reference/cat/allocation.asciidoc @@ -4,6 +4,12 @@ cat allocation ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs].. +-- + [IMPORTANT] ==== cat APIs are only intended for human consumption using the command line or {kib} diff --git a/docs/reference/cat/anomaly-detectors.asciidoc b/docs/reference/cat/anomaly-detectors.asciidoc index 3416c256881af..350917170a5da 100644 --- a/docs/reference/cat/anomaly-detectors.asciidoc +++ b/docs/reference/cat/anomaly-detectors.asciidoc @@ -5,6 +5,12 @@ cat anomaly detectors ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs].. +-- + [IMPORTANT] ==== cat APIs are only intended for human consumption using the command line or {kib} diff --git a/docs/reference/cat/component-templates.asciidoc b/docs/reference/cat/component-templates.asciidoc index 596c86befd1b7..4e905173d7dcc 100644 --- a/docs/reference/cat/component-templates.asciidoc +++ b/docs/reference/cat/component-templates.asciidoc @@ -4,10 +4,16 @@ cat component templates ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs].. +-- + [IMPORTANT] ==== -cat APIs are only intended for human consumption using the command line or {kib} -console. They are _not_ intended for use by applications. For application +cat APIs are only intended for human consumption using the command line or {kib} +console. They are _not_ intended for use by applications. For application consumption, use the <>. ==== @@ -82,8 +88,7 @@ PUT _component_template/my-template-2 "type": "keyword" }, "created_at": { - "type": "date", - "format": "EEE MMM dd HH:mm:ss Z yyyy" + "type": "date" } } } diff --git a/docs/reference/cat/count.asciidoc b/docs/reference/cat/count.asciidoc index 37e602c759020..7adcd1464dab1 100644 --- a/docs/reference/cat/count.asciidoc +++ b/docs/reference/cat/count.asciidoc @@ -4,6 +4,12 @@ cat count ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs].. +-- + [IMPORTANT] ==== cat APIs are only intended for human consumption using the command line or {kib} diff --git a/docs/reference/cat/datafeeds.asciidoc b/docs/reference/cat/datafeeds.asciidoc index 9b6481191e59d..44a2ccd2faa80 100644 --- a/docs/reference/cat/datafeeds.asciidoc +++ b/docs/reference/cat/datafeeds.asciidoc @@ -5,6 +5,12 @@ cat {dfeeds} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs].. +-- + [IMPORTANT] ==== cat APIs are only intended for human consumption using the command line or {kib} diff --git a/docs/reference/cat/dataframeanalytics.asciidoc b/docs/reference/cat/dataframeanalytics.asciidoc index 4c236ecf61ffc..4d57160f171bf 100644 --- a/docs/reference/cat/dataframeanalytics.asciidoc +++ b/docs/reference/cat/dataframeanalytics.asciidoc @@ -5,6 +5,12 @@ cat {dfanalytics} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs].. +-- + [IMPORTANT] ==== cat APIs are only intended for human consumption using the command line or {kib} diff --git a/docs/reference/cat/fielddata.asciidoc b/docs/reference/cat/fielddata.asciidoc index 376ef1d97057c..f11e40263ec2b 100644 --- a/docs/reference/cat/fielddata.asciidoc +++ b/docs/reference/cat/fielddata.asciidoc @@ -4,6 +4,12 @@ cat fielddata ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs].. +-- + [IMPORTANT] ==== cat APIs are only intended for human consumption using the command line or {kib} diff --git a/docs/reference/cat/health.asciidoc b/docs/reference/cat/health.asciidoc index ad39ace310807..7ffc170ec8515 100644 --- a/docs/reference/cat/health.asciidoc +++ b/docs/reference/cat/health.asciidoc @@ -4,6 +4,12 @@ cat health ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs].. +-- + [IMPORTANT] ==== cat APIs are only intended for human consumption using the command line or {kib} diff --git a/docs/reference/cat/indices.asciidoc b/docs/reference/cat/indices.asciidoc index 64b90c4f8e353..01771847555bb 100644 --- a/docs/reference/cat/indices.asciidoc +++ b/docs/reference/cat/indices.asciidoc @@ -4,10 +4,16 @@ cat indices ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs].. +-- + [IMPORTANT] ==== -cat APIs are only intended for human consumption using the command line or {kib} -console. They are _not_ intended for use by applications. For application +cat APIs are only intended for human consumption using the command line or {kib} +console. They are _not_ intended for use by applications. For application consumption, use the <>. ==== @@ -50,6 +56,10 @@ indexing and search. As a result, all document counts include hidden To get an accurate count of {es} documents, use the <> or <> APIs. +Note that information such as document count, deleted document count and store size are not shown for +indices restored from <> since these indices +do not contain the relevant data structures to retrieve this information from. + [[cat-indices-api-path-params]] ==== {api-path-parms-title} diff --git a/docs/reference/cat/master.asciidoc b/docs/reference/cat/master.asciidoc index 42348fc4939df..422651061f021 100644 --- a/docs/reference/cat/master.asciidoc +++ b/docs/reference/cat/master.asciidoc @@ -4,6 +4,12 @@ cat master ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs].. +-- + [IMPORTANT] ==== cat APIs are only intended for human consumption using the command line or {kib} diff --git a/docs/reference/cat/nodeattrs.asciidoc b/docs/reference/cat/nodeattrs.asciidoc index 2db0c3fc20279..71a63442df522 100644 --- a/docs/reference/cat/nodeattrs.asciidoc +++ b/docs/reference/cat/nodeattrs.asciidoc @@ -4,6 +4,12 @@ cat nodeattrs ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs].. +-- + [IMPORTANT] ==== cat APIs are only intended for human consumption using the command line or {kib} diff --git a/docs/reference/cat/nodes.asciidoc b/docs/reference/cat/nodes.asciidoc index fc5b01f9234e3..591dd63d8ef91 100644 --- a/docs/reference/cat/nodes.asciidoc +++ b/docs/reference/cat/nodes.asciidoc @@ -5,6 +5,12 @@ cat nodes ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs].. +-- + [IMPORTANT] ==== cat APIs are only intended for human consumption using the command line or {kib} @@ -50,16 +56,16 @@ Valid columns are: (Default) IP address, such as `127.0.1.1`. `heap.percent`, `hp`, `heapPercent`:: -(Default) Maximum configured heap, such as `7`. +(Default) Used percentage of total allocated Elasticsearch JVM heap, such as `7`. This reflects only the {es} process running within the operating system and is the most direct indicator of its JVM/heap/memory resource performance. `heap.max`, `hm`, `heapMax`:: -(Default) Total heap, such as `4gb`. +Total heap, such as `4gb`. `ram.percent`, `rp`, `ramPercent`:: -(Default) Used total memory percentage, such as `47`. +(Default) Used percentage of total operating system's memory, such as `47`. This reflects all processes running on operating system instead of only {es} and is not guaranteed to correlate to its performance. `file_desc.percent`, `fdp`, `fileDescriptorPercent`:: -(Default) Used file descriptors percentage, such as `1`. +Used file descriptors percentage, such as `1`. `node.role`, `r`, `role`, `nodeRole`:: (Default) Roles of the node. @@ -138,16 +144,16 @@ Used file descriptors, such as `123`. Maximum number of file descriptors, such as `1024`. `cpu`:: -Recent system CPU usage as percent, such as `12`. +(Default) Recent system CPU usage as percent, such as `12`. `load_1m`, `l`:: -Most recent load average, such as `0.22`. +(Default) Most recent load average, such as `0.22`. `load_5m`, `l`:: -Load average for the last five minutes, such as `0.78`. +(Default) Load average for the last five minutes, such as `0.78`. `load_15m`, `l`:: -Load average for the last fifteen minutes, such as `1.24`. +(Default) Load average for the last fifteen minutes, such as `1.24`. `uptime`, `u`:: Node uptime, such as `17.3m`. diff --git a/docs/reference/cat/pending_tasks.asciidoc b/docs/reference/cat/pending_tasks.asciidoc index 5dd6cb0688145..a40eb00b21d0d 100644 --- a/docs/reference/cat/pending_tasks.asciidoc +++ b/docs/reference/cat/pending_tasks.asciidoc @@ -4,6 +4,12 @@ cat pending tasks ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs].. +-- + [IMPORTANT] ==== cat APIs are only intended for human consumption using the command line or {kib} diff --git a/docs/reference/cat/plugins.asciidoc b/docs/reference/cat/plugins.asciidoc index a812556887b74..dffbebcada299 100644 --- a/docs/reference/cat/plugins.asciidoc +++ b/docs/reference/cat/plugins.asciidoc @@ -4,6 +4,11 @@ cat plugins ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs].. +-- [IMPORTANT] ==== diff --git a/docs/reference/cat/recovery.asciidoc b/docs/reference/cat/recovery.asciidoc index c3292fc9971ee..cda551fd0a1d0 100644 --- a/docs/reference/cat/recovery.asciidoc +++ b/docs/reference/cat/recovery.asciidoc @@ -4,6 +4,12 @@ cat recovery ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs].. +-- + [IMPORTANT] ==== cat APIs are only intended for human consumption using the command line or {kib} diff --git a/docs/reference/cat/repositories.asciidoc b/docs/reference/cat/repositories.asciidoc index ec2f243c27bec..7e6283336e17a 100644 --- a/docs/reference/cat/repositories.asciidoc +++ b/docs/reference/cat/repositories.asciidoc @@ -4,6 +4,12 @@ cat repositories ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs].. +-- + [IMPORTANT] ==== cat APIs are only intended for human consumption using the command line or {kib} diff --git a/docs/reference/cat/segments.asciidoc b/docs/reference/cat/segments.asciidoc index 872af679642d0..f7f1df542535e 100644 --- a/docs/reference/cat/segments.asciidoc +++ b/docs/reference/cat/segments.asciidoc @@ -4,6 +4,12 @@ cat segments ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs].. +-- + [IMPORTANT] ==== cat APIs are only intended for human consumption using the command line or {kib} diff --git a/docs/reference/cat/shards.asciidoc b/docs/reference/cat/shards.asciidoc index a2f8541be4abc..5e77f2ef3360e 100644 --- a/docs/reference/cat/shards.asciidoc +++ b/docs/reference/cat/shards.asciidoc @@ -5,6 +5,12 @@ cat shards ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs].. +-- + [IMPORTANT] ==== cat APIs are only intended for human consumption using the command line or {kib} @@ -30,7 +36,7 @@ For data streams, the API returns information about the stream's backing indices * If the {es} {security-features} are enabled, you must have the `monitor` or `manage` <> to use this API. You must also have the `monitor` or `manage` <> -for any data stream, index, or alias you retrieve. +to view the full information for any data stream, index, or alias you retrieve. [[cat-shards-path-params]] ==== {api-path-parms-title} diff --git a/docs/reference/cat/snapshots.asciidoc b/docs/reference/cat/snapshots.asciidoc index 820c4b56c783d..1da739b20272f 100644 --- a/docs/reference/cat/snapshots.asciidoc +++ b/docs/reference/cat/snapshots.asciidoc @@ -4,6 +4,12 @@ cat snapshots ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs].. +-- + [IMPORTANT] ==== cat APIs are only intended for human consumption using the command line or {kib} diff --git a/docs/reference/cat/tasks.asciidoc b/docs/reference/cat/tasks.asciidoc index 91d67baa72d70..ff654b30de992 100644 --- a/docs/reference/cat/tasks.asciidoc +++ b/docs/reference/cat/tasks.asciidoc @@ -6,6 +6,12 @@ beta::["The cat task management API is new and should still be considered a beta feature. The API may change in ways that are not backwards compatible.",{es-issue}51628] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs].. +-- + [IMPORTANT] ==== cat APIs are only intended for human consumption using the command line or {kib} diff --git a/docs/reference/cat/templates.asciidoc b/docs/reference/cat/templates.asciidoc index bcc8e9e4f5dc4..78ff60c663d2f 100644 --- a/docs/reference/cat/templates.asciidoc +++ b/docs/reference/cat/templates.asciidoc @@ -4,6 +4,12 @@ cat templates ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs].. +-- + [IMPORTANT] ==== cat APIs are only intended for human consumption using the command line or {kib} diff --git a/docs/reference/cat/thread_pool.asciidoc b/docs/reference/cat/thread_pool.asciidoc index 948ed9a1a7a30..1d8517f170aed 100644 --- a/docs/reference/cat/thread_pool.asciidoc +++ b/docs/reference/cat/thread_pool.asciidoc @@ -4,6 +4,12 @@ cat thread pool ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs].. +-- + [IMPORTANT] ==== cat APIs are only intended for human consumption using the command line or {kib} diff --git a/docs/reference/cat/trainedmodel.asciidoc b/docs/reference/cat/trainedmodel.asciidoc index 74e83525ddfe1..cbb3204450bcd 100644 --- a/docs/reference/cat/trainedmodel.asciidoc +++ b/docs/reference/cat/trainedmodel.asciidoc @@ -5,6 +5,12 @@ cat trained model ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs].. +-- + [IMPORTANT] ==== cat APIs are only intended for human consumption using the command line or {kib} diff --git a/docs/reference/cat/transforms.asciidoc b/docs/reference/cat/transforms.asciidoc index 0d4e9b691ac5a..caa814a5e3979 100644 --- a/docs/reference/cat/transforms.asciidoc +++ b/docs/reference/cat/transforms.asciidoc @@ -5,6 +5,12 @@ cat transforms ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cat[Compact and aligned text (CAT) APIs].. +-- + [IMPORTANT] ==== cat APIs are only intended for human consumption using the command line or {kib} diff --git a/docs/reference/ccr/apis/auto-follow/delete-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/delete-auto-follow-pattern.asciidoc index b510163bab50b..2ad50d68b923a 100644 --- a/docs/reference/ccr/apis/auto-follow/delete-auto-follow-pattern.asciidoc +++ b/docs/reference/ccr/apis/auto-follow/delete-auto-follow-pattern.asciidoc @@ -5,6 +5,12 @@ Delete auto-follow pattern ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ccr[Cross-cluster replication APIs]. +-- + Delete {ccr} <>. [[ccr-delete-auto-follow-pattern-request]] diff --git a/docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc index a2969e993ddfb..951185d14e920 100644 --- a/docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc +++ b/docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc @@ -5,6 +5,12 @@ Get auto-follow pattern ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ccr[Cross-cluster replication APIs]. +-- + Get {ccr} <>. [[ccr-get-auto-follow-pattern-request]] diff --git a/docs/reference/ccr/apis/auto-follow/pause-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/pause-auto-follow-pattern.asciidoc index c5ae5a7b4af9d..462ee213ed4e4 100644 --- a/docs/reference/ccr/apis/auto-follow/pause-auto-follow-pattern.asciidoc +++ b/docs/reference/ccr/apis/auto-follow/pause-auto-follow-pattern.asciidoc @@ -5,6 +5,12 @@ Pause auto-follow pattern ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ccr[Cross-cluster replication APIs]. +-- + Pauses a {ccr} <>. [[ccr-pause-auto-follow-pattern-request]] diff --git a/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc index 6769f21ca5cef..672a11302fdd5 100644 --- a/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc +++ b/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc @@ -5,6 +5,12 @@ Create auto-follow pattern ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ccr[Cross-cluster replication APIs]. +-- + Creates a {ccr} <>. [[ccr-put-auto-follow-pattern-request]] diff --git a/docs/reference/ccr/apis/auto-follow/resume-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/resume-auto-follow-pattern.asciidoc index a580bb3838f9b..d97c41d67c1ea 100644 --- a/docs/reference/ccr/apis/auto-follow/resume-auto-follow-pattern.asciidoc +++ b/docs/reference/ccr/apis/auto-follow/resume-auto-follow-pattern.asciidoc @@ -5,6 +5,12 @@ Resume auto-follow pattern ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ccr[Cross-cluster replication APIs]. +-- + Resumes a {ccr} <>. [[ccr-resume-auto-follow-pattern-request]] diff --git a/docs/reference/ccr/apis/ccr-apis.asciidoc b/docs/reference/ccr/apis/ccr-apis.asciidoc index ae94e1931af85..0decb98197d31 100644 --- a/docs/reference/ccr/apis/ccr-apis.asciidoc +++ b/docs/reference/ccr/apis/ccr-apis.asciidoc @@ -2,6 +2,12 @@ [[ccr-apis]] == {ccr-cap} APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ccr[Cross-cluster replication APIs]. +-- + You can use the following APIs to perform <> operations. [discrete] diff --git a/docs/reference/ccr/apis/follow/get-follow-info.asciidoc b/docs/reference/ccr/apis/follow/get-follow-info.asciidoc index 6c049d9c92b59..4c05faa0a7db8 100644 --- a/docs/reference/ccr/apis/follow/get-follow-info.asciidoc +++ b/docs/reference/ccr/apis/follow/get-follow-info.asciidoc @@ -5,6 +5,12 @@ Get follower info ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ccr[Cross-cluster replication APIs]. +-- + Retrieves information about all <> follower indices. [[ccr-get-follow-info-request]] diff --git a/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc b/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc index 4892f86b3523d..29000a98f64b4 100644 --- a/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc +++ b/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc @@ -5,6 +5,12 @@ Get follower stats ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ccr[Cross-cluster replication APIs]. +-- + Get <> follower stats. [[ccr-get-follow-stats-request]] diff --git a/docs/reference/ccr/apis/follow/post-forget-follower.asciidoc b/docs/reference/ccr/apis/follow/post-forget-follower.asciidoc index 1917c08d6640d..8a7887072f6a2 100644 --- a/docs/reference/ccr/apis/follow/post-forget-follower.asciidoc +++ b/docs/reference/ccr/apis/follow/post-forget-follower.asciidoc @@ -5,6 +5,12 @@ Forget follower ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ccr[Cross-cluster replication APIs]. +-- + Removes the <> follower retention leases from the leader. [[ccr-post-forget-follower-request]] diff --git a/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc b/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc index 6d4730d10efe6..c49e9a49b56c9 100644 --- a/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc +++ b/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc @@ -5,6 +5,12 @@ Pause follower ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ccr[Cross-cluster replication APIs]. +-- + Pauses a <> follower index. [[ccr-post-pause-follow-request]] diff --git a/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc b/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc index b023a8cb5cb70..f6da0110d5c24 100644 --- a/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc +++ b/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc @@ -5,6 +5,12 @@ Resume follower ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ccr[Cross-cluster replication APIs]. +-- + Resumes a <> follower index. [[ccr-post-resume-follow-request]] diff --git a/docs/reference/ccr/apis/follow/post-unfollow.asciidoc b/docs/reference/ccr/apis/follow/post-unfollow.asciidoc index dab11ef9e7a54..56b3195e8a134 100644 --- a/docs/reference/ccr/apis/follow/post-unfollow.asciidoc +++ b/docs/reference/ccr/apis/follow/post-unfollow.asciidoc @@ -5,6 +5,12 @@ Unfollow ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ccr[Cross-cluster replication APIs]. +-- + Converts a <> follower index to a regular index. [[ccr-post-unfollow-request]] diff --git a/docs/reference/ccr/apis/follow/put-follow.asciidoc b/docs/reference/ccr/apis/follow/put-follow.asciidoc index b7ae9ac987474..d9538fc436563 100644 --- a/docs/reference/ccr/apis/follow/put-follow.asciidoc +++ b/docs/reference/ccr/apis/follow/put-follow.asciidoc @@ -5,6 +5,12 @@ Create follower ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ccr[Cross-cluster replication APIs]. +-- + Creates a <> follower index. [[ccr-put-follow-request]] diff --git a/docs/reference/ccr/apis/get-ccr-stats.asciidoc b/docs/reference/ccr/apis/get-ccr-stats.asciidoc index 92e6bae0bdce8..e92ad17e10437 100644 --- a/docs/reference/ccr/apis/get-ccr-stats.asciidoc +++ b/docs/reference/ccr/apis/get-ccr-stats.asciidoc @@ -6,6 +6,12 @@ Get {ccr-init} stats ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ccr[Cross-cluster replication APIs]. +-- + Get <> stats. [[ccr-get-stats-request]] diff --git a/docs/reference/cluster.asciidoc b/docs/reference/cluster.asciidoc index 3de386d3288c6..b4359c2bf4fbb 100644 --- a/docs/reference/cluster.asciidoc +++ b/docs/reference/cluster.asciidoc @@ -1,6 +1,12 @@ [[cluster]] == Cluster APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs]. +-- + ["float",id="cluster-nodes"] === Node specification diff --git a/docs/reference/cluster/allocation-explain.asciidoc b/docs/reference/cluster/allocation-explain.asciidoc index 7547dd74c5ecd..f5d7be7f52f30 100644 --- a/docs/reference/cluster/allocation-explain.asciidoc +++ b/docs/reference/cluster/allocation-explain.asciidoc @@ -4,6 +4,12 @@ Cluster allocation explain ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs]. +-- + Provides an explanation for a shard's current <>. [source,console] @@ -159,10 +165,13 @@ node. <5> The decider which led to the `no` decision for the node. <6> An explanation as to why the decider returned a `no` decision, with a helpful hint pointing to the setting that led to the decision. In this example, a newly created index has <> that requires that it only be allocated to a node named `nonexistent_node`, which does not exist, so the index is unable to allocate. +See https://www.youtube.com/watch?v=5z3n2VgusLE[this video] for a walkthrough of troubleshooting a node and index setting mismatch. + +[[maximum-number-of-retries-exceeded]] ====== Maximum number of retries exceeded The following response contains an allocation explanation for an unassigned -primary shard that has reached the maximum number of allocation retry attempts. +primary shard that has reached the maximum number of allocation retry attempts. [source,js] ---- @@ -195,7 +204,7 @@ primary shard that has reached the maximum number of allocation retry attempts. { "decider": "max_retry", "decision" : "NO", - "explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2024-07-30T21:04:12.166Z], failed_attempts[5], failed_nodes[[mEKjwwzLT1yJVb8UxT6anw]], delayed=false, details[failed shard on node [mEKjwwzLT1yJVb8UxT6anw]: failed recovery, failure RecoveryFailedException], allocation_status[deciders_no]]]" + "explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [POST /_cluster/reroute?retry_failed&metric=none] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2024-07-30T21:04:12.166Z], failed_attempts[5], failed_nodes[[mEKjwwzLT1yJVb8UxT6anw]], delayed=false, details[failed shard on node [mEKjwwzLT1yJVb8UxT6anw]: failed recovery, failure RecoveryFailedException], allocation_status[deciders_no]]]" } ] } @@ -203,9 +212,11 @@ primary shard that has reached the maximum number of allocation retry attempts. } ---- // NOTCONSOLE - -If decider message indicates a transient allocation issue, use -<> to retry allocation. +When Elasticsearch is unable to allocate a shard, it will attempt to retry allocation up to +the maximum number of retries allowed. After this, Elasticsearch will stop attempting to +allocate the shard in order to prevent infinite retries which may impact cluster +performance. Run the <> API to retry allocation, which +will allocate the shard if the issue preventing allocation has been resolved. ====== No valid shard copy @@ -231,7 +242,9 @@ primary shard that was previously allocated. ---- // NOTCONSOLE -TIP: If a shard is unassigned with an allocation status of `no_valid_shard_copy`, then you should <>. If all the nodes containing in-sync copies of a shard are lost, then you can <>. +If a shard is unassigned with an allocation status of `no_valid_shard_copy`, then you should <>. If all the nodes containing in-sync copies of a shard are lost, then you can <>. + +See https://www.youtube.com/watch?v=6OAg9IyXFO4[this video] for a walkthrough of troubleshooting `no_valid_shard_copy`. ===== Unassigned replica shard @@ -334,7 +347,7 @@ queued to allocate but currently waiting on other queued shards. ---- // NOTCONSOLE -This is a transient message that might appear when a large amount of shards are allocating. +This is a transient message that might appear when a large amount of shards are allocating. ===== Assigned shard @@ -437,7 +450,7 @@ cluster balance. ===== No arguments If you call the API with no arguments, {es} retrieves an allocation explanation -for an arbitrary unassigned primary or replica shard, returning any unassigned primary shards first. +for an arbitrary unassigned primary or replica shard, returning any unassigned primary shards first. [source,console] ---- diff --git a/docs/reference/cluster/cluster-info.asciidoc b/docs/reference/cluster/cluster-info.asciidoc index 7d67f1602aeaa..52ae637d8f46c 100644 --- a/docs/reference/cluster/cluster-info.asciidoc +++ b/docs/reference/cluster/cluster-info.asciidoc @@ -7,6 +7,12 @@ experimental::[] Cluster Info ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs]. +-- + Returns cluster information. [[cluster-info-api-request]] diff --git a/docs/reference/cluster/delete-desired-balance.asciidoc b/docs/reference/cluster/delete-desired-balance.asciidoc index c67834269e505..45fa147258b78 100644 --- a/docs/reference/cluster/delete-desired-balance.asciidoc +++ b/docs/reference/cluster/delete-desired-balance.asciidoc @@ -6,6 +6,12 @@ NOTE: {cloud-only} +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs]. +-- + Discards the current <> and computes a new desired balance starting from the current allocation of shards. This can sometimes help {es} find a desired balance which needs fewer shard movements to achieve, especially if the cluster has experienced changes so substantial that the current desired balance is no longer optimal without {es} having diff --git a/docs/reference/cluster/delete-desired-nodes.asciidoc b/docs/reference/cluster/delete-desired-nodes.asciidoc index a58d19e2dfa3f..883bc22f6d964 100644 --- a/docs/reference/cluster/delete-desired-nodes.asciidoc +++ b/docs/reference/cluster/delete-desired-nodes.asciidoc @@ -6,6 +6,12 @@ NOTE: {cloud-only} +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs]. +-- + Delete desired nodes. [[delete-desired-nodes-request]] diff --git a/docs/reference/cluster/get-desired-balance.asciidoc b/docs/reference/cluster/get-desired-balance.asciidoc index 74afdaa52daf1..3c6b1dc48719c 100644 --- a/docs/reference/cluster/get-desired-balance.asciidoc +++ b/docs/reference/cluster/get-desired-balance.asciidoc @@ -6,6 +6,12 @@ NOTE: {cloud-only} +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs]. +-- + Exposes: * the <> computation and reconciliation stats diff --git a/docs/reference/cluster/get-desired-nodes.asciidoc b/docs/reference/cluster/get-desired-nodes.asciidoc index de27bd657b3ff..56af6913e34ac 100644 --- a/docs/reference/cluster/get-desired-nodes.asciidoc +++ b/docs/reference/cluster/get-desired-nodes.asciidoc @@ -6,6 +6,12 @@ NOTE: {cloud-only} +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs]. +-- + Get desired nodes. [[get-desired-nodes-request]] diff --git a/docs/reference/cluster/get-settings.asciidoc b/docs/reference/cluster/get-settings.asciidoc index 32c186e4ef24c..5c0fe7a2026c7 100644 --- a/docs/reference/cluster/get-settings.asciidoc +++ b/docs/reference/cluster/get-settings.asciidoc @@ -4,6 +4,12 @@ Cluster get settings ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs]. +-- + Returns cluster-wide settings. [source,console] diff --git a/docs/reference/cluster/health.asciidoc b/docs/reference/cluster/health.asciidoc index 94eb80a03d12e..374dd5d4a6f82 100644 --- a/docs/reference/cluster/health.asciidoc +++ b/docs/reference/cluster/health.asciidoc @@ -4,6 +4,12 @@ Cluster health ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs]. +-- + Returns the health status of a cluster. [[cluster-health-api-request]] diff --git a/docs/reference/cluster/nodes-hot-threads.asciidoc b/docs/reference/cluster/nodes-hot-threads.asciidoc index 446a8fede69ec..f8b414453ae66 100644 --- a/docs/reference/cluster/nodes-hot-threads.asciidoc +++ b/docs/reference/cluster/nodes-hot-threads.asciidoc @@ -4,6 +4,12 @@ Nodes hot threads ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs]. +-- + Returns the hot threads on each selected node in the cluster. [[cluster-nodes-hot-threads-api-request]] diff --git a/docs/reference/cluster/nodes-info.asciidoc b/docs/reference/cluster/nodes-info.asciidoc index 6f1d769e696c5..7ae6db7aa9a56 100644 --- a/docs/reference/cluster/nodes-info.asciidoc +++ b/docs/reference/cluster/nodes-info.asciidoc @@ -4,8 +4,13 @@ Nodes info ++++ -Returns cluster nodes information. +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs]. +-- +Returns cluster nodes information. [[cluster-nodes-info-api-request]] ==== {api-request-title} diff --git a/docs/reference/cluster/nodes-reload-secure-settings.asciidoc b/docs/reference/cluster/nodes-reload-secure-settings.asciidoc index f86304d2e9ba0..842ca30c335f9 100644 --- a/docs/reference/cluster/nodes-reload-secure-settings.asciidoc +++ b/docs/reference/cluster/nodes-reload-secure-settings.asciidoc @@ -4,6 +4,12 @@ Nodes reload secure settings ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs]. +-- + Reloads the keystore on nodes in the cluster. [[cluster-nodes-reload-secure-settings-api-request]] diff --git a/docs/reference/cluster/nodes-stats.asciidoc b/docs/reference/cluster/nodes-stats.asciidoc index 61c58cea95b83..522983035079d 100644 --- a/docs/reference/cluster/nodes-stats.asciidoc +++ b/docs/reference/cluster/nodes-stats.asciidoc @@ -5,6 +5,12 @@ Nodes stats ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs]. +-- + Returns cluster nodes statistics. [[cluster-nodes-stats-api-request]] @@ -1716,6 +1722,10 @@ See <> for more information about disk watermarks a `io_stats` (Linux only):: (objects) Contains I/O statistics for the node. + +NOTE: These statistics are derived from the `/proc/diskstats` kernel interface. +This interface accounts for IO performed by all processes on the system, even +if you are running {es} within a container. + .Properties of `io_stats` [%collapsible%open] diff --git a/docs/reference/cluster/nodes-usage.asciidoc b/docs/reference/cluster/nodes-usage.asciidoc index 486edf67bba87..c7994e32204a6 100644 --- a/docs/reference/cluster/nodes-usage.asciidoc +++ b/docs/reference/cluster/nodes-usage.asciidoc @@ -4,8 +4,13 @@ Nodes feature usage ++++ -Returns information on the usage of features. +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs]. +-- +Returns information on the usage of features. [[cluster-nodes-usage-api-request]] ==== {api-request-title} diff --git a/docs/reference/cluster/pending.asciidoc b/docs/reference/cluster/pending.asciidoc index 3e87234c7d26c..f5d42a6df76a6 100644 --- a/docs/reference/cluster/pending.asciidoc +++ b/docs/reference/cluster/pending.asciidoc @@ -4,6 +4,12 @@ Pending cluster tasks ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs]. +-- + Returns cluster-level changes that have not yet been executed. diff --git a/docs/reference/cluster/prevalidate-node-removal.asciidoc b/docs/reference/cluster/prevalidate-node-removal.asciidoc index 16bf28c586687..0a09f1adda77c 100644 --- a/docs/reference/cluster/prevalidate-node-removal.asciidoc +++ b/docs/reference/cluster/prevalidate-node-removal.asciidoc @@ -6,6 +6,12 @@ NOTE: {cloud-only} +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs]. +-- + Prevalidate node removal. [[prevalidate-node-removal-api-request]] diff --git a/docs/reference/cluster/remote-info.asciidoc b/docs/reference/cluster/remote-info.asciidoc index 8f2923846df0e..691acafd8ddbe 100644 --- a/docs/reference/cluster/remote-info.asciidoc +++ b/docs/reference/cluster/remote-info.asciidoc @@ -4,8 +4,13 @@ Remote cluster info ++++ -Returns configured remote cluster information. +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs]. +-- +Returns configured remote cluster information. [[cluster-remote-info-api-request]] ==== {api-request-title} diff --git a/docs/reference/cluster/reroute.asciidoc b/docs/reference/cluster/reroute.asciidoc index b4e4809ae73b4..217a0b6ccd3de 100644 --- a/docs/reference/cluster/reroute.asciidoc +++ b/docs/reference/cluster/reroute.asciidoc @@ -4,8 +4,13 @@ Cluster reroute ++++ -Changes the allocation of shards in a cluster. +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs]. +-- +Changes the allocation of shards in a cluster. [[cluster-reroute-api-request]] ==== {api-request-title} diff --git a/docs/reference/cluster/state.asciidoc b/docs/reference/cluster/state.asciidoc index fcb2f5f2f5dcd..bf2863018893a 100644 --- a/docs/reference/cluster/state.asciidoc +++ b/docs/reference/cluster/state.asciidoc @@ -4,6 +4,12 @@ Cluster state ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs]. +-- + Returns an internal representation of the cluster state for debugging or diagnostic purposes. diff --git a/docs/reference/cluster/stats.asciidoc b/docs/reference/cluster/stats.asciidoc index 575a6457804a6..f078fd2b7f2ee 100644 --- a/docs/reference/cluster/stats.asciidoc +++ b/docs/reference/cluster/stats.asciidoc @@ -5,6 +5,12 @@ Cluster stats ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs]. +-- + Returns cluster statistics. [[cluster-stats-api-request]] @@ -19,7 +25,6 @@ Returns cluster statistics. * If the {es} {security-features} are enabled, you must have the `monitor` or `manage` <> to use this API. - [[cluster-stats-api-desc]] ==== {api-description-title} @@ -40,6 +45,10 @@ If a node does not respond before its timeout expires, the response does not inc However, timed out nodes are included in the response's `_nodes.failed` property. Defaults to no timeout. +`include_remotes`:: +(Optional, Boolean) If `true`, includes remote cluster information in the response. +Defaults to `false`, so no remote cluster information is returned. + [role="child_attributes"] [[cluster-stats-api-response-body]] ==== {api-response-body-title} @@ -183,12 +192,11 @@ This number is based on documents in Lucene segments and may include documents f This number is based on documents in Lucene segments. {es} reclaims the disk space of deleted Lucene documents when a segment is merged. `total_size_in_bytes`:: -(integer) -Total size in bytes across all primary shards assigned to selected nodes. +(integer) Total size in bytes across all primary shards assigned to selected nodes. `total_size`:: -(string) -Total size across all primary shards assigned to selected nodes, as a human-readable string. +(string) Total size across all primary shards assigned to selected nodes, as a human-readable string. + ===== `store`:: @@ -429,6 +437,15 @@ To get information on segment files, use the <> repositories defined in the cluster, broken down -by repository type. +(object) Contains statistics about the <> repositories defined in the cluster, broken down by repository type. + .Properties of `repositories` [%collapsible%open] @@ -1314,13 +1334,74 @@ Each repository type may also include other statistics about the repositories of [%collapsible%open] ===== +`clusters`::: +(object) Contains remote cluster settings and metrics collected from them. +The keys are cluster names, and the values are per-cluster data. +Only present if `include_remotes` option is set to `true`. + ++ +.Properties of `clusters` +[%collapsible%open] +====== + +`cluster_uuid`::: +(string) The UUID of the remote cluster. + +`mode`::: +(string) The <> used to communicate with the remote cluster. + +`skip_unavailable`::: +(Boolean) The `skip_unavailable` <> used for this remote cluster. + +`transport.compress`::: +(string) Transport compression setting used for this remote cluster. + +`version`::: +(array of strings) The list of {es} versions used by the nodes on the remote cluster. + +`status`::: +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cluster-health-status] ++ +See <>. + +`nodes_count`::: +(integer) The total count of nodes in the remote cluster. + +`shards_count`::: +(integer) The total number of shards in the remote cluster. + +`indices_count`::: +(integer) The total number of indices in the remote cluster. + +`indices_total_size_in_bytes`::: +(integer) Total data set size, in bytes, of all shards assigned to selected nodes. + +`indices_total_size`::: +(string) Total data set size, in bytes, of all shards assigned to selected nodes, as a human-readable string. + +`max_heap_in_bytes`::: +(integer) Maximum amount of memory, in bytes, available for use by the heap across the nodes of the remote cluster. + +`max_heap`::: +(string) Maximum amount of memory, in bytes, available for use by the heap across the nodes of the remote cluster, +as a human-readable string. + +`mem_total_in_bytes`::: +(integer) Total amount, in bytes, of physical memory across the nodes of the remote cluster. + +`mem_total`::: +(string) Total amount, in bytes, of physical memory across the nodes of the remote cluster, as a human-readable string. + +====== + `_search`::: -(object) Contains the telemetry information about the <> usage in the cluster. +(object) Contains information about <> usage. + .Properties of `_search` [%collapsible%open] ====== + `total`::: (integer) The total number of {ccs} requests that have been executed by the cluster. @@ -1336,6 +1417,7 @@ Each repository type may also include other statistics about the repositories of .Properties of `took` [%collapsible%open] ======= + `max`::: (integer) The maximum time taken to execute a {ccs} request, in milliseconds. @@ -1344,6 +1426,7 @@ Each repository type may also include other statistics about the repositories of `p90`::: (integer) The 90th percentile of the time taken to execute {ccs} requests, in milliseconds. + ======= `took_mrt_true`:: @@ -1361,6 +1444,7 @@ Each repository type may also include other statistics about the repositories of `p90`::: (integer) The 90th percentile of the time taken to execute {ccs} requests, in milliseconds. + ======= `took_mrt_false`:: @@ -1378,6 +1462,7 @@ Each repository type may also include other statistics about the repositories of `p90`::: (integer) The 90th percentile of the time taken to execute {ccs} requests, in milliseconds. + ======= `remotes_per_search_max`:: @@ -1391,9 +1476,10 @@ Each repository type may also include other statistics about the repositories of The keys are the failure reason names and the values are the number of requests that failed for that reason. `features`:: -(object) Contains statistics about the features used in {ccs} requests. The keys are the names of the search feature, -and the values are the number of requests that used that feature. Single request can use more than one feature -(e.g. both `async` and `wildcard`). Known features are: +(object) Contains statistics about the features used in {ccs} requests. +The keys are the names of the search feature, and the values are the number of requests that used that feature. +Single request can use more than one feature (e.g. both `async` and `wildcard`). +Known features are: * `async` - <> @@ -1427,6 +1513,7 @@ This may include requests where partial results were returned, but not requests .Properties of `took` [%collapsible%open] ======== + `max`::: (integer) The maximum time taken to execute a {ccs} request, in milliseconds. @@ -1435,11 +1522,16 @@ This may include requests where partial results were returned, but not requests `p90`::: (integer) The 90th percentile of the time taken to execute {ccs} requests, in milliseconds. + ======== ======= + ====== +`_esql`::: +(object) Contains information about <> usage. +The structure of the object is the same as the `_search` object above. ===== @@ -1561,7 +1653,10 @@ The API returns the following response: "total_deduplicated_mapping_size": "0b", "total_deduplicated_mapping_size_in_bytes": 0, "field_types": [], - "runtime_field_types": [] + "runtime_field_types": [], + "source_modes" : { + "stored": 0 + } }, "analysis": { "char_filter_types": [], @@ -1812,3 +1907,37 @@ This API can be restricted to a subset of the nodes using <Cluster update settings ++++ -Configures <>. +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs]. +-- +Configures <>. [[cluster-update-settings-api-request]] ==== {api-request-title} diff --git a/docs/reference/cluster/voting-exclusions.asciidoc b/docs/reference/cluster/voting-exclusions.asciidoc index e5b8544a16554..e60b3be26508d 100644 --- a/docs/reference/cluster/voting-exclusions.asciidoc +++ b/docs/reference/cluster/voting-exclusions.asciidoc @@ -4,10 +4,15 @@ Voting configuration exclusions ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs]. +-- + Adds or removes master-eligible nodes from the <>. - [[voting-config-exclusions-api-request]] ==== {api-request-title} @@ -28,7 +33,7 @@ users can use this API. [[voting-config-exclusions-api-desc]] ==== {api-description-title} - + By default, if there are more than three master-eligible nodes in the cluster and you remove fewer than half of the master-eligible nodes in the cluster at once, the <> automatically @@ -50,14 +55,19 @@ use `DELETE /_cluster/voting_config_exclusions?wait_for_removal=false` to clear the voting configuration exclusions without waiting for the nodes to leave the cluster. -If the API fails, you can safely retry it. Only a successful response -guarantees that the node has been removed from the voting configuration and -will not be reinstated. +A response to `POST /_cluster/voting_config_exclusions` with an HTTP status +code of `200 OK` guarantees that the node has been removed from the voting +configuration and will not be reinstated until the voting configuration +exclusions are cleared by calling `DELETE /_cluster/voting_config_exclusions`. +If the call to `POST /_cluster/voting_config_exclusions` fails or returns a +response with an HTTP status code other than `200 OK` then the node may not +have been removed from the voting configuration. In that case, you may safely +retry the call. NOTE: Voting exclusions are required only when you remove at least half of the master-eligible nodes from a cluster in a short time period. They are not -required when removing master-ineligible nodes or fewer than half of the -master-eligible nodes. +required when removing master-ineligible nodes or when removing fewer than half +of the master-eligible nodes. For more information, see <>. @@ -94,7 +104,7 @@ list. Defaults to `true`, meaning that all excluded nodes must be removed from the cluster before this API takes any action. If set to `false` then the voting configuration exclusions list is cleared even if some excluded nodes are still in the cluster. Only applies to the `DELETE` form of this API. - + [[voting-config-exclusions-api-example]] ==== {api-examples-title} @@ -102,7 +112,7 @@ Adds nodes named `nodeName1` and `nodeName2` to the voting configuration exclusions list: [source,console] --------------------------------------------------- +-------------------------------------------------- POST /_cluster/voting_config_exclusions?node_names=nodeName1,nodeName2 -------------------------------------------------- diff --git a/docs/reference/commands/cli-jvm-options.asciidoc b/docs/reference/commands/cli-jvm-options.asciidoc index 546884f428c12..0428ead60b626 100644 --- a/docs/reference/commands/cli-jvm-options.asciidoc +++ b/docs/reference/commands/cli-jvm-options.asciidoc @@ -3,7 +3,7 @@ ==== JVM options CLI tools run with 64MB of heap. For most tools, this value is fine. However, if -needed this can be overriden by setting the `CLI_JAVA_OPTS` environment variable. +needed this can be overridden by setting the `CLI_JAVA_OPTS` environment variable. For example, the following increases the heap size used by the `pass:a[elasticsearch-{tool-name}]` tool to 1GB. diff --git a/docs/reference/connector/apis/cancel-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/cancel-connector-sync-job-api.asciidoc index 105d395e42c8b..d4db8ce62bc46 100644 --- a/docs/reference/connector/apis/cancel-connector-sync-job-api.asciidoc +++ b/docs/reference/connector/apis/cancel-connector-sync-job-api.asciidoc @@ -6,9 +6,16 @@ beta::[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs]. +-- + Cancels a connector sync job. -To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. +To get started with Connector APIs, check out <>. + [[cancel-connector-sync-job-api-request]] ==== {api-request-title} @@ -17,7 +24,7 @@ To get started with Connector APIs, check out the {enterprise-search-ref}/connec [[cancel-connector-sync-job-api-prereqs]] ==== {api-prereq-title} -* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors. +* To sync data using self-managed connectors, you need to deploy the <>. on your own infrastructure. This service runs automatically on Elastic Cloud for Elastic managed connectors. * The `connector_sync_job_id` parameter should reference an existing connector sync job. [[cancel-connector-sync-job-api-desc]] diff --git a/docs/reference/connector/apis/check-in-connector-api.asciidoc b/docs/reference/connector/apis/check-in-connector-api.asciidoc index 8c6b5161a3a72..be7521e937316 100644 --- a/docs/reference/connector/apis/check-in-connector-api.asciidoc +++ b/docs/reference/connector/apis/check-in-connector-api.asciidoc @@ -6,9 +6,16 @@ preview::[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs]. +-- + Updates the `last_seen` field of a connector with current timestamp. -To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. +To get started with Connector APIs, check out <>. + [[check-in-connector-api-request]] ==== {api-request-title} @@ -18,7 +25,7 @@ To get started with Connector APIs, check out the {enterprise-search-ref}/connec [[check-in-connector-api-prereq]] ==== {api-prereq-title} -* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors. +* To sync data using self-managed connectors, you need to deploy the <>. on your own infrastructure. This service runs automatically on Elastic Cloud for Elastic managed connectors. * The `connector_id` parameter should reference an existing connector. [[check-in-connector-api-path-params]] diff --git a/docs/reference/connector/apis/check-in-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/check-in-connector-sync-job-api.asciidoc index a052fbb2418cc..5f9b584621c2f 100644 --- a/docs/reference/connector/apis/check-in-connector-sync-job-api.asciidoc +++ b/docs/reference/connector/apis/check-in-connector-sync-job-api.asciidoc @@ -6,9 +6,16 @@ preview::[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs]. +-- + Checks in a connector sync job (updates `last_seen` to the current time). -To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. +To get started with Connector APIs, check out <>. + [[check-in-connector-sync-job-api-request]] ==== {api-request-title} @@ -17,7 +24,7 @@ To get started with Connector APIs, check out the {enterprise-search-ref}/connec [[check-in-connector-sync-job-api-prereqs]] ==== {api-prereq-title} -* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors. +* To sync data using self-managed connectors, you need to deploy the <>. on your own infrastructure. This service runs automatically on Elastic Cloud for Elastic managed connectors. * The `connector_sync_job_id` parameter should reference an existing connector sync job. [[check-in-connector-sync-job-api-desc]] diff --git a/docs/reference/connector/apis/claim-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/claim-connector-sync-job-api.asciidoc index 2fb28f9e9fb37..565a39c2083af 100644 --- a/docs/reference/connector/apis/claim-connector-sync-job-api.asciidoc +++ b/docs/reference/connector/apis/claim-connector-sync-job-api.asciidoc @@ -6,11 +6,18 @@ preview::[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs]. +-- + Claims a connector sync job. The `_claim` endpoint is not intended for direct connector management by users. It is there to support the implementation of services that utilize the https://github.com/elastic/connectors/blob/main/docs/CONNECTOR_PROTOCOL.md[Connector Protocol] to communicate with {es}. -To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. +To get started with Connector APIs, check out <>. + [[claim-connector-sync-job-api-request]] ==== {api-request-title} @@ -19,7 +26,7 @@ To get started with Connector APIs, check out the {enterprise-search-ref}/connec [[claim-connector-sync-job-api-prereqs]] ==== {api-prereq-title} -* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors. +* To sync data using self-managed connectors, you need to deploy the <>. on your own infrastructure. This service runs automatically on Elastic Cloud for Elastic managed connectors. * The `connector_sync_job_id` parameter should reference an existing connector sync job. [[claim-connector-sync-job-api-desc]] diff --git a/docs/reference/connector/apis/connector-apis.asciidoc b/docs/reference/connector/apis/connector-apis.asciidoc index 987f82f6b4ce4..719db5a315714 100644 --- a/docs/reference/connector/apis/connector-apis.asciidoc +++ b/docs/reference/connector/apis/connector-apis.asciidoc @@ -1,16 +1,24 @@ [[connector-apis]] == Connector APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs]. +-- + beta::[] -The connector and sync jobs APIs provide a convenient way to create and manage Elastic {enterprise-search-ref}/connectors.html[connectors^] and sync jobs in an internal index. To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. +The connector and sync jobs APIs provide a convenient way to create and manage Elastic <>. + and sync jobs in an internal index. To get started with Connector APIs, check out <>. + Connectors are {es} integrations that bring content from third-party data sources, which can be deployed on {ecloud} or hosted on your own infrastructure: -* *Native connectors* are a managed service on {ecloud} -* *Connector clients* are self-managed on your infrastructure +* *Managed connectors* are a managed service on {ecloud} +* *Self-managed connectors* are self-hosted on your infrastructure -Find a list of all supported service types in the {enterprise-search-ref}/connectors.html[connectors documentation^]. +Find a list of all supported service types in the <>. This API provides an alternative to relying solely on {kib} UI for connector and sync job management. The API comes with a set of validations and assertions to ensure that the state representation in the internal index remains valid. @@ -82,7 +90,7 @@ beta:[] preview::[] -*Connector Service APIs* are a subset of Connector API endpoints, that represent framework-level operations defined in the https://github.com/elastic/connectors/blob/main/docs/CONNECTOR_PROTOCOL.md[Connector Protocol]. These APIs are not intended for direct connector management by users but are there to support the implementation of services that utilize the Conector Protocol to communicate with {es}. +*Connector Service APIs* are a subset of Connector API endpoints, that represent framework-level operations defined in the https://github.com/elastic/connectors/blob/main/docs/CONNECTOR_PROTOCOL.md[Connector Protocol]. These APIs are not intended for direct connector management by users but are there to support the implementation of services that utilize the Connector Protocol to communicate with {es}. [TIP] ==== diff --git a/docs/reference/connector/apis/create-connector-api.asciidoc b/docs/reference/connector/apis/create-connector-api.asciidoc index 9bd49a3c5ef94..3300ce270c345 100644 --- a/docs/reference/connector/apis/create-connector-api.asciidoc +++ b/docs/reference/connector/apis/create-connector-api.asciidoc @@ -6,15 +6,22 @@ beta::[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs]. +-- + Creates an Elastic connector. Connectors are {es} integrations that bring content from third-party data sources, which can be deployed on {ecloud} or hosted on your own infrastructure: -* *Native connectors* are a managed service on {ecloud} -* *Connector clients* are self-managed on your infrastructure +* *Managed connectors* are a managed service on {ecloud} +* *Self-managed connectors* are self-hosted on your infrastructure + +Find a list of all supported service types in the <>. -Find a list of all supported service types in the {enterprise-search-ref}/connectors.html[connectors documentation^]. +To get started with Connector APIs, check out <>. -To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. [source,console] -------------------------------------------------- @@ -43,8 +50,8 @@ DELETE _connector/my-connector [[create-connector-api-prereqs]] ==== {api-prereq-title} -* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors. -* The `service_type` parameter should reference a supported third-party service. See the available service types for {enterprise-search-ref}/native-connectors.html[native] and {enterprise-search-ref}/build-connector.html[self-managed] connectors. This can also reference the service type of your custom connector. +* To sync data using self-managed connectors, you need to deploy the <>. on your own infrastructure. This service runs automatically on Elastic Cloud for Elastic managed connectors. +* The `service_type` parameter should reference a supported third-party service. See the available service types for <> and <> connectors. This can also reference the service type of your custom connector. [[create-connector-api-desc]] @@ -73,13 +80,13 @@ Creates a connector document in the internal index and initializes its configura (Optional, string) The name of the connector. Setting the connector name is recommended when managing connectors in {kib}. `is_native`:: -(Optional, boolean) Indicates if it's a native connector. Defaults to `false`. +(Optional, boolean) Indicates if it's a managed connector. Defaults to `false`. `language`:: (Optional, string) Language analyzer for the data. Limited to supported languages. `service_type`:: -(Optional, string) Connector service type. Can reference Elastic-supported third-party services or a custom connector type. See the available service types for {enterprise-search-ref}/native-connectors.html[native] and {enterprise-search-ref}/build-connector.html[self-managed] connectors. +(Optional, string) Connector service type. Can reference Elastic-supported third-party services or a custom connector type. See the available service types for <> and <> connectors. [role="child_attributes"] @@ -115,7 +122,7 @@ PUT _connector/my-connector "name": "My Connector", "description": "My Connector to sync data to Elastic index from Google Drive", "service_type": "google_drive", - "language": "english" + "language": "en" } ---- diff --git a/docs/reference/connector/apis/create-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/create-connector-sync-job-api.asciidoc index c7cc866930dfb..1a66d47578a8b 100644 --- a/docs/reference/connector/apis/create-connector-sync-job-api.asciidoc +++ b/docs/reference/connector/apis/create-connector-sync-job-api.asciidoc @@ -6,10 +6,16 @@ beta::[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs]. +-- Creates a connector sync job. -To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. +To get started with Connector APIs, check out <>. + [source, console] -------------------------------------------------- @@ -31,7 +37,7 @@ POST _connector/_sync_job [[create-connector-sync-job-api-prereqs]] ==== {api-prereq-title} -* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors. +* To sync data using self-managed connectors, you need to deploy the <>. on your own infrastructure. This service runs automatically on Elastic Cloud for Elastic managed connectors. * The `id` parameter should reference an existing connector. [[create-connector-sync-job-api-desc]] diff --git a/docs/reference/connector/apis/delete-connector-api.asciidoc b/docs/reference/connector/apis/delete-connector-api.asciidoc index 23acd1b4755b1..b36a99bc2d8cc 100644 --- a/docs/reference/connector/apis/delete-connector-api.asciidoc +++ b/docs/reference/connector/apis/delete-connector-api.asciidoc @@ -6,12 +6,19 @@ beta::[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs]. +-- + Removes a connector and associated sync jobs. This is a destructive action that is not recoverable. Note: this action doesn't delete any API key, ingest pipeline or data index associated with the connector. These need to be removed manually. -To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. +To get started with Connector APIs, check out <>. + [[delete-connector-api-request]] ==== {api-request-title} @@ -21,7 +28,7 @@ To get started with Connector APIs, check out the {enterprise-search-ref}/connec [[delete-connector-api-prereq]] ==== {api-prereq-title} -* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors. +* To sync data using self-managed connectors, you need to deploy the <>. on your own infrastructure. This service runs automatically on Elastic Cloud for Elastic managed connectors. * The `connector_id` parameter should reference an existing connector. [[delete-connector-api-path-params]] diff --git a/docs/reference/connector/apis/delete-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/delete-connector-sync-job-api.asciidoc index 7cdabb22f05ee..bc906d12cae40 100644 --- a/docs/reference/connector/apis/delete-connector-sync-job-api.asciidoc +++ b/docs/reference/connector/apis/delete-connector-sync-job-api.asciidoc @@ -6,10 +6,17 @@ beta::[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs]. +-- + Removes a connector sync job and its associated data. This is a destructive action that is not recoverable. -To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. +To get started with Connector APIs, check out <>. + [[delete-connector-sync-job-api-request]] ==== {api-request-title} @@ -19,7 +26,7 @@ To get started with Connector APIs, check out the {enterprise-search-ref}/connec [[delete-connector-sync-job-api-prereq]] ==== {api-prereq-title} -* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors. +* To sync data using self-managed connectors, you need to deploy the <>. on your own infrastructure. This service runs automatically on Elastic Cloud for Elastic managed connectors. [[delete-connector-sync-job-api-path-params]] ==== {api-path-parms-title} diff --git a/docs/reference/connector/apis/get-connector-api.asciidoc b/docs/reference/connector/apis/get-connector-api.asciidoc index 4df792c8a0a1a..cff13539f80cc 100644 --- a/docs/reference/connector/apis/get-connector-api.asciidoc +++ b/docs/reference/connector/apis/get-connector-api.asciidoc @@ -6,9 +6,16 @@ beta::[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs]. +-- + Retrieves the details about a connector. -To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. +To get started with Connector APIs, check out <>. + [[get-connector-api-request]] ==== {api-request-title} @@ -18,7 +25,7 @@ To get started with Connector APIs, check out the {enterprise-search-ref}/connec [[get-connector-api-prereq]] ==== {api-prereq-title} -* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors. +* To sync data using self-managed connectors, you need to deploy the <>. on your own infrastructure. This service runs automatically on Elastic Cloud for Elastic managed connectors. [[get-connector-api-path-params]] ==== {api-path-parms-title} diff --git a/docs/reference/connector/apis/get-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/get-connector-sync-job-api.asciidoc index fffdada2a2a82..f4ccc59e0315e 100644 --- a/docs/reference/connector/apis/get-connector-sync-job-api.asciidoc +++ b/docs/reference/connector/apis/get-connector-sync-job-api.asciidoc @@ -6,9 +6,16 @@ beta::[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs]. +-- + Retrieves the details about a connector sync job. -To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. +To get started with Connector APIs, check out <>. + [[get-connector-sync-job-api-request]] ==== {api-request-title} @@ -18,7 +25,7 @@ To get started with Connector APIs, check out the {enterprise-search-ref}/connec [[get-connector-sync-job-api-prereq]] ==== {api-prereq-title} -* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors. +* To sync data using self-managed connectors, you need to deploy the <>. on your own infrastructure. This service runs automatically on Elastic Cloud for Elastic managed connectors. [[get-connector-sync-job-api-path-params]] ==== {api-path-parms-title} diff --git a/docs/reference/connector/apis/list-connector-sync-jobs-api.asciidoc b/docs/reference/connector/apis/list-connector-sync-jobs-api.asciidoc index 730dad852adee..b5f52e31ac296 100644 --- a/docs/reference/connector/apis/list-connector-sync-jobs-api.asciidoc +++ b/docs/reference/connector/apis/list-connector-sync-jobs-api.asciidoc @@ -7,9 +7,16 @@ beta::[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs]. +-- + Returns information about all stored connector sync jobs ordered by their creation date in ascending order. -To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. +To get started with Connector APIs, check out <>. + [[list-connector-sync-jobs-api-request]] ==== {api-request-title} @@ -19,7 +26,7 @@ To get started with Connector APIs, check out the {enterprise-search-ref}/connec [[list-connector-sync-jobs-api-prereq]] ==== {api-prereq-title} -* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors. +* To sync data using self-managed connectors, you need to deploy the <>. on your own infrastructure. This service runs automatically on Elastic Cloud for Elastic managed connectors. [[list-connector-sync-jobs-api-path-params]] ==== {api-path-parms-title} diff --git a/docs/reference/connector/apis/list-connectors-api.asciidoc b/docs/reference/connector/apis/list-connectors-api.asciidoc index c7ea2afd8102f..5cc099a6b67ee 100644 --- a/docs/reference/connector/apis/list-connectors-api.asciidoc +++ b/docs/reference/connector/apis/list-connectors-api.asciidoc @@ -7,9 +7,16 @@ beta::[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs]. +-- + Returns information about all created connectors. -To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. +To get started with Connector APIs, check out <>. + [[list-connector-api-request]] @@ -20,7 +27,7 @@ To get started with Connector APIs, check out the {enterprise-search-ref}/connec [[list-connector-api-prereq]] ==== {api-prereq-title} -* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors. +* To sync data using self-managed connectors, you need to deploy the <>. on your own infrastructure. This service runs automatically on Elastic Cloud for Elastic managed connectors. [[list-connector-api-path-params]] ==== {api-path-parms-title} diff --git a/docs/reference/connector/apis/set-connector-sync-job-error-api.asciidoc b/docs/reference/connector/apis/set-connector-sync-job-error-api.asciidoc index 42203ed8e6103..b5a0cd667f3ee 100644 --- a/docs/reference/connector/apis/set-connector-sync-job-error-api.asciidoc +++ b/docs/reference/connector/apis/set-connector-sync-job-error-api.asciidoc @@ -6,9 +6,16 @@ preview::[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs]. +-- + Sets a connector sync job error. -To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. +To get started with Connector APIs, check out <>. + [[set-connector-sync-job-error-api-request]] ==== {api-request-title} @@ -17,7 +24,7 @@ To get started with Connector APIs, check out the {enterprise-search-ref}/connec [[set-connector-sync-job-error-api-prereqs]] ==== {api-prereq-title} -* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors. +* To sync data using self-managed connectors, you need to deploy the <>. on your own infrastructure. This service runs automatically on Elastic Cloud for Elastic managed connectors. * The `connector_sync_job_id` parameter should reference an existing connector sync job. [[set-connector-sync-job-error-api-desc]] diff --git a/docs/reference/connector/apis/set-connector-sync-job-stats-api.asciidoc b/docs/reference/connector/apis/set-connector-sync-job-stats-api.asciidoc index 1427269d22b86..5691280a30dd7 100644 --- a/docs/reference/connector/apis/set-connector-sync-job-stats-api.asciidoc +++ b/docs/reference/connector/apis/set-connector-sync-job-stats-api.asciidoc @@ -6,9 +6,16 @@ preview::[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs]. +-- + Sets connector sync job stats. -To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. +To get started with Connector APIs, check out <>. + [[set-connector-sync-job-stats-api-request]] ==== {api-request-title} @@ -17,7 +24,7 @@ To get started with Connector APIs, check out the {enterprise-search-ref}/connec [[set-connector-sync-job-stats-api-prereqs]] ==== {api-prereq-title} -* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors. +* To sync data using self-managed connectors, you need to deploy the <>. on your own infrastructure. This service runs automatically on Elastic Cloud for Elastic managed connectors. * The `connector_sync_job_id` parameter should reference an existing connector sync job. [[set-connector-sync-job-stats-api-desc]] diff --git a/docs/reference/connector/apis/update-connector-api-key-id-api.asciidoc b/docs/reference/connector/apis/update-connector-api-key-id-api.asciidoc index 112ec821df7c9..8df49c43c128e 100644 --- a/docs/reference/connector/apis/update-connector-api-key-id-api.asciidoc +++ b/docs/reference/connector/apis/update-connector-api-key-id-api.asciidoc @@ -6,16 +6,23 @@ beta::[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs]. +-- + Updates the `api_key_id` and/or `api_key_secret_id` field(s) of a connector, specifying: . The ID of the API key used for authorization . The ID of the Connector Secret where the API key is stored -The Connector Secret ID is only required for native connectors. -Connector clients do not use this field. -See the documentation for {enterprise-search-ref}/native-connectors.html#native-connectors-manage-API-keys-programmatically[managing native connector API keys programmatically^] for more details. +The Connector Secret ID is only required for Elastic managed connectors. +Self-managed connectors do not use this field. +See the documentation for <> for more details. + +To get started with Connector APIs, check out <>. -To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. [[update-connector-api-key-id-api-request]] ==== {api-request-title} @@ -25,7 +32,7 @@ To get started with Connector APIs, check out the {enterprise-search-ref}/connec [[update-connector-api-key-id-api-prereq]] ==== {api-prereq-title} -* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors. +* To sync data using self-managed connectors, you need to deploy the <>. on your own infrastructure. This service runs automatically on Elastic Cloud for Elastic managed connectors. * The `connector_id` parameter should reference an existing connector. * The `api_key_id` parameter should reference an existing API key. * The `api_key_secret_id` parameter should reference an existing Connector Secret containing an encoded API key value. @@ -44,7 +51,7 @@ To get started with Connector APIs, check out the {enterprise-search-ref}/connec (Optional, string) ID of the API key that the connector will use to authorize access to required indices. Each connector can be associated with at most one API key. `api_key_secret_id`:: -(Optional, string) ID of the Connector Secret that contains the encoded API key. This should be the same API key as `api_key_id` references. This is only required for native connectors. +(Optional, string) ID of the Connector Secret that contains the encoded API key. This should be the same API key as `api_key_id` references. This is only required for Elastic managed connectors. [[update-connector-api-key-id-api-response-codes]] ==== {api-response-codes-title} diff --git a/docs/reference/connector/apis/update-connector-configuration-api.asciidoc b/docs/reference/connector/apis/update-connector-configuration-api.asciidoc index e8a710cdacff0..d02c332d7f34b 100644 --- a/docs/reference/connector/apis/update-connector-configuration-api.asciidoc +++ b/docs/reference/connector/apis/update-connector-configuration-api.asciidoc @@ -6,9 +6,16 @@ beta::[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs]. +-- + Updates a connector's `configuration`, allowing for config value updates within a registered configuration schema. -To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. +To get started with Connector APIs, check out <>. + [[update-connector-configuration-api-request]] ==== {api-request-title} @@ -18,10 +25,10 @@ To get started with Connector APIs, check out the {enterprise-search-ref}/connec [[update-connector-configuration-api-prereq]] ==== {api-prereq-title} -* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors. +* To sync data using self-managed connectors, you need to deploy the <>. on your own infrastructure. This service runs automatically on Elastic Cloud for Elastic managed connectors. * The `connector_id` parameter should reference an existing connector. * To update configuration `values`, the connector `configuration` schema must be first registered by a running instance of Elastic connector service. -* Make sure configuration fields are compatible with the configuration schema for the third-party data source. Refer to the individual {enterprise-search-ref}/connectors-references.html[connectors references] for details. +* Make sure configuration fields are compatible with the configuration schema for the third-party data source. Refer to the individual <> for details. [[update-connector-configuration-api-path-params]] ==== {api-path-parms-title} @@ -55,7 +62,7 @@ No connector matching `connector_id` could be found. [[update-connector-configuration-api-example]] ==== {api-examples-title} -The following example configures a `sharepoint_online` connector. Find the supported configuration options in the {enterprise-search-ref}/connectors-sharepoint-online.html[Sharepoint Online connector documentation] or by inspecting the schema in the connector's `configuration` field using the <>. +The following example configures a `sharepoint_online` connector. Find the supported configuration options in the <>, or by inspecting the schema in the connector's `configuration` field using the <>. //// [source, console] diff --git a/docs/reference/connector/apis/update-connector-error-api.asciidoc b/docs/reference/connector/apis/update-connector-error-api.asciidoc index c6ac0c9a1ac22..859a48c31c0ca 100644 --- a/docs/reference/connector/apis/update-connector-error-api.asciidoc +++ b/docs/reference/connector/apis/update-connector-error-api.asciidoc @@ -6,9 +6,16 @@ preview::[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs]. +-- + Updates the `error` field of a connector. -To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. +To get started with Connector APIs, check out <>. + [[update-connector-error-api-request]] ==== {api-request-title} @@ -18,7 +25,7 @@ To get started with Connector APIs, check out the {enterprise-search-ref}/connec [[update-connector-error-api-prereq]] ==== {api-prereq-title} -* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors. +* To sync data using self-managed connectors, you need to deploy the <>. on your own infrastructure. This service runs automatically on Elastic Cloud for Elastic managed connectors. * The `connector_id` parameter should reference an existing connector. [[update-connector-error-api-desc]] diff --git a/docs/reference/connector/apis/update-connector-features-api.asciidoc b/docs/reference/connector/apis/update-connector-features-api.asciidoc index 0d3457b9bd584..74c512e42cd1b 100644 --- a/docs/reference/connector/apis/update-connector-features-api.asciidoc +++ b/docs/reference/connector/apis/update-connector-features-api.asciidoc @@ -6,6 +6,12 @@ beta::[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs]. +-- + Manages the `features` of a connector. This endpoint can be used to control the following aspects of a connector: * document-level security @@ -15,7 +21,8 @@ Manages the `features` of a connector. This endpoint can be used to control the Normally, the running connector service automatically manages these features. However, you can use this API to override the default behavior. -To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. +To get started with Connector APIs, check out <>. + [[update-connector-features-api-request]] ==== {api-request-title} @@ -25,7 +32,7 @@ To get started with Connector APIs, check out the {enterprise-search-ref}/connec [[update-connector-features-api-prereq]] ==== {api-prereq-title} -* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors. +* To sync data using self-managed connectors, you need to deploy the <>. on your own infrastructure. This service runs automatically on Elastic Cloud for Elastic managed connectors. * The `connector_id` parameter should reference an existing connector. [[update-connector-features-api-path-params]] @@ -43,7 +50,7 @@ To get started with Connector APIs, check out the {enterprise-search-ref}/connec * `document_level_security` (Optional, object) Controls whether document-level security is enabled with the `enabled` flag. * `incremental_sync` (Optional, object) Controls whether incremental syncs are enabled with the `enabled` flag. -* `native_connector_api_keys`(Optional, object) Controls whether native connector API keys are enabled with the `enabled` flag. +* `native_connector_api_keys`(Optional, object) Controls whether managed connector API keys are enabled with the `enabled` flag. * `sync_rules` (Optional, object) Controls sync rules. ** `advanced` (Optional, object) Controls whether advanced sync rules are enabled with the `enabled` flag. ** `basic`(Optional, object) Controls whether basic sync rules are enabled with the `enabled` flag. diff --git a/docs/reference/connector/apis/update-connector-filtering-api.asciidoc b/docs/reference/connector/apis/update-connector-filtering-api.asciidoc index 861e72481a59a..7ba0080cde28f 100644 --- a/docs/reference/connector/apis/update-connector-filtering-api.asciidoc +++ b/docs/reference/connector/apis/update-connector-filtering-api.asciidoc @@ -6,11 +6,18 @@ beta::[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs]. +-- + Updates the draft `filtering` configuration of a connector and marks the draft validation state as `edited`. The filtering draft is activated once validated by the running Elastic connector service. -The filtering property is used to configure sync rules (both basic and advanced) for a connector. Learn more in the {enterprise-search-ref}/sync-rules.html[sync rules documentation]. +The filtering property is used to configure sync rules (both basic and advanced) for a connector. Learn more in the <>. + +To get started with Connector APIs, check out <>. -To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. [[update-connector-filtering-api-request]] ==== {api-request-title} @@ -20,7 +27,7 @@ To get started with Connector APIs, check out the {enterprise-search-ref}/connec [[update-connector-filtering-api-prereq]] ==== {api-prereq-title} -* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors. +* To sync data using self-managed connectors, you need to deploy the <>. on your own infrastructure. This service runs automatically on Elastic Cloud for Elastic managed connectors. * The `connector_id` parameter should reference an existing connector. * Filtering draft is activated once validated by the running Elastic connector service, the `draft.validation.state` must be `valid`. * If, after a validation attempt, the `draft.validation.state` equals to `invalid`, inspect `draft.validation.errors` and fix any issues. @@ -37,7 +44,7 @@ To get started with Connector APIs, check out the {enterprise-search-ref}/connec `rules`:: (Optional, array of objects) -An array of {enterprise-search-ref}/sync-rules.html#sync-rules-basic[basic sync rules], each with the following sub-attributes: +An array of <>, each with the following sub-attributes: * `id` (Required, string) + A unique identifier for the rule. * `policy` (Required, string) + @@ -57,7 +64,7 @@ The timestamp when the rule was last edited. Defaults to `now` UTC timestamp. `advanced_snippet`:: (Optional, object) -Used for {enterprise-search-ref}/sync-rules.html#sync-rules-advanced[advanced filtering] at query time, with the following sub-attributes: +Used for <> at query time, with the following sub-attributes: * `value` (Required, object or array) + A JSON object/array passed directly to the connector for advanced filtering. * `created_at` (Optional, datetime) + @@ -81,7 +88,7 @@ No connector matching `connector_id` could be found. [[update-connector-filtering-api-example]] ==== {api-examples-title} -The following example updates the draft {enterprise-search-ref}/sync-rules.html#sync-rules-basic[basic sync rules] for a Google Drive connector with ID `my-g-drive-connector`. All Google Drive files with `.txt` extension will be skipped: +The following example updates the draft <> for a Google Drive connector with ID `my-g-drive-connector`. All Google Drive files with `.txt` extension will be skipped: //// [source, console] @@ -143,7 +150,7 @@ PUT _connector/my-g-drive-connector/_filtering } ---- -The following example updates the draft advanced sync rules for a MySQL connector with id `my-sql-connector`. Advanced sync rules are specific to each connector type. Refer to the references for connectors that support {enterprise-search-ref}/sync-rules.html#sync-rules-advanced[advanced sync rules] for syntax and examples. +The following example updates the draft advanced sync rules for a MySQL connector with id `my-sql-connector`. Advanced sync rules are specific to each connector type. Refer to the references for connectors that support <> for syntax and examples. [source,console] ---- diff --git a/docs/reference/connector/apis/update-connector-index-name-api.asciidoc b/docs/reference/connector/apis/update-connector-index-name-api.asciidoc index d07007438e09c..fbbef6e66ca40 100644 --- a/docs/reference/connector/apis/update-connector-index-name-api.asciidoc +++ b/docs/reference/connector/apis/update-connector-index-name-api.asciidoc @@ -6,9 +6,16 @@ beta::[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs]. +-- + Updates the `index_name` field of a connector, specifying the index where the data ingested by the connector is stored. -To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. +To get started with Connector APIs, check out <>. + [[update-connector-index-name-api-request]] ==== {api-request-title} @@ -18,7 +25,7 @@ To get started with Connector APIs, check out the {enterprise-search-ref}/connec [[update-connector-index-name-api-prereq]] ==== {api-prereq-title} -* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors. +* To sync data using self-managed connectors, you need to deploy the <>. on your own infrastructure. This service runs automatically on Elastic Cloud for Elastic managed connectors. * The `connector_id` parameter should reference an existing connector. [[update-connector-index-name-api-path-params]] diff --git a/docs/reference/connector/apis/update-connector-last-sync-api.asciidoc b/docs/reference/connector/apis/update-connector-last-sync-api.asciidoc index 918bf4f80a010..d827d25c12b4e 100644 --- a/docs/reference/connector/apis/update-connector-last-sync-api.asciidoc +++ b/docs/reference/connector/apis/update-connector-last-sync-api.asciidoc @@ -6,11 +6,18 @@ preview::[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs]. +-- + Updates the fields related to the last sync of a connector. This action is used for analytics and monitoring. -To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. +To get started with Connector APIs, check out <>. + [[update-connector-last-sync-api-request]] ==== {api-request-title} @@ -20,7 +27,7 @@ To get started with Connector APIs, check out the {enterprise-search-ref}/connec [[update-connector-last-sync-api-prereq]] ==== {api-prereq-title} -* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors. +* To sync data using self-managed connectors, you need to deploy the <>. on your own infrastructure. This service runs automatically on Elastic Cloud for Elastic managed connectors. * The `connector_id` parameter should reference an existing connector. [[update-connector-last-sync-api-path-params]] diff --git a/docs/reference/connector/apis/update-connector-name-description-api.asciidoc b/docs/reference/connector/apis/update-connector-name-description-api.asciidoc index 7e16874da9fb4..c0d0568baef35 100644 --- a/docs/reference/connector/apis/update-connector-name-description-api.asciidoc +++ b/docs/reference/connector/apis/update-connector-name-description-api.asciidoc @@ -6,10 +6,16 @@ beta::[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs]. +-- Updates the `name` and `description` fields of a connector. -To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. +To get started with Connector APIs, check out <>. + [[update-connector-name-description-api-request]] ==== {api-request-title} @@ -19,7 +25,7 @@ To get started with Connector APIs, check out the {enterprise-search-ref}/connec [[update-connector-name-description-api-prereq]] ==== {api-prereq-title} -* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors. +* To sync data using self-managed connectors, you need to deploy the <>. on your own infrastructure. This service runs automatically on Elastic Cloud for Elastic managed connectors. * The `connector_id` parameter should reference an existing connector. [[update-connector-name-description-api-path-params]] diff --git a/docs/reference/connector/apis/update-connector-pipeline-api.asciidoc b/docs/reference/connector/apis/update-connector-pipeline-api.asciidoc index 01ed2e39702ea..a886fe6f20da7 100644 --- a/docs/reference/connector/apis/update-connector-pipeline-api.asciidoc +++ b/docs/reference/connector/apis/update-connector-pipeline-api.asciidoc @@ -6,11 +6,18 @@ beta::[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs]. +-- + Updates the `pipeline` configuration of a connector. When you create a new connector, the configuration of an <> is populated with default settings. -To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. +To get started with Connector APIs, check out <>. + [[update-connector-pipeline-api-request]] ==== {api-request-title} @@ -20,7 +27,7 @@ To get started with Connector APIs, check out the {enterprise-search-ref}/connec [[update-connector-pipeline-api-prereq]] ==== {api-prereq-title} -* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors. +* To sync data using self-managed connectors, you need to deploy the <>. on your own infrastructure. This service runs automatically on Elastic Cloud for Elastic managed connectors. * The `connector_id` parameter should reference an existing connector. [[update-connector-pipeline-api-path-params]] diff --git a/docs/reference/connector/apis/update-connector-scheduling-api.asciidoc b/docs/reference/connector/apis/update-connector-scheduling-api.asciidoc index f932f4c959de2..eed3d14ea1d97 100644 --- a/docs/reference/connector/apis/update-connector-scheduling-api.asciidoc +++ b/docs/reference/connector/apis/update-connector-scheduling-api.asciidoc @@ -6,9 +6,16 @@ beta::[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs]. +-- + Updates the `scheduling` configuration of a connector. -To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. +To get started with Connector APIs, check out <>. + [[update-connector-scheduling-api-request]] ==== {api-request-title} @@ -18,7 +25,7 @@ To get started with Connector APIs, check out the {enterprise-search-ref}/connec [[update-connector-scheduling-api-prereq]] ==== {api-prereq-title} -* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors. +* To sync data using self-managed connectors, you need to deploy the <>. on your own infrastructure. This service runs automatically on Elastic Cloud for Elastic managed connectors. * The `connector_id` parameter should reference an existing connector. [[update-connector-scheduling-api-path-params]] diff --git a/docs/reference/connector/apis/update-connector-service-type-api.asciidoc b/docs/reference/connector/apis/update-connector-service-type-api.asciidoc index 139e9eddf4076..28b4c72b682d2 100644 --- a/docs/reference/connector/apis/update-connector-service-type-api.asciidoc +++ b/docs/reference/connector/apis/update-connector-service-type-api.asciidoc @@ -6,9 +6,16 @@ beta::[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs]. +-- + Updates the `service_type` of a connector. -To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. +To get started with Connector APIs, check out <>. + [[update-connector-service-type-api-request]] ==== {api-request-title} @@ -18,7 +25,7 @@ To get started with Connector APIs, check out the {enterprise-search-ref}/connec [[update-connector-service-type-api-prereq]] ==== {api-prereq-title} -* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors. +* To sync data using self-managed connectors, you need to deploy the <>. on your own infrastructure. This service runs automatically on Elastic Cloud for Elastic managed connectors. * The `connector_id` parameter should reference an existing connector. * The `service_type` must be a valid type as defined by the Connector framework. ** When you change a configured connector's `service_type`, you'll also need to reset its configuration to ensure compatibility. diff --git a/docs/reference/connector/apis/update-connector-status-api.asciidoc b/docs/reference/connector/apis/update-connector-status-api.asciidoc index ee9dfcb5f880f..7812cbff89d1b 100644 --- a/docs/reference/connector/apis/update-connector-status-api.asciidoc +++ b/docs/reference/connector/apis/update-connector-status-api.asciidoc @@ -6,9 +6,16 @@ preview::[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs]. +-- + Updates the `status` of a connector. -To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. +To get started with Connector APIs, check out <>. + [[update-connector-status-api-request]] ==== {api-request-title} @@ -18,7 +25,7 @@ To get started with Connector APIs, check out the {enterprise-search-ref}/connec [[update-connector-status-api-prereq]] ==== {api-prereq-title} -* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors. +* To sync data using self-managed connectors, you need to deploy the <>. on your own infrastructure. This service runs automatically on Elastic Cloud for Elastic managed connectors. * The `connector_id` parameter should reference an existing connector. * The change of `status` must be a valid status transition according to the https://github.com/elastic/connectors/blob/main/docs/CONNECTOR_PROTOCOL.md[Connector Protocol]. diff --git a/docs/reference/connector/docs/_connectors-create-client.asciidoc b/docs/reference/connector/docs/_connectors-create-client.asciidoc new file mode 100644 index 0000000000000..917777a2ac786 --- /dev/null +++ b/docs/reference/connector/docs/_connectors-create-client.asciidoc @@ -0,0 +1,76 @@ +[discrete#es-connectors-{service-name-stub}-client-create-use-the-ui] +==== Use the UI + +To create a new {service-name} connector: + +. In the Kibana UI, navigate to the *Search -> Content -> Connectors* page from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. +. Follow the instructions to create a new *{service-name}* self-managed connector. + +[discrete#es-connectors-{service-name-stub}-client-create-use-the-api] +==== Use the API + +You can use the {es} {ref}/connector-apis.html[Create connector API] to create a new self-managed {service-name} self-managed connector. + +For example: + +[source, console,subs="+attributes"] +---- +PUT _connector/my-{service-name-stub}-connector +{ + "index_name": "my-elasticsearch-index", + "name": "Content synced from {service-name}", + "service_type": "{service-name-stub}" +} +---- +// TEST[skip:can't test in isolation] + + +.You'll also need to *create an API key* for the connector to use. + + +[%collapsible] +=================================== + +[NOTE] +==== +The user needs the cluster privileges `manage_api_key`, `manage_connector` and `write_connector_secrets` to generate API keys programmatically. +==== + +To create an API key for the connector: + +. Run the following command, replacing values where indicated. +Note the `encoded` return values from the response: ++ +[source, console,subs="+attributes"] +---- +POST /_security/api_key +{ + "name": "connector_name-connector-api-key", + "role_descriptors": { + "connector_name-connector-role": { + "cluster": [ + "monitor", + "manage_connector" + ], + "indices": [ + { + "names": [ + "index_name", + ".search-acl-filter-index_name", + ".elastic-connectors*" + ], + "privileges": [ + "all" + ], + "allow_restricted_indices": false + } + ] + } + } +} +---- ++ +. Update your `config.yml` file with the API key `encoded` value. +=================================== + +Refer to the {ref}/connector-apis.html[{es} API documentation] for details of all available Connector APIs. diff --git a/docs/reference/connector/docs/_connectors-create-native-api-key.asciidoc b/docs/reference/connector/docs/_connectors-create-native-api-key.asciidoc new file mode 100644 index 0000000000000..99fde477eea5b --- /dev/null +++ b/docs/reference/connector/docs/_connectors-create-native-api-key.asciidoc @@ -0,0 +1,61 @@ +[NOTE] +==== +The user needs the cluster privileges `manage_api_key`, `manage_connector` and `write_connector_secrets` to generate API keys programmatically. +==== + +To create an API key for the connector: + +. Run the following command, replacing values where indicated. +Note the `id` and `encoded` return values from the response: ++ +[source, console,subs="+attributes"] +---- +POST /_security/api_key +{ + "name": "my-connector-api-key", + "role_descriptors": { + "my-connector-connector-role": { + "cluster": [ + "monitor", + "manage_connector" + ], + "indices": [ + { + "names": [ + "my-index_name", + ".search-acl-filter-my-index_name", + ".elastic-connectors*" + ], + "privileges": [ + "all" + ], + "allow_restricted_indices": false + } + ] + } + } +} +---- ++ +. Use the `encoded` value to store a connector secret, and note the `id` return value from this response: ++ +[source, console,subs="+attributes"] +---- +POST _connector/_secret +{ + "value": "encoded_api_key" +} +---- +// TEST[skip:need to retrieve ids from the response] ++ +. Use the API key `id` and the connector secret `id` to update the connector: ++ +[source, console,subs="+attributes"] +---- +PUT /_connector/my_connector_id>/_api_key_id +{ + "api_key_id": "API key_id", + "api_key_secret_id": "secret_id" +} +---- +// TEST[skip:need to retrieve ids from the response] diff --git a/docs/reference/connector/docs/_connectors-create-native.asciidoc b/docs/reference/connector/docs/_connectors-create-native.asciidoc new file mode 100644 index 0000000000000..8023fbbe61136 --- /dev/null +++ b/docs/reference/connector/docs/_connectors-create-native.asciidoc @@ -0,0 +1,38 @@ +[discrete#es-connectors-{service-name-stub}-create-use-the-ui] +==== Use the UI + +To create a new {service-name} connector: + +. In the Kibana UI, navigate to the *Search -> Content -> Connectors* page from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. +. Follow the instructions to create a new native *{service-name}* connector. + +For additional operations, see <>. + +[discrete#es-connectors-{service-name-stub}-create-use-the-api] +==== Use the API + +You can use the {es} {ref}/connector-apis.html[Create connector API] to create a new native {service-name} connector. + +For example: + +[source, console,subs="+attributes"] +---- +PUT _connector/my-{service-name-stub}-connector +{ + "index_name": "my-elasticsearch-index", + "name": "Content synced from {service-name}", + "service_type": "{service-name-stub}", + "is_native": true +} +---- +// TEST[skip:can't test in isolation] + +.You'll also need to *create an API key* for the connector to use. + +[%collapsible] +=================================== +include::_connectors-create-native-api-key.asciidoc[] +=================================== + +Refer to the {ref}/connector-apis.html[{es} API documentation] for details of all available Connector APIs. + diff --git a/docs/reference/connector/docs/_connectors-docker-instructions.asciidoc b/docs/reference/connector/docs/_connectors-docker-instructions.asciidoc new file mode 100644 index 0000000000000..db536099f1aad --- /dev/null +++ b/docs/reference/connector/docs/_connectors-docker-instructions.asciidoc @@ -0,0 +1,76 @@ +You can deploy the {service-name} connector as a self-managed connector using Docker. +Follow these instructions. + +.*Step 1: Download sample configuration file* +[%collapsible] +==== +Download the sample configuration file. +You can either download it manually or run the following command: + +[source,sh] +---- +curl https://raw.githubusercontent.com/elastic/connectors/main/config.yml.example --output ~/connectors-config/config.yml +---- +// NOTCONSOLE + +Remember to update the `--output` argument value if your directory name is different, or you want to use a different config file name. +==== + +.*Step 2: Update the configuration file for your self-managed connector* +[%collapsible] +==== +Update the configuration file with the following settings to match your environment: + +* `elasticsearch.host` +* `elasticsearch.api_key` +* `connectors` + +If you're running the connector service against a Dockerized version of Elasticsearch and Kibana, your config file will look like this: + +[source,yaml,subs="attributes"] +---- +# When connecting to your cloud deployment you should edit the host value +elasticsearch.host: http://host.docker.internal:9200 +elasticsearch.api_key: + +connectors: + - + connector_id: + service_type: {service-name-stub} + api_key: # Optional. If not provided, the connector will use the elasticsearch.api_key instead + +---- + +Using the `elasticsearch.api_key` is the recommended authentication method. However, you can also use `elasticsearch.username` and `elasticsearch.password` to authenticate with your Elasticsearch instance. + +Note: You can change other default configurations by simply uncommenting specific settings in the configuration file and modifying their values. + +==== + +.*Step 3: Run the Docker image* +[%collapsible] +==== +Run the Docker image with the Connector Service using the following command: + +[source,sh,subs="attributes"] +---- +docker run \ +-v ~/connectors-config:/config \ +--network "elastic" \ +--tty \ +--rm \ +docker.elastic.co/enterprise-search/elastic-connectors:{version}.0 \ +/app/bin/elastic-ingest \ +-c /config/config.yml +---- +==== + +Refer to {connectors-python}/docs/DOCKER.md[`DOCKER.md`^] in the `elastic/connectors` repo for more details. + +Find all available Docker images in the https://www.docker.elastic.co/r/enterprise-search/elastic-connectors[official registry]. + +[TIP] +==== +We also have a quickstart self-managed option using Docker Compose, so you can spin up all required services at once: Elasticsearch, Kibana, and the connectors service. +Refer to this https://github.com/elastic/connectors/tree/main/scripts/stack#readme[README] in the `elastic/connectors` repo for more information. +==== \ No newline at end of file diff --git a/docs/reference/connector/docs/_connectors-list-advanced-rules.asciidoc b/docs/reference/connector/docs/_connectors-list-advanced-rules.asciidoc new file mode 100644 index 0000000000000..b09aa7f1d4e7e --- /dev/null +++ b/docs/reference/connector/docs/_connectors-list-advanced-rules.asciidoc @@ -0,0 +1,14 @@ +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> diff --git a/docs/reference/connector/docs/_connectors-list-clients.asciidoc b/docs/reference/connector/docs/_connectors-list-clients.asciidoc new file mode 100644 index 0000000000000..b56d7458d2924 --- /dev/null +++ b/docs/reference/connector/docs/_connectors-list-clients.asciidoc @@ -0,0 +1,29 @@ +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> diff --git a/docs/reference/connector/docs/_connectors-list-dls.asciidoc b/docs/reference/connector/docs/_connectors-list-dls.asciidoc new file mode 100644 index 0000000000000..f5ea6c1d43c5c --- /dev/null +++ b/docs/reference/connector/docs/_connectors-list-dls.asciidoc @@ -0,0 +1,13 @@ +* <> +* <> +* <> (including Jira Data Center) +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> \ No newline at end of file diff --git a/docs/reference/connector/docs/_connectors-list-incremental.asciidoc b/docs/reference/connector/docs/_connectors-list-incremental.asciidoc new file mode 100644 index 0000000000000..88822164258bb --- /dev/null +++ b/docs/reference/connector/docs/_connectors-list-incremental.asciidoc @@ -0,0 +1,19 @@ +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> \ No newline at end of file diff --git a/docs/reference/connector/docs/_connectors-list-local-content-extraction.asciidoc b/docs/reference/connector/docs/_connectors-list-local-content-extraction.asciidoc new file mode 100644 index 0000000000000..d86583644d574 --- /dev/null +++ b/docs/reference/connector/docs/_connectors-list-local-content-extraction.asciidoc @@ -0,0 +1,16 @@ +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> diff --git a/docs/reference/connector/docs/_connectors-list-native.asciidoc b/docs/reference/connector/docs/_connectors-list-native.asciidoc new file mode 100644 index 0000000000000..9222abe11fc3a --- /dev/null +++ b/docs/reference/connector/docs/_connectors-list-native.asciidoc @@ -0,0 +1,26 @@ +* <> +* <> +* <> (including Confluence Data Center) +* <> +* <> +* <> +* <> +* <> +* <> (including Jira Data Center) +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> \ No newline at end of file diff --git a/docs/reference/connector/docs/_connectors-list-workplace-search.asciidoc b/docs/reference/connector/docs/_connectors-list-workplace-search.asciidoc new file mode 100644 index 0000000000000..f9391f3bf4a1e --- /dev/null +++ b/docs/reference/connector/docs/_connectors-list-workplace-search.asciidoc @@ -0,0 +1,22 @@ +* {workplace-search-ref}/workplace-search-box-connector.html[Box^] +* {workplace-search-ref}/workplace-search-confluence-cloud-connector.html[Confluence Cloud^] +* https://github.com/elastic/connectors-ruby/tree/8.3/lib/connectors_sdk/confluence_cloud[Confluence Cloud^] (Ruby connector package) +* {workplace-search-ref}/workplace-search-confluence-server-connector.html[Confluence Server^] +* {workplace-search-ref}/workplace-search-dropbox-connector.html[Dropbox^] +* {workplace-search-ref}/workplace-search-github-connector.html[GitHub^] +* {workplace-search-ref}/workplace-search-gmail-connector.html[Gmail^] +* {workplace-search-ref}/workplace-search-google-drive-connector.html[Google Drive^] +* {workplace-search-ref}/workplace-search-jira-cloud-connector.html[Jira Cloud^] +* {workplace-search-ref}/workplace-search-jira-server-connector.html[Jira Server^] +* https://github.com/elastic/enterprise-search-network-drive-connector[Network drives^] (Python connector package) +* {workplace-search-ref}/workplace-search-onedrive-connector.html[OneDrive^] +* https://github.com/elastic/enterprise-search-microsoft-outlook-connector[Outlook^] (Python connector package) +* {workplace-search-ref}/workplace-search-salesforce-connector.html[Salesforce^] +* {workplace-search-ref}/workplace-search-servicenow-connector.html[ServiceNow^] +* {workplace-search-ref}/workplace-search-sharepoint-online-connector.html[SharePoint Online^] +* https://github.com/elastic/connectors-ruby/tree/8.3/lib/connectors_sdk/share_point[SharePoint Online^] (Ruby connector package) +* https://github.com/elastic/enterprise-search-sharepoint-server-connector[SharePoint Server^] (Python connector package) +* {workplace-search-ref}/workplace-search-slack-connector.html[Slack^] +* {workplace-search-ref}/microsoft-teams.html[Teams^] (Python connector package) +* {workplace-search-ref}/workplace-search-zendesk-connector.html[Zendesk^] +* {workplace-search-ref}/zoom.html[Zoom^] (Python connector package) diff --git a/docs/reference/connector/docs/_connectors-overview-table.asciidoc b/docs/reference/connector/docs/_connectors-overview-table.asciidoc new file mode 100644 index 0000000000000..f5f8103349dde --- /dev/null +++ b/docs/reference/connector/docs/_connectors-overview-table.asciidoc @@ -0,0 +1,51 @@ +This table provides an overview of our available connectors, their current support status, and the features they support. + +The columns provide specific information about each connector: + +* *Status*: Indicates whether the connector is in General Availability (GA), Technical Preview, Beta, or is an Example connector. +* *Native (Elastic Cloud)*: Specifies the versions in which the connector is available as a managed service, if applicable. +* *Advanced sync rules*: Specifies the versions in which advanced sync rules are supported, if applicable. +* *Local binary extraction service*: Specifies the versions in which the local binary extraction service is supported, if applicable. +* *Incremental syncs*: Specifies the version in which incremental syncs are supported, if applicable. +* *Document level security*: Specifies the version in which document level security is supported, if applicable. +* *Code*: Provides a link to the connector's source code in the https://github.com/elastic/connectors[`elastic/connectors` GitHub repository]. + +NOTE: All connectors are available as self-managed <>. + +[options="header"] +|==================================================================================================================================================================================================== +| Connector| Status| Native (Elastic Cloud) | <> | <> | <> | <> | Source code +|<>|*GA*|8.9+|-|8.11+|8.13+|-|https://github.com/elastic/connectors/tree/main/connectors/sources/azure_blob_storage.py[View code] +|<>|*Preview*|8.14+|-|-|8.13+|-|https://github.com/elastic/connectors/tree/main/connectors/sources/box.py[View code] +|<>|*GA*|8.9+|8.9+|8.11+|8.13+|8.10|https://github.com/elastic/connectors/tree/main/connectors/sources/confluence.py[View code] +|<>|*Preview*|8.13+|8.13+|8.13+|8.13+|8.14+|https://github.com/elastic/connectors/tree/main/connectors/sources/confluence.py[View code] +|<>|*GA*|8.9+|8.9+|8.11+|8.13+|8.14+|https://github.com/elastic/connectors/tree/main/connectors/sources/confluence.py[View code] +|<>|*GA*|8.10+|-|8.11+|8.13+|8.12+|https://github.com/elastic/connectors/tree/main/connectors/sources/dropbox.py[View code] +|<>|*GA*|8.11+|8.10+|8.11+|8.13+|8.12+|https://github.com/elastic/connectors/tree/main/connectors/sources/github.py[View code] +|<>|*GA*|8.13+|-|-|8.13+|8.10+|https://github.com/elastic/connectors/tree/main/connectors/sources/gmail.py[View code] +|<>|*GA*|8.12+|-|8.11+|8.13+|-|https://github.com/elastic/connectors/tree/main/connectors/sources/google_cloud_storage.py[View code] +|<>|*GA*|8.11+|-|8.11+|8.13+|8.10+|https://github.com/elastic/connectors/tree/main/connectors/sources/google_drive.py[View code] +|<>|*Preview*|-|-|-|-|-|https://github.com/elastic/connectors/tree/main/connectors/sources/graphql.py[View code] +|<>|*GA*|8.9+|8.9+|8.11+|8.13+|8.10+|https://github.com/elastic/connectors/tree/main/connectors/sources/jira.py[View code] +|<>|*Preview*|8.13+|8.13+|8.13+|8.13+|8.13+*|https://github.com/elastic/connectors/tree/main/connectors/sources/jira.py[View code] +|<>|*GA*|8.9+|8.9+|8.11+|8.13+|-|https://github.com/elastic/connectors/tree/main/connectors/sources/jira.py[View code] +|<>|*GA*|8.8|8.8 native/ 8.12 self-managed|-|-|-|https://github.com/elastic/connectors/tree/main/connectors/sources/mongo.py[View code] +|<>|*GA*|8.8+|8.11+|-|-|-|https://github.com/elastic/connectors/tree/main/connectors/sources/mssql.py[View code] +|<>|*GA*|8.5+|8.8+|-|-|-|https://github.com/elastic/connectors/tree/main/connectors/sources/mysql.py[View code] +|<>|*GA*|8.9+|8.10+|8.14+|8.13+|8.11+|https://github.com/elastic/connectors/tree/main/connectors/sources/network_drive.py[View code] +|<>|*GA*|8.14+|8.14+|-|-|-|https://github.com/elastic/connectors/tree/main/connectors/sources/notion.py[View code] +|<>|*GA*|8.11+|8.11+|8.11+|8.13+|8.11+|https://github.com/elastic/connectors/tree/main/connectors/sources/onedrive.py[View code] +|<>|Example|n/a|n/a|n/a|n/a|-|https://github.com/elastic/connectors/tree/main/connectors/sources/opentext_documentum.py[View code] +|<>|*GA*|8.12+|-|-|-|-|https://github.com/elastic/connectors/tree/main/connectors/sources/oracle.py[View code] +|<>|*GA*|8.13+|-|8.11+|8.13+|8.14+|https://github.com/elastic/connectors/tree/main/connectors/sources/outlook.py[View code] +|<>|*GA*|8.8+|8.11+|-|-|-|https://github.com/elastic/connectors/tree/main/connectors/sources/postgresql.py[View code] +|<>|*Preview*|-|-|-|-|-|https://github.com/elastic/connectors/tree/main/connectors/sources/redis.py[View code] +|<>|*GA*|8.12+|8.12+|8.11+|-|-|https://github.com/elastic/connectors/tree/main/connectors/sources/s3.py[View code] +|<>|*GA*|8.12+|8.12+|8.11+|8.13+|8.13+|https://github.com/elastic/connectors/tree/main/connectors/sources/salesforce.py[View code] +|<>|*GA*|8.10+|8.10+|8.11+|8.13+|8.13+|https://github.com/elastic/connectors/tree/main/connectors/sources/servicenow.py[View code] +|<>|*GA*|8.9+|8.9+|8.9+|8.9+|8.9+|https://github.com/elastic/connectors/tree/main/connectors/sources/sharepoint_online.py[View code] +|<>|*Beta*|8.15+|-|8.11+|8.13+|8.15+|https://github.com/elastic/connectors/tree/main/connectors/sources/sharepoint_server.py[View code] +|<>|*Preview*|8.14+|-|-|-|-|https://github.com/elastic/connectors/tree/main/connectors/sources/slack.py[View code] +|<>|*Preview*|8.14+|-|-|8.13+|-|https://github.com/elastic/connectors/tree/main/connectors/sources/teams.py[View code] +|<>|*Preview*|8.14+|-|8.11+|8.13+|-|https://github.com/elastic/connectors/tree/main/connectors/sources/zoom.py[View code] +|==================================================================================================================================================================================================== diff --git a/docs/reference/connector/docs/connectors-API-tutorial.asciidoc b/docs/reference/connector/docs/connectors-API-tutorial.asciidoc new file mode 100644 index 0000000000000..2e26e0d2a361d --- /dev/null +++ b/docs/reference/connector/docs/connectors-API-tutorial.asciidoc @@ -0,0 +1,487 @@ +[#es-connectors-tutorial-api] +=== Connector API tutorial +++++ +API tutorial +++++ + +Learn how to set up a self-managed connector using the {ref}/connector-apis.html[{es} Connector APIs]. + +For this example we'll use the connectors-postgresql,PostgreSQL connector to sync data from a PostgreSQL database to {es}. +We'll spin up a simple PostgreSQL instance in Docker with some example data, create a connector, and sync the data to {es}. +You can follow the same steps to set up a connector for another data source. + +[TIP] +==== +This tutorial focuses on running a self-managed connector on your own infrastructure, and managing syncs using the Connector APIs. +See connectors for an overview of how connectors work. + +If you're just getting started with {es}, this tutorial might be a bit advanced. +Refer to {ref}/getting-started.html[quickstart] for a more beginner-friendly introduction to {es}. + +If you're just getting started with connectors, you might want to start in the UI first. +We have two tutorials that focus on managing connectors using the UI: + +* <>. Set up a native MongoDB connector, fully managed in Elastic Cloud. +* <>. Set up a self-managed PostgreSQL connector. +==== + +[discrete#es-connectors-tutorial-api-prerequisites] +==== Prerequisites + +* You should be familiar with how connectors, connectors work, to understand how the API calls relate to the overall connector setup. +* You need to have https://www.docker.com/products/docker-desktop/[Docker Desktop] installed. +* You need to have {es} running, and an API key to access it. +Refer to the next section for details, if you don't have an {es} deployment yet. + +[discrete#es-connectors-tutorial-api-setup-es] +==== Set up {es} + +If you already have an {es} deployment on Elastic Cloud (_Hosted deployment_ or _Serverless project_), you're good to go. +To spin up {es} in local dev mode in Docker for testing purposes, open the collapsible section below. + +.*Run local {es} in Docker* +[%collapsible] +=============== + +[source,sh,subs="attributes+"] +---- +docker run -p 9200:9200 -d --name elasticsearch \ + -e "discovery.type=single-node" \ + -e "xpack.security.enabled=false" \ + -e "xpack.security.http.ssl.enabled=false" \ + -e "xpack.license.self_generated.type=trial" \ + docker.elastic.co/elasticsearch/elasticsearch:{version} +---- + +[WARNING] +==== +This {es} setup is for development purposes only. +Never use this configuration in production. +Refer to {ref}/setup.html[Set up {es}] for production-grade installation instructions, including Docker. +==== + +We will use the default password `changeme` for the `elastic` user. For production environments, always ensure your cluster runs with security enabled. + +[source,sh] +---- +export ELASTIC_PASSWORD="changeme" +---- + +Since we run our cluster locally with security disabled, we won't use API keys to authenticate against the {es}. Instead, in each cURL request, we will use the `-u` flag for authentication. + +Let's test that we can access {es}: + +[source,sh] +---- +curl -s -X GET -u elastic:$ELASTIC_PASSWORD http://localhost:9200 +---- +// NOTCONSOLE + +Note: With {es} running locally, you will need to pass the username and password to authenticate against {es} in the configuration file for the connector service. + +=============== + +.Running API calls +**** + +You can run API calls using the https://www.elastic.co/guide/en/kibana/master/console-kibana.html[Dev Tools Console] in Kibana, using `curl` in your terminal, or with our programming language clients. +Our example widget allows you to copy code examples in both Dev Tools Console syntax and curl syntax. +To use curl, you'll need to add authentication headers to your request. + +Here's an example of how to do that. Note that if you want the connector ID to be auto-generated, use the `POST _connector` endpoint. + +[source,sh] +---- +curl -s -X PUT http://localhost:9200/_connector/my-connector-id \ +-H "Authorization: APIKey $APIKEY" \ +-H "Content-Type: application/json" \ +-d '{ + "name": "Music catalog", + "index_name": "music", + "service_type": "postgresql" +}' +---- +// NOTCONSOLE + +Refer to connectors-tutorial-api-create-api-key for instructions on creating an API key. +**** + +[discrete#es-connectors-tutorial-api-setup-postgres] +==== Run PostgreSQL instance in Docker (optional) + +For this tutorial, we'll set up a PostgreSQL instance in Docker with some example data. +Of course, you can *skip this step and use your own existing PostgreSQL instance* if you have one. +Keep in mind that using a different instance might require adjustments to the connector configuration described in the next steps. + +.*Expand* to run simple PostgreSQL instance in Docker and import example data +[%collapsible] +=============== + +Let's launch a PostgreSQL container with a user and password, exposed at port `5432`: + +[source,sh] +---- +docker run --name postgres -e POSTGRES_USER=myuser -e POSTGRES_PASSWORD=mypassword -p 5432:5432 -d postgres +---- + +*Download and import example data* + +Next we need to create a directory to store our example dataset for this tutorial. +In your terminal, run the following command: + +[source,sh] +---- +mkdir -p ~/data +---- + +We will use the https://github.com/lerocha/chinook-database/blob/master/ChinookDatabase/DataSources/Chinook_PostgreSql.sql[Chinook dataset] example data. + +Run the following command to download the file to the `~/data` directory: + +[source,sh] +---- +curl -L https://raw.githubusercontent.com/lerocha/chinook-database/master/ChinookDatabase/DataSources/Chinook_PostgreSql.sql -o ~/data/Chinook_PostgreSql.sql +---- +// NOTCONSOLE + +Now we need to import the example data into the PostgreSQL container and create the tables. + +Run the following Docker commands to copy our sample data into the container and execute the `psql` script: + +[source,sh] +---- +docker cp ~/data/Chinook_PostgreSql.sql postgres:/ +docker exec -it postgres psql -U myuser -f /Chinook_PostgreSql.sql +---- + +Let's verify that the tables are created correctly in the `chinook` database: + +[source,sh] +---- +docker exec -it postgres psql -U myuser -d chinook -c "\dt" +---- + +The `album` table should contain *347* entries and the `artist` table should contain *275* entries. +=============== + +This tutorial uses a very basic setup. To use advanced functionality such as filtering rules and incremental syncs, enable `track_commit_timestamp` on your PostgreSQL database. Refer to postgresql-connector-client-tutorial for more details. + +Now it's time for the real fun! We'll set up a connector to create a searchable mirror of our PostgreSQL data in {es}. + +[discrete#es-connectors-tutorial-api-create-connector] +==== Create a connector + +We'll use the https://www.elastic.co/guide/en/elasticsearch/reference/master/create-connector-api.html[Create connector API] to create a PostgreSQL connector instance. + +Run the following API call, using the https://www.elastic.co/guide/en/kibana/master/console-kibana.html[Dev Tools Console] or `curl`: + +[source,console] +---- +PUT _connector/my-connector-id +{ + "name": "Music catalog", + "index_name": "music", + "service_type": "postgresql" +} +---- +// TEST[skip:TODO] + +[TIP] +==== +`service_type` refers to the third-party data source you're connecting to. +==== + +Note that we specified the `my-connector-id` ID as a part of the `PUT` request. +We'll need the connector ID to set up and run the connector service locally. + +If you'd prefer to use an autogenerated ID, replace `PUT _connector/my-connector-id` with `POST _connector`. + +[discrete#es-connectors-tutorial-api-deploy-connector] +==== Run connector service + +[NOTE] +==== +The connector service runs automatically in Elastic Cloud, if you're using our managed Elastic managed connectors. +Because we're running a self-managed connector, we need to spin up this service locally. +==== + +Now we'll run the connector service so we can start syncing data from our PostgreSQL instance to {es}. +We'll use the steps outlined in connectors-run-from-docker. + +When running the connectors service on your own infrastructure, you need to provide a configuration file with the following details: + +* Your {es} endpoint (`elasticsearch.host`) +* An {es} API key (`elasticsearch.api_key`) +* Your third-party data source type (`service_type`) +* Your connector ID (`connector_id`) + +[discrete#es-connectors-tutorial-api-create-api-key] +===== Create an API key + +If you haven't already created an API key to access {es}, you can use the {ref}/security-api-create-api-key.html[_security/api_key] endpoint. + +Here, we assume your target {es} index name is `music`. If you use a different index name, adjust the request body accordingly. + +[source,console] +---- +POST /_security/api_key +{ + "name": "music-connector", + "role_descriptors": { + "music-connector-role": { + "cluster": [ + "monitor", + "manage_connector" + ], + "indices": [ + { + "names": [ + "music", + ".search-acl-filter-music", + ".elastic-connectors*" + ], + "privileges": [ + "all" + ], + "allow_restricted_indices": false + } + ] + } + } +} +---- +// TEST[skip:TODO] + +You'll need to use the `encoded` value from the response as the `elasticsearch.api_key` in your configuration file. + +[TIP] +==== +You can also create an API key in the {kib} and Serverless UIs. +==== + +[discrete#es-connectors-tutorial-api-prepare-configuration-file] +===== Prepare the configuration file + +Let's create a directory and a `config.yml` file to store the connector configuration: + +[source,sh] +---- +mkdir -p ~/connectors-config +touch ~/connectors-config/config.yml +---- + +Now, let's add our connector details to the config file. +Open `config.yml` and paste the following configuration, replacing placeholders with your own values: + +[source,yaml] +---- +elasticsearch.host: # Your Elasticsearch endpoint +elasticsearch.api_key: # Your Elasticsearch API key + +connectors: + - connector_id: "my-connector-id" + service_type: "postgresql" +---- + +We provide an https://raw.githubusercontent.com/elastic/connectors/main/config.yml.example[example configuration file] in the `elastic/connectors` repository for reference. + +[discrete#es-connectors-tutorial-api-run-connector-service] +===== Run the connector service + +Now that we have the configuration file set up, we can run the connector service locally. +This will point your connector instance at your {es} deployment. + +Run the following Docker command to start the connector service: + +[source,sh,subs="attributes+"] +---- +docker run \ +-v "$HOME/connectors-config:/config" \ +--rm \ +--tty -i \ +--network host \ +docker.elastic.co/enterprise-search/elastic-connectors:{version}.0 \ +/app/bin/elastic-ingest \ +-c /config/config.yml +---- + +Verify your connector is connected by getting the connector status (should be `needs_configuration`) and `last_seen` field (note that time is reported in UTC). +The `last_seen` field indicates that the connector successfully connected to {es}. + +[source, console] +---- +GET _connector/my-connector-id +---- +// TEST[skip:TODO] + +[discrete#es-connectors-tutorial-api-update-connector-configuration] +==== Configure connector + +Now our connector instance is up and running, but it doesn't yet know _where_ to sync data from. +The final piece of the puzzle is to configure our connector with details about our PostgreSQL instance. +When setting up a connector in the Elastic Cloud or Serverless UIs, you're prompted to add these details in the user interface. + +But because this tutorial is all about working with connectors _programmatically_, we'll use the {ref}/update-connector-configuration-api.html[Update connector configuration API] to add our configuration details. + +[TIP] +==== +Before configuring the connector, ensure that the configuration schema is registered by the service. +For Elastic managed connectors, this occurs shortly after creation via the API. +For self-managed connectors, the schema registers on service startup (once the `config.yml` is populated). + +Configuration updates via the API are possible only _after schema registration_. +Verify this by checking the configuration property returned by the `GET _connector/my-connector-id` request. +It should be non-empty. +==== + +Run the following API call to configure the connector with our connectors-postgresql-client-configuration,PostgreSQL configuration details: + +[source, console] +---- +PUT _connector/my-connector-id/_configuration +{ + "values": { + "host": "127.0.0.1", + "port": 5432, + "username": "myuser", + "password": "mypassword", + "database": "chinook", + "schema": "public", + "tables": "album,artist" + } +} +---- +// TEST[skip:TODO] + +[NOTE] +==== +Configuration details are specific to the connector type. +The keys and values will differ depending on which third-party data source you're connecting to. +Refer to the individual connectors-references,connector references for these configuration details. +==== + +[discrete#es-connectors-tutorial-api-sync] +==== Sync data + +[NOTE] +==== +We're using a self-managed connector in this tutorial. +To use these APIs with an Elastic managed connector, there's some extra setup for API keys. +Refer to <> for details. +==== + +We're now ready to sync our PostgreSQL data to {es}. +Run the following API call to start a full sync job: + +[source, console] +---- +POST _connector/_sync_job +{ + "id": "my-connector-id", + "job_type": "full" +} +---- +// TEST[skip:TODO] + +To store data in {es}, the connector needs to create an index. +When we created the connector, we specified the `music` index. +The connector will create and configure this {es} index before launching the sync job. + +[TIP] +==== +In the approach we've used here, the connector will use {ref}/mapping.html#mapping-dynamic[dynamic mappings] to automatically infer the data types of your fields. +In a real-world scenario you would use the {es} {ref}/indices-create-index.html[Create index API] to first create the index with the desired field mappings and index settings. +Defining your own mappings upfront gives you more control over how your data is indexed. +==== + +[discrete#es-connectors-tutorial-api-check-sync-status] +===== Check sync status + +Use the {ref}/get-connector-sync-job-api.html[Get sync job API] to track the status and progress of the sync job. +By default, the most recent job statuses are returned first. +Run the following API call to check the status of the sync job: + +[source, console] +---- +GET _connector/_sync_job?connector_id=my-connector-id&size=1 +---- +// TEST[skip:TODO] + +The job document will be updated as the sync progresses, you can check it as often as you'd like to poll for updates. + +Once the job completes, the status should be `completed` and `indexed_document_count` should be *622*. + +Verify that data is present in the `music` index with the following API call: + +[source, console] +---- +GET music/_count +---- +// TEST[skip:TODO] + +{es} stores data in documents, which are JSON objects. +List the individual documents with the following API call: + +[source, console] +---- +GET music/_search +---- +// TEST[skip:TODO] + +[discrete#es-connectors-tutorial-api-troubleshooting] +=== Troubleshooting + +Use the following command to inspect the latest sync job's status: + +[source, console] +---- +GET _connector/_sync_job?connector_id=my-connector-id&size=1 +---- +// TEST[skip:TODO] + +If the connector encountered any errors during the sync, you'll find these in the `error` field. + +[discrete#es-connectors-tutorial-api-cleanup] +==== Cleaning up + +To delete the connector and its associated sync jobs run this command: + +[source, console] +---- +DELETE _connector/my-connector-id&delete_sync_jobs=true +---- +// TEST[skip:TODO] + +This won't delete the Elasticsearch index that was created by the connector to store the data. +Delete the `music` index by running the following command: + +[source, console] +---- +DELETE music +---- +// TEST[skip:TODO] + +To remove the PostgreSQL container, run the following commands: + +[source,sh] +---- +docker stop postgres +docker rm postgres +---- + +To remove the connector service, run the following commands: +[source,sh] +---- +docker stop +docker rm +---- + +[discrete#es-connectors-tutorial-api-next-steps] +==== Next steps + +Congratulations! You've successfully set up a self-managed connector using the Connector APIs. + +Here are some next steps to explore: + +* Learn more about the {ref}/connector-apis.html[Connector APIs]. +* Learn how to deploy {es}, {kib}, and the connectors service using Docker Compose in our https://github.com/elastic/connectors/tree/main/scripts/stack#readme[quickstart guide]. diff --git a/docs/reference/connector/docs/connectors-APIs.asciidoc b/docs/reference/connector/docs/connectors-APIs.asciidoc new file mode 100644 index 0000000000000..ad6ad78353fc0 --- /dev/null +++ b/docs/reference/connector/docs/connectors-APIs.asciidoc @@ -0,0 +1,22 @@ +[#es-connectors-apis] +== Connector APIs + +In 8.12 we introduced a set of APIs to create and manage Elastic connectors and sync jobs. +Learn more in the {es} REST API documentation: + +* {ref}/connector-apis.html[Connector APIs] +* {ref}/connector-apis.html#sync-job-apis[Sync job APIs] + +.Connector API tutorial +[sidebar] +-- +Check out this concrete <> to get started with the Connector APIs. +-- + +[discrete#es-connectors-apis-cli] +=== Command-line interface + +We also provide a command-line interface. +Learn more in the https://github.com/elastic/connectors/blob/main/docs/CLI.md[`elastic/connectors` repo]. + +Use these tools if you'd like to work with connectors and sync jobs programmatically, without using the UI. \ No newline at end of file diff --git a/docs/reference/connector/docs/connectors-architecture.asciidoc b/docs/reference/connector/docs/connectors-architecture.asciidoc new file mode 100644 index 0000000000000..3269f2badb308 --- /dev/null +++ b/docs/reference/connector/docs/connectors-architecture.asciidoc @@ -0,0 +1,45 @@ +[#es-connectors-overview-architecture] +=== Internal knowledge search architecture +++++ +Internal knowledge search +++++ + +The following section provides a high-level overview of common architecture approaches for the internal knowledge search use case (AKA workplace search). + +[discrete#es-connectors-overview-architecture-hybrid] +==== Hybrid architecture + +Data is synced to an Elastic Cloud deployment through managed connectors and/or self-managed connectors. +A self-managed search application exposes the relevant data that your end users are authorized to see in a search experience. + +Summary: + +* The best combination in terms of flexibility and out-of-the box functionality +* Integrates with Elastic Cloud hosted managed connectors to bring data to Elasticsearch with minimal operational overhead +* Self-managed connectors allow enterprises to adhere to strict access policies when using firewalls that don't allow incoming connections to data sources, while outgoing traffic is easier to control +* Provides additional functionality available for self-managed connectors such as the <> +* Basic functionality available for Standard licenses, advanced features for Platinum licenses + +The following diagram provides a high-level overview of the hybrid internal knowledge search architecture. + +[.screenshot] +image::images/hybrid-architecture.png[align="center",width="50%"] + +[discrete#es-connectors-overview-architecture-self-managed] +==== Self-managed architecture + +Data is synced to an Elastic deployment through self-managed connectors. +A self-managed search application exposes the relevant data that your end users are authorized to see in a search experience. + +Summary: + +* Gives flexibility to build custom solutions tailored to specific business requirements and internal processes +* Allows enterprises to adhere to strict access policies when using firewalls that don't allow incoming connections to data sources, while outgoing traffic is easier to control +* Provides additional functionality available for self-managed connectors such as the <> +* Feasible for air-gapped environments +* Requires Platinum license for full spectrum of features and self-managed connectors + +The following diagram provides a high-level overview of the self-managed internal knowledge search architecture. + +[.screenshot] +image::images/self-managed-architecture.png[align="center",width="50%"] \ No newline at end of file diff --git a/docs/reference/connector/docs/connectors-azure-blob.asciidoc b/docs/reference/connector/docs/connectors-azure-blob.asciidoc new file mode 100644 index 0000000000000..0e0978a423e3c --- /dev/null +++ b/docs/reference/connector/docs/connectors-azure-blob.asciidoc @@ -0,0 +1,291 @@ +[#es-connectors-azure-blob] +=== Elastic Azure Blob Storage connector reference +++++ +Azure Blob Storage +++++ +// Attributes used in this file +:service-name: Azure Blob Storage +:service-name-stub: azure_blob_storage + +The _Elastic Azure Blob Storage connector_ is a <> for https://azure.microsoft.com/en-us/services/storage/blobs/[Azure Blob Storage^]. + +This connector is written in Python using the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/{service-name-stub}.py[*source code* for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + +.Choose your connector reference +******************************* +Are you using a managed connector on Elastic Cloud or a self-managed connector? Expand the documentation based on your deployment method. +******************************* + +// //////// //// //// //// //// //// //// //////// +// //////// NATIVE CONNECTOR REFERENCE /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-azure-blob-native-connector-reference] +==== *Elastic managed connector reference* + +.View *Elastic managed connector* reference + +[%collapsible] +=============== + +[discrete#es-connectors-azure-blob-availability-prerequisites] +===== Availability and prerequisites + +This connector is available as a *managed connector* on Elastic Cloud, as of *8.9.1*. + +To use this connector natively in Elastic Cloud, satisfy all <>. + +[discrete#es-connectors-azure-blob-compatability] +===== Compatibility + +This connector has not been tested with Azure Government. +Therefore we cannot guarantee that it will work with Azure Government endpoints. +For more information on Azure Government compared to Global Azure, refer to the + https://learn.microsoft.com/en-us/azure/azure-government/compare-azure-government-global-azure[official Microsoft documentation^]. + +[discrete#es-connectors-{service-name-stub}-create-native-connector] +===== Create {service-name} connector + +include::_connectors-create-native.asciidoc[] + +[discrete#es-connectors-azure-blob-usage] +===== Usage + +To use this connector as a *managed connector*, see <>. + +For additional operations, see <>. + +[discrete#es-connectors-azure-blob-configuration] +===== Configuration + +The following configuration fields are required to set up the connector: + +Account name:: +Name of Azure Blob Storage account. + +Account key:: +https://learn.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage?tabs=azure-portal[Account key^] for the Azure Blob Storage account. + +Blob endpoint:: +Endpoint for the Blob Service. + +Containers:: +List of containers to index. +`*` will index all containers. + +[discrete#es-connectors-azure-blob-documents-syncs] +===== Documents and syncs + +The connector will fetch all data available in the container. + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted. (Self-managed connectors can use the <> to handle larger binary files.) +* Permissions are not synced. +**All documents** indexed to an Elastic deployment will be visible to **all users with access** to that Elastic Deployment. +==== + +[discrete#es-connectors-azure-blob-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-azure-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +Advanced sync rules are not available for this connector in the present version. +Currently filtering is controlled via ingest pipelines. + +[discrete#es-connectors-azure-blob-content-extraction] +===== Content extraction + +See <>. + +[discrete#es-connectors-azure-blob-known-issues] +===== Known issues + +This connector has the following known issues: + +* *`lease data` and `tier` fields are not updated in Elasticsearch indices* ++ +This is because the blob timestamp is not updated. +Refer to https://github.com/elastic/connectors-python/issues/289[Github issue^]. + +[discrete#es-connectors-azure-blob-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-azure-blob-security] +===== Security + +See <>. + +View the {connectors-python}/connectors/sources/azure_blob_storage.py[source code for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_) + +// Closing the collapsible section +=============== + + +// //////// //// //// //// //// //// //// //////// +// //////// CONNECTOR CLIENT REFERENCE /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-azure-blob-connector-client-reference] +==== *Self-managed connector* + +.View *self-managed connector* reference + +[%collapsible] +=============== + +[discrete#es-connectors-azure-blob-client-availability-prerequisites] +===== Availability and prerequisites + +This connector is available as a self-managed *self-managed connector*. +This self-managed connector is compatible with Elastic versions *8.6.0+*. +To use this connector, satisfy all <>. + +[discrete#es-connectors-azure-blob-client-compatability] +===== Compatibility + +This connector has not been tested with Azure Government. +Therefore we cannot guarantee that it will work with Azure Government endpoints. +For more information on Azure Government compared to Global Azure, refer to the + https://learn.microsoft.com/en-us/azure/azure-government/compare-azure-government-global-azure[official Microsoft documentation^]. + +[discrete#es-connectors-{service-name-stub}-create-connector-client] +===== Create {service-name} connector + +include::_connectors-create-client.asciidoc[] + +[discrete#es-connectors-azure-blob-client-usage] +===== Usage + +To use this connector as a *self-managed connector*, see <> +For additional usage operations, see <>. + +[discrete#es-connectors-azure-blob-client-configuration] +===== Configuration + +[TIP] +==== +When using the <> workflow, initially these fields will use the default configuration set in the {connectors-python}/connectors/sources/azure_blob_storage.py[connector source code^]. +These are set in the `get_default_configuration` function definition. + +These configurable fields will be rendered with their respective *labels* in the Kibana UI. +Once connected, you'll be able to update these values in Kibana. +==== + +The following configuration fields are required to set up the connector: + +`account_name`:: +Name of Azure Blob Storage account. + +`account_key`:: +https://learn.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage?tabs=azure-portal[Account key^] for the Azure Blob Storage account. + +`blob_endpoint`:: +Endpoint for the Blob Service. + +`containers`:: +List of containers to index. +`*` will index all containers. + +`retry_count`:: +Number of retry attempts after a failed call. +Default value is `3`. + +`concurrent_downloads`:: +Number of concurrent downloads for fetching content. +Default value is `100`. + +`use_text_extraction_service`:: +Requires a separate deployment of the <>. Requires that ingest pipeline settings disable text extraction. +Default value is `False`. + +[discrete#es-connectors-azure-blob-client-docker] +===== Deployment using Docker + +include::_connectors-docker-instructions.asciidoc[] + +[discrete#es-connectors-azure-blob-client-documents-syncs] +===== Documents and syncs + +The connector will fetch all data available in the container. + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted by default. You can use the <> to handle larger binary files. +* Permissions are not synced. +**All documents** indexed to an Elastic deployment will be visible to **all users with access** to that Elastic Deployment. +==== + +[discrete#es-connectors-azure-blob-client-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-azure-blob-client-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +Advanced sync rules are not available for this connector in the present version. +Currently filtering is controlled via ingest pipelines. + +[discrete#es-connectors-azure-blob-client-content-extraction] +===== Content extraction + +See <>. + +[discrete#es-connectors-azure-blob-client-testing] +===== End-to-end testing + +The connector framework enables operators to run functional tests against a real data source. +Refer to <> for more details. + +To perform E2E testing for the Azure Blob Storage connector, run the following command: + +[source,shell] +---- +$ make ftest NAME=azure_blob_storage +---- + +For faster tests, add the `DATA_SIZE=small` flag: + +[source,shell] +---- +make ftest NAME=azure_blob_storage DATA_SIZE=small +---- + +[discrete#es-connectors-azure-blob-client-known-issues] +===== Known issues + +This connector has the following known issues: + +* *`lease data` and `tier` fields are not updated in Elasticsearch indices* ++ +This is because the blob timestamp is not updated. +Refer to https://github.com/elastic/connectors/issues/289[Github issue^]. + +[discrete#es-connectors-azure-blob-client-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-azure-blob-client-security] +===== Security + +See <>. + +// Closing the collapsible section +=============== diff --git a/docs/reference/connector/docs/connectors-box.asciidoc b/docs/reference/connector/docs/connectors-box.asciidoc new file mode 100644 index 0000000000000..3e95f15d16ccd --- /dev/null +++ b/docs/reference/connector/docs/connectors-box.asciidoc @@ -0,0 +1,379 @@ +[#es-connectors-box] +=== Elastic Box connector reference +++++ +Box +++++ +// Attributes used in this file +:service-name: Box +:service-name-stub: box + +Th Box connector is written in Python using the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/box.py[source code for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + +// //////// //// //// //// //// //// //// //////// +// //////// NATIVE CONNECTOR REFERENCE (MANAGED SERVICE) /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-box-native-connector-reference] +==== *Elastic managed connector reference* + +.View *Elastic managed connector* reference +[%collapsible] +=============== + +[discrete#es-connectors-box-availability-and-prerequisites] +===== Availability and prerequisites + +This connector is available as a *managed connector* as of Elastic version *8.14.0*. + +To use this connector natively in Elastic Cloud, satisfy all <>. + +[NOTE] +==== +This connector is in *technical preview* and is subject to change. +The design and code is less mature than official GA features and is being provided as-is with no warranties. +Technical preview features are not subject to the support SLA of official GA features. +==== + +[discrete#es-connectors-box-create-connector-native] +===== Create a {service-name} connector +include::_connectors-create-native.asciidoc[] + +[discrete#es-connectors-box-usage] +===== Usage + +To use this connector as a *self-managed connector*, use the *Box* tile from the connectors list OR *Customized connector* workflow. + +For additional operations, see <>. + +[discrete#es-connectors-box-api-authorization] +===== Box API Authorization + +[discrete#es-connectors-box-free-account] +====== Box Free Account + +[discrete#es-connectors-box-create-oauth-custom-app] +*Create Box User Authentication (OAuth 2.0) Custom App* + +You'll need to create an OAuth app in the Box developer console by following these steps: + +1. Register a new app in the https://app.box.com/developers/console[Box dev console] with custom App and select User authentication (OAuth 2.0). +2. Add the URL of the web page in *Redirect URIs*, which is accessible by you. +3. Check "Write all files and folders stored in Box" in Application Scopes. +4. Once the app is created, *Client ID* and *Client secret* values are available in the configuration tab. Keep these handy. + +[discrete#es-connectors-box-connector-generate-a-refresh-token] +*Generate a refresh Token* + +To generate a refresh token, follow these steps: + +1. Go to the following URL, replacing `` with the *Client ID* value saved earlier. +For example: ++ +[source,bash] +---- +https://account.box.com/api/oauth2/authorize?response_type=code&client_id= +---- ++ +2. Grant access to your application. +3. You will now be redirected to the web page that you configured in *Redirect URIs*, and the HTTP response should contain an *authorization code* that you'll use to generate a refresh token. +*Note:* Authorization codes to generate refresh tokens can only be used once and are only valid for 30 seconds. +4. In your terminal, run the following `curl` command, replacing ``, ` and ` with the values you saved earlier: ++ +[source,bash] +---- +curl -i -X POST "https://api.box.com/oauth2/token" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "client_id=" \ + -d "client_secret=" \ + -d "code=" \ + -d "grant_type=authorization_code" +---- ++ +Save the refresh token from the response. You'll need this for the connector configuration. + +[discrete#es-connectors-box-enterprise-account] +====== Box Enterprise Account + +[discrete#es-connectors-box-connector-create-box-server-authentication-client-credentials-grant-custom-app] +*Create Box Server Authentication (Client Credentials Grant) Custom App* + +1. Register a new app in the https://app.box.com/developers/console[Box dev console] with custom App and select Server Authentication (Client Credentials Grant). +2. Check following permissions: ++ +* "Write all files and folders stored in Box" in Application Scopes +* "Make API calls using the as-user header" in Advanced Features +3. Select `App + Enterprise Access` in App Access Level. +4. Authorize your application from the admin console. +Save the *Client Credentials* and *Enterprise ID*. You'll need these to configure the connector. + +[discrete#es-connectors-box-configuration] +===== Configuration + +`Box Account` (required):: +Dropdown to determine Box Account type: `Box Free Account` or `Box Enterprise Account`. Default value is `Box Free Account`. + +`Client ID` (required):: +The Client ID to authenticate with Box instance. + +`Client Secret` (required):: +The Client Secret to authenticate with Box instance. + +`Refresh Token` (required if Box Account is Box Free):: +The Refresh Token to generate Access Token. +*NOTE:* If the process terminates, you'll need to generate a new refresh token. + +`Enterprise ID` (required if Box Account is Box Enterprise):: +The Enterprise ID to authenticate with Box instance. + +[discrete#es-connectors-box-content-extraction] +===== Content Extraction + +Refer to <>. + +[discrete#es-connectors-box-documents-and-syncs] +===== Documents and syncs + +The connector syncs the following objects and entities: + +* *Files* +* *Folders* + +[NOTE] +==== +* Files bigger than 10 MB won't be extracted. +* Permissions are not synced. *All documents* indexed to an Elastic deployment will be visible to *all users with access* to that Elastic Deployment. +==== + +[discrete#es-connectors-box-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-box-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +[discrete#es-connectors-box-advanced-sync-rules] +===== Advanced Sync Rules + +Advanced sync rules are not available for this connector in the present version. + +[discrete#es-connectors-box-known-issues] +===== Known issues + +There are no known issues for this connector. +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-box-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-box-security] +===== Security + +See <>. + +// Closing the collapsible section +=============== + + +// //////// //// //// //// //// //// //// //////// +// //////// CONNECTOR CLIENT REFERENCE (SELF-MANAGED) /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-box-connector-client-reference] +==== *Self-managed connector reference* + +.View *self-managed connector* reference +[%collapsible] +=============== + +[discrete#es-connectors-box-client-availability-and-prerequisites] +===== Availability and prerequisites + +This connector is available as a self-managed *self-managed connector*. To use this connector, satisfy all <>. + +[NOTE] +==== +This connector is in *technical preview* and is subject to change. +The design and code is less mature than official GA features and is being provided as-is with no warranties. +Technical preview features are not subject to the support SLA of official GA features. +==== + +[discrete#es-connectors-box-client-create-connector-client] +===== Create a {service-name} connector +include::_connectors-create-client.asciidoc[] + +[discrete#es-connectors-box-client-usage] +===== Usage + +To use this connector as a *self-managed connector*, use the *Box* tile from the connectors list OR *Customized connector* workflow. + +For additional operations, see <>. + +[discrete#es-connectors-box-client-api-authorization] +===== Box API Authorization + +[discrete#es-connectors-box-client-free-account] +====== Box Free Account + +[discrete#es-connectors-box-client-create-oauth-custom-app] +*Create Box User Authentication (OAuth 2.0) Custom App* + +You'll need to create an OAuth app in the Box developer console by following these steps: + +1. Register a new app in the https://app.box.com/developers/console[Box dev console] with custom App and select User authentication (OAuth 2.0). +2. Add the URL of the web page in *Redirect URIs*, which is accessible by you. +3. Check "Write all files and folders stored in Box" in Application Scopes. +4. Once the app is created, *Client ID* and *Client secret* values are available in the configuration tab. Keep these handy. + +[discrete#es-connectors-box-client-connector-generate-a-refresh-token] +*Generate a refresh Token* + +To generate a refresh token, follow these steps: + +1. Go to the following URL, replacing `` with the *Client ID* value saved earlier. +For example: ++ +[source,bash] +---- +https://account.box.com/api/oauth2/authorize?response_type=code&client_id= +---- ++ +2. Grant access to your application. +3. You will now be redirected to the web page that you configured in *Redirect URIs*, and the HTTP response should contain an *authorization code* that you'll use to generate a refresh token. +*Note:* Authorization codes to generate refresh tokens can only be used once and are only valid for 30 seconds. +4. In your terminal, run the following `curl` command, replacing ``, ` and ` with the values you saved earlier: ++ +[source,bash] +---- +curl -i -X POST "https://api.box.com/oauth2/token" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "client_id=" \ + -d "client_secret=" \ + -d "code=" \ + -d "grant_type=authorization_code" +---- ++ +Save the refresh token from the response. You'll need this for the connector configuration. + +[discrete#es-connectors-box-client-enterprise-account] +====== Box Enterprise Account + +[discrete#es-connectors-box-client-connector-create-box-server-authentication-client-credentials-grant-custom-app] +*Create Box Server Authentication (Client Credentials Grant) Custom App* + +1. Register a new app in the https://app.box.com/developers/console[Box dev console] with custom App and select Server Authentication (Client Credentials Grant). +2. Check following permissions: ++ +* "Write all files and folders stored in Box" in Application Scopes +* "Make API calls using the as-user header" in Advanced Features +3. Select `App + Enterprise Access` in App Access Level. +4. Authorize your application from the admin console. +Save the *Client Credentials* and *Enterprise ID*. You'll need these to configure the connector. + +[discrete#es-connectors-box-client-configuration] +===== Configuration + +`Box Account` (required):: +Dropdown to determine Box Account type: `Box Free Account` or `Box Enterprise Account`. Default value is `Box Free Account`. + +`Client ID` (required):: +The Client ID to authenticate with Box instance. + +`Client Secret` (required):: +The Client Secret to authenticate with Box instance. + +`Refresh Token` (required if Box Account is Box Free):: +The Refresh Token to generate Access Token. +*NOTE:* If the process terminates, you'll need to generate a new refresh token. + +`Enterprise ID` (required if Box Account is Box Enterprise):: +The Enterprise ID to authenticate with Box instance. + +[discrete#es-connectors-box-client-client-docker] +====== Deployment using Docker + +include::_connectors-docker-instructions.asciidoc[] + +[discrete#es-connectors-box-client-content-extraction] +===== Content Extraction + +Refer to <>. + +[discrete#es-connectors-box-client-documents-and-syncs] +===== Documents and syncs + +The connector syncs the following objects and entities: + +* *Files* +* *Folders* + +[NOTE] +==== +* Files bigger than 10 MB won't be extracted. +* Permissions are not synced. *All documents* indexed to an Elastic deployment will be visible to *all users with access* to that Elastic Deployment. +==== + +[discrete#es-connectors-box-client-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-box-client-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +[discrete#es-connectors-box-client-advanced-sync-rules] +===== Advanced Sync Rules + +Advanced sync rules are not available for this connector in the present version. + +[discrete#es-connectors-box-client-end-to-end-testing] +===== End-to-end Testing + +The connector framework enables operators to run functional tests against a real data source. +Refer to <> for more details. + +To perform E2E testing for the Box connector, run the following command: + +[source,shell] +---- +$ make ftest NAME=box +---- + +For faster tests, add the `DATA_SIZE=small` flag: + +[source,shell] +---- +make ftest NAME=box DATA_SIZE=small +---- + +[discrete#es-connectors-box-client-known-issues] +===== Known issues + +There are no known issues for this connector. +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-box-client-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-box-client-security] +===== Security + +See <>. + + +// Closing the collapsible section +=============== diff --git a/docs/reference/connector/docs/connectors-confluence.asciidoc b/docs/reference/connector/docs/connectors-confluence.asciidoc new file mode 100644 index 0000000000000..61946745195fc --- /dev/null +++ b/docs/reference/connector/docs/connectors-confluence.asciidoc @@ -0,0 +1,541 @@ +[#es-connectors-confluence] +=== Elastic Confluence connector reference +++++ +Confluence +++++ +// Attributes used in this file +:service-name: Confluence +:service-name-stub: confluence + +The _Elastic Confluence connector_ is a <> for https://www.atlassian.com/software/confluence[Atlassian Confluence^]. +This connector is written in Python using the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/{service-name-stub}.py[*source code* for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + +.Choose your connector reference +******************************* +Are you using a managed connector on Elastic Cloud or a self-managed connector? Expand the documentation based on your deployment method. +******************************* + +// //////// //// //// //// //// //// //// //////// +// //////// NATIVE CONNECTOR REFERENCE /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-confluence-native-connector-reference] +==== *Elastic managed connector reference* + +.View *Elastic managed connector* reference + +[%collapsible] +=============== + +[discrete#es-connectors-confluence-availability-prerequisites] +===== Availability and prerequisites + +This connector is available as a *managed connector* on Elastic Cloud, as of *8.9.1*. + +[NOTE] +==== +Confluence Data Center support was added in 8.13.0 in technical preview and is subject to change. The design and code is less mature than official GA features and is being provided as-is with no warranties. Technical preview features are not subject to the support SLA of official GA features. +==== + +To use this connector natively in Elastic Cloud, satisfy all <>. + +[discrete#es-connectors-confluence-create-native-connector] +===== Create a {service-name} connector +include::_connectors-create-native.asciidoc[] + +[discrete#es-connectors-confluence-usage] +===== Usage + +To use this connector as a *managed connector*, see <>. + +For additional operations, see <>. + +[discrete#es-connectors-confluence-compatability] +===== Compatibility + +* Confluence Cloud or Confluence Server/Data Center *versions 7 or later*. + +[discrete#es-connectors-confluence-configuration] +===== Configuration + +The following configuration fields are required to set up the connector: + +Confluence data source:: +Dropdown to determine the Confluence platform type: `Confluence Cloud`, `Confluence Server`, or `Confluence Data Center`. Default value is `Confluence Server`. + +Confluence Data Center username:: +The username of the account for Confluence Data Center. + +Confluence Data Center password:: +The password of the account to be used for the Confluence Data Center. + +Confluence Server username:: +The username of the account for Confluence server. + +Confluence Server password:: +The password of the account to be used for Confluence Server. + +Confluence Cloud account email:: +The account email for Confluence Cloud. + +Confluence Cloud API token:: +The API Token to authenticate with Confluence cloud. + +Confluence URL label:: +The domain where the Confluence is hosted. Examples: + +* `https://192.158.1.38:8080/` +* `https://test_user.atlassian.net/` + +Confluence space keys:: +Comma-separated list of https://confluence.atlassian.com/doc/space-keys-829076188.html[Space Keys] to fetch data from Confluence server or cloud. If the value is `*`, the connector will fetch data from all spaces present in the configured `spaces`. Default value is `*`. Examples: ++ +* `EC`, `TP` +* `*` + +Enable indexing labels:: +Toggle to enable syncing of labels from pages. +NOTE: This will increase the amount of network calls to the source, and may decrease performance. + +Enable SSL:: +Whether SSL verification will be enabled. Default value is `False`. + +SSL certificate:: +Content of SSL certificate. Note: If `ssl_enabled` is `False`, the value in this field is ignored. Example certificate: ++ +[source, txt] +---- +-----BEGIN CERTIFICATE----- +MIID+jCCAuKgAwIBAgIGAJJMzlxLMA0GCSqGSIb3DQEBCwUAMHoxCzAJBgNVBAYT +... +7RhLQyWn2u00L7/9Omw= +-----END CERTIFICATE----- +---- + +Enable document level security:: +Toggle to enable <>. +When enabled, full syncs will fetch access control lists for each document and store them in the `_allow_access_control` field. +Access control syncs will fetch users' access control lists and store them in a separate index. +[NOTE] +==== +To access user data in Jira Administration, the account you created must be granted *Product Access* for Jira Administration. +This access needs to be provided by an administrator from the http://admin.atlassian.com/[Atlassian Admin], and the access level granted should be `Product Admin`. +==== + +[discrete#es-connectors-confluence-documents-syncs] +===== Documents and syncs + +The connector syncs the following Confluence object types: + +* Pages +* Spaces +* Blog Posts +* Attachments + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted. (Self-managed connectors can use the <> to handle larger binary files.) +* Permissions are not synced by default. +You must first enable <>. +Otherwise, *all documents* indexed to an Elastic deployment will be visible to *all users with access* to that Elastic Deployment. +==== + +[discrete#es-connectors-confluence-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-confluence-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +This connector supports <> for remote filtering. +These rules cover complex query-and-filter scenarios that cannot be expressed with = now('-5w')" + }, + { + "query": "lastmodified < startOfYear()" + } +] +---- +// NOTCONSOLE + +*Example 3*: Query for indexing only given types in a *Space* with key 'SD'. + +[source,js] +---- +[ + { + "query": "type in ('page', 'attachment') AND space.key = 'SD'" + } +] +---- +// NOTCONSOLE + +[NOTE] +==== +Syncing recently created/updated items in Confluence may be delayed when using advanced sync rules, because the search endpoint used for CQL queries returns stale results in the response. +For more details refer to the following issue in the https://jira.atlassian.com/browse/CONFCLOUD-73997[Confluence documentation^]. +==== + +[discrete#es-connectors-confluence-document-level-security] +===== Document level security + +[NOTE] +==== +DLS is automatically available for Atlassian Confluence Cloud since 8.9.0. +DLS is available since 8.14.0 for Confluence Server and Confluence Data Center, but requires installing https://marketplace.atlassian.com/apps/1217507/extender-for-confluence?tab=overview&hosting=datacenter[Extender for Confluence]. +==== + +Document level security (DLS) enables you to restrict access to documents based on a user’s permissions. +Refer to <> on this page for how to enable DLS for this connector. + + +[WARNING] +==== +When the `data_source` is set to Confluence Data Center or Server, the connector will only fetch 1000 users for access control syncs, due a https://auth0.com/docs/manage-users/user-search/retrieve-users-with-get-users-endpoint#limitations[limitation in the API used^]. +==== + +[NOTE] +==== +Refer to <> to learn how to ingest data from a connector with DLS enabled, when building a search application. +The example uses SharePoint Online as the data source, but the same steps apply to every connector. +==== + +[discrete#es-connectors-confluence-content-extraction] +===== Content Extraction + +See <>. + +[discrete#es-connectors-confluence-known-issues] +===== Known issues + +There are currently no known issues for this connector. +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-confluence-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-confluence-security] +===== Security + +See <>. + +// Closing the collapsible section +=============== + + +// //////// //// //// //// //// //// //// //////// +// //////// CONNECTOR CLIENT REFERENCE /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-confluence-connector-client-reference] +==== *Self-managed connector* + +.View *self-managed connector* reference + +[%collapsible] +=============== + +[discrete#es-connectors-confluence-client-availability-prerequisites] +===== Availability and prerequisites + +This connector is available as a *self-managed connector* using the *Elastic connector framework*. +This self-managed connector is compatible with Elastic versions *8.7.0+*. + +[NOTE] +==== +Confluence Data Center support was added in 8.13.0 in technical preview and is subject to change. The design and code is less mature than official GA features and is being provided as-is with no warranties. Technical preview features are not subject to the support SLA of official GA features. +==== + +To use this connector, satisfy all <>. + +[discrete#es-connectors-confluence-create-connector-client] +===== Create a {service-name} connector +include::_connectors-create-client.asciidoc[] + +[discrete#es-connectors-confluence-client-usage] +===== Usage + +To use this connector as a *self-managed connector*, see <> +For additional usage operations, see <>. + +[discrete#es-connectors-confluence-client-compatability] +===== Compatibility + +* Confluence Cloud or Confluence Server/Data Center *versions 7 or later* + +[discrete#es-connectors-confluence-client-configuration] +===== Configuration + +[TIP] +==== +When using the <> workflow, initially these fields will use the default configuration set in the {connectors-python}/connectors/sources/confluence.py[connector source code^]. +These are set in the `get_default_configuration` function definition. + +These configurable fields will be rendered with their respective *labels* in the Kibana UI. +Once connected, you'll be able to update these values in Kibana. +==== + +The following configuration fields are required to set up the connector: + +`data_source`:: +Dropdown to determine the Confluence platform type: `Confluence Cloud`, `Confluence Server`, or `Confluence Data Center`. Default value is `Confluence Server`. + +`data_center_username`:: +The username of the account for Confluence Data Center. + +`data_center_password`:: +The password of the account to be used for the Confluence Data Center. + +`username`:: +The username of the account for Confluence Server. + +`password`:: +The password of the account to be used for the Confluence server. + +`account_email`:: +The account email for the Confluence Cloud. + +`api_token`:: +The API Token to authenticate with Confluence Cloud. + +`confluence_url`:: +The domain where the Confluence instance is hosted. Examples: + +* `https://192.158.1.38:8080/` +* `https://test_user.atlassian.net/` + +`spaces`:: +Comma-separated list of https://confluence.atlassian.com/doc/space-keys-829076188.html[Space Keys] to fetch data from Confluence. If the value is `*`, the connector will fetch data from all spaces present in the configured `spaces`. Default value is `*`. Examples: ++ +* `EC`, `TP` +* `*` + +`index_labels`:: +Toggle to enable syncing of labels from pages. +NOTE: This will increase the amount of network calls to the source, and may decrease performance. + +`ssl_enabled`:: +Whether SSL verification will be enabled. Default value is `False`. + +`ssl_ca`:: +Content of SSL certificate. Note: If `ssl_enabled` is `False`, the value in this field is ignored. Example certificate: ++ +[source, txt] +---- +-----BEGIN CERTIFICATE----- +MIID+jCCAuKgAwIBAgIGAJJMzlxLMA0GCSqGSIb3DQEBCwUAMHoxCzAJBgNVBAYT +... +7RhLQyWn2u00L7/9Omw= +-----END CERTIFICATE----- +---- + +`retry_count`:: +The number of retry attempts after failed request to Confluence. Default value is `3`. + +`concurrent_downloads`:: +The number of concurrent downloads for fetching the attachment content. This speeds up the content extraction of attachments. Defaults to `50`. + +`use_document_level_security`:: +Toggle to enable <>. ++ +When enabled, full syncs will fetch access control lists for each document and store them in the `_allow_access_control` field. +Access control syncs will fetch users' access control lists and store them in a separate index. ++ +[NOTE] +==== +To access user data in Jira Administration, the account you created must be granted *Product Access* for Jira Administration. +This access needs to be provided by an administrator from the http://admin.atlassian.com/[Atlassian Admin], and the access level granted should be `Product Admin`. +==== + +`use_text_extraction_service`:: +Toggle to enable the local text extraction service. Default value is `False`. +Requires a separate deployment of the Elastic Text Extraction Service. +Requires that ingest pipeline settings disable text extraction. + + +[discrete#es-connectors-confluence-client-docker] +===== Deployment using Docker + +include::_connectors-docker-instructions.asciidoc[] + +[discrete#es-connectors-confluence-client-documents-syncs] +===== Documents and syncs + +The connector syncs the following Confluence object types: + +* Pages +* Spaces +* Blog Posts +* Attachments + +[NOTE] +==== +* Content of files bigger than 10 MB won't be extracted. +* Permissions are not synced. **All documents** indexed to an Elastic deployment will be visible to **all users with access** to that Elastic Deployment. +==== + +[discrete#es-connectors-confluence-client-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-confluence-client-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +This connector supports <> for remote filtering. +These rules cover complex query-and-filter scenarios that cannot be expressed with = now('-5w')" + }, + { + "query": "lastmodified < startOfYear()" + } +] +---- +// NOTCONSOLE + +*Example 3*: Query for indexing only given types in a *Space* with key 'SD'. + +[source,js] +---- +[ + { + "query": "type in ('page', 'attachment') AND space.key = 'SD'" + } +] +---- +// NOTCONSOLE + +[NOTE] +==== +Syncing recently created/updated items in Confluence may be delayed when using advanced sync rules, because the search endpoint used for CQL queries returns stale results in the response. +For more details refer to the following issue in the https://jira.atlassian.com/browse/CONFCLOUD-73997[Confluence documentation^]. +==== + +[discrete#es-connectors-confluence-client-document-level-security] +===== Document level security + +[NOTE] +==== +DLS is automatically available for Atlassian Confluence Cloud since 8.9.0. +DLS is available since 8.14.0 for Confluence Server and Confluence Data Center, but requires installing https://marketplace.atlassian.com/apps/1217507/extender-for-confluence?tab=overview&hosting=datacenter[Extender for Confluence]. +==== + +Document level security (DLS) enables you to restrict access to documents based on a user’s permissions. +Refer to <> on this page for how to enable DLS for this connector. + +[WARNING] +==== +When the `data_source` is set to Confluence Data Center or Server, the connector will only fetch 1000 users for access control syncs, due a https://auth0.com/docs/manage-users/user-search/retrieve-users-with-get-users-endpoint#limitations[limitation in the API used^]. +==== + +[NOTE] +==== +Refer to <> to learn how to ingest data from a connector with DLS enabled, when building a search application. +The example uses SharePoint Online as the data source, but the same steps apply to every connector. +==== + +[discrete#es-connectors-confluence-client-content-extraction] +===== Content Extraction + +See <>. + +[discrete#es-connectors-confluence-client-connector-client-operations] +===== Self-managed connector operations + +[discrete#es-connectors-confluence-client-testing] +===== End-to-end testing + +The connector framework enables operators to run functional tests against a real data source. +Refer to <> for more details. + +To perform E2E testing for the Confluence connector, run the following command: + +[source,shell] +---- +$ make ftest NAME=confluence +---- + +For faster tests, add the `DATA_SIZE=small` flag: + +[source,shell] +---- +make ftest NAME=confluence DATA_SIZE=small +---- + +[discrete#es-connectors-confluence-client-known-issues] +===== Known issues + +There are currently no known issues for this connector. +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-confluence-client-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-confluence-client-security] +===== Security + +See <>. + +// Closing the collapsible section +=============== diff --git a/docs/reference/connector/docs/connectors-content-extraction.asciidoc b/docs/reference/connector/docs/connectors-content-extraction.asciidoc new file mode 100644 index 0000000000000..def59e22ca1e9 --- /dev/null +++ b/docs/reference/connector/docs/connectors-content-extraction.asciidoc @@ -0,0 +1,360 @@ +[#es-connectors-content-extraction] +=== Content extraction + +Connectors use the {ref}/attachment.html[Elastic ingest attachment processor^] to extract file contents. +The processor extracts files using the https://tika.apache.org[Apache Tika^] text extraction library. +The logic for content extraction is defined in {connectors-python}/connectors/utils.py[utils.py^]. + +While intended primarily for PDF and Microsoft Office formats, you can use any of the <>. + +Enterprise Search uses an {ref}/ingest.html[Elasticsearch ingest pipeline^] to power the web crawler's binary content extraction. +The default pipeline, `ent-search-generic-ingestion`, is automatically created when Enterprise Search first starts. + +You can {ref}/ingest.html#create-manage-ingest-pipelines[view^] this pipeline in Kibana. +Customizing your pipeline usage is also an option. +See {ref}/ingest-pipeline-search.html[Ingest pipelines for Search indices]. + +For advanced use cases, the <> can be used to extract content from files larger than 10MB. + +[discrete#es-connectors-content-extraction-supported-file-types] +==== Supported file types + +The following file types are supported: + +* `.txt` +* `.py` +* `.rst` +* `.html` +* `.markdown` +* `.json` +* `.xml` +* `.csv` +* `.md` +* `.ppt` +* `.rtf` +* `.docx` +* `.odt` +* `.xls` +* `.xlsx` +* `.rb` +* `.paper` +* `.sh` +* `.pptx` +* `.pdf` +* `.doc` + +[NOTE] +==== +The ingest attachment processor does not support compressed files, e.g., an archive file containing a set of PDFs. +Expand the archive file and make individual uncompressed files available for the connector to process. +==== + +[discrete#es-connectors-content-extraction-local] +==== Extraction Service + +[NOTE] +==== +Currently, content extraction from large files via the Extraction Service is available for a subset of our **self-managed connectors**. +It is not available for Elastic managed connectors running on Elastic Cloud. +This feature is in *beta*. +==== + +Standard content extraction is done via the Attachment Processor, through Elasticsearch Ingest Pipelines. +The self-managed connector limits file sizes for pipeline extraction to 10MB per file (Elasticsearch also has a hard limit of 100MB per file). + +For use cases that require extracting content from files larger than these limits, the *self-managed extraction service* can be used for self-managed connectors. +Instead of sending the file as an `attachment` to Elasticsearch, the file's content is extracted at the edge by the extraction service before ingestion. +The extracted text is then included as the `body` field of a document when it is ingested. + +To use this feature, you will need to do the following: + +* <> +* <> +* Set the value of the configurable field `use_text_extraction_service` to `true` + +[TIP] +==== +The data extraction service code is now available in this public repository: https://github.com/elastic/data-extraction-service. +==== + +[discrete#es-connectors-content-extraction-available-connectors] +===== Available connectors + +Local content extraction is available for the following self-managed connectors: + +include::_connectors-list-local-content-extraction.asciidoc[] + +[discrete#es-connectors-content-extraction-data-extraction-service] +===== Running the extraction service + +Self-hosted content extraction is handled by a *separate* extraction service. + +The versions for the extraction service do not align with the Elastic stack. +For versions after `8.11.x` (including {version}), you should use extraction service version `0.3.x`. + +You can run the service with the following command: + +[source,bash] +---- +$ docker run \ + -p 8090:8090 \ + -it \ + --name extraction-service \ + docker.elastic.co/integrations/data-extraction-service:$EXTRACTION_SERVICE_VERSION +---- + +[discrete#es-connectors-extraction-service-configuration] +===== Configuring the extraction service + +You can enable your self-managed connector to use the self-hosted extraction service by adding the required configuration. +The self-managed connector determines if the extraction service is enabled by the presence of these fields in the configuration file. + +1. Open the `config.yml` configuration file in your text editor of choice. +2. Add the following fields. They can be added anywhere in the file, so long as they begin at the root level. + +[source,yaml] +---- +# data-extraction-service settings +extraction_service: + host: http://localhost:8090 +---- + +[NOTE] +==== +There is no password protection between the self-managed connector and the extraction service. +Self-hosted extraction should only be used if the two services are running on the same network and behind the same firewall. +==== + +[options="header"] +|======= +|Field|Description +|`host`|The endpoint for the extraction service. `http://localhost:8090` can be used if it is running on the same server as your self-managed connector. +|======= + +The self-managed connector will perform a preflight check against the configured `host` value at startup. +The following line will be output to the log if the data extraction service was found and is running normally. + +[source,bash] +---- +Data extraction service found at . +---- + +If you don't see this log at startup, refer to <>. + +[discrete#es-connectors-content-extraction-advanced-configuration] +====== Advanced configuration + +The following fields can be included in the configuration file. +They are optional and will fallback on default values if they are not specified. + +[source,yaml] +---- +# data-extraction-service settings +extraction_service: + host: http://localhost:8090 + timeout: 30 + use_file_pointers: false + stream_chunk_size: 65536 + shared_volume_dir: '/app/files' +---- + +[options="header"] +|======= +|Advanced Field|Description +|`timeout`|Timeout limit in seconds for content extraction. Defaults to `30` if not set. Increase this if you have very large files that timeout during content extraction. In the event of a timeout, the indexed document's `body` field will be an empty string. +|`use_file_pointers`|Whether or not to use file pointers instead of sending files to the extraction service. Defaults to `false`. Refer to <> for more details about this setting. +|`stream_chunk_size`|The size that files are chunked to facilitate streaming to extraction service, in bytes. Defaults to 65536 (64 KB). Only applicable if `use_file_pointers` is `false`. Increasing this value may speed up the connector, but will also increase memory usage. +|`shared_volume_dir`|The shared volume from which the data extraction service will extract files. Defaults to `/app/files`. Only applicable if `use_file_pointers` is `true`. +|======= + +[discrete#es-connectors-content-extraction-data-extraction-service-file-pointers] +===== Using file pointers + +The self-hosted extraction service can be set up to use file pointers instead of sending files via HTTP requests. +File pointers are faster than sending files and consume less memory, but require the connector framework and the extraction service to be able to share a file system. +This can be set up with both a dockerized and non-dockerized self-managed connector. + +[discrete#es-connectors-content-extraction-data-extraction-service-file-pointers-configuration] +====== Configuration for non-dockerized self-managed connectors + +If you are running a non-dockerized version of the self-managed connector, you need to determine the local directory where you'll download files for extraction. +This can be anywhere on your file system that you are comfortable using. +Be aware that the self-managed connector will download files with randomized filenames to this directory, so there is a chance that any files already present will be overwritten. +For that reason, we recommend using a dedicated directory for self-hosted extraction. + +[discrete#es-connectors-content-extraction-data-extraction-service-file-pointers-configuration-example] +*Example* + +1. For this example, we will be using `/app/files` as both our local directory and our container directory. +When you run the extraction service docker container, you can mount the directory as a volume using the command-line option `-v /app/files:/app/files`. ++ +[source,bash] +---- +$ docker run \ + -p 8090:8090 \ + -it \ + -v /app/files:/app/files \ + --name extraction-service \ + docker.elastic.co/integrations/data-extraction-service:$EXTRACTION_SERVICE_VERSION +---- ++ +[NOTE] +==== +Due to how this feature works in the codebase for non-dockerized setups, **the local filepath and the docker container's filepath must be identical**. +For example, if using `/app/files`, you must mount the directory as `-v /app/files:/app/files`. +If either directory is different, the self-managed connector will be unable to provide an accurate file pointer for the extraction service. This is not a factor when using a dockerized self-managed connector. +==== ++ +2. Next, before running the self-managed connector, be sure to update the config file with the correct information. ++ +[source,yaml] +---- +# data-extraction-service settings +extraction_service: + host: http://localhost:8090 + use_file_pointers: true + shared_volume_dir: '/app/files' +---- ++ +3. Then all that's left is to start the self-managed connector and run a sync. +If you encounter any unexpected errors, refer to <>. + +[discrete#es-connectors-content-extraction-data-extraction-service-file-pointers-configuration-dockerized] +====== Configuration for dockerized self-managed connectors + +When using self-hosted extraction from a dockerized self-managed connector, there are a few extra steps required on top of {connectors-python}/docs/DOCKER.md[running the self-managed connector in docker^]. + +* The self-hosted extraction service will need to share the same network that the self-managed connector and Elasticsearch are sharing. +* The self-managed connector and the extraction service will also need to share a volume. You can decide what directory inside these docker containers the volume will be mounted onto, but the directory must be the same for both docker containers. + +[discrete#es-connectors-content-extraction-data-extraction-service-file-pointers-configuration-dockerized-example] +*Example* + +1. First, set up a volume for the two docker containers to share. +This will be where files are downloaded into and then extracted from. ++ +[source,bash] +---- +$ docker volume create --name extraction-service-volume +---- ++ +2. If you haven't set up a network yet, you can create it now. ++ +[source,bash] +---- +$ docker network create elastic +---- ++ +3. Include the docker volume name and the network as arguments when running the extraction service. +For this example, we will be using `/app/files` as our container directory. ++ +[source,bash] +---- +$ docker run \ + -p 8090:8090 \ + -it \ + -v extraction-service-volume:/app/files \ + --network "elastic" \ + --name extraction-service \ + docker.elastic.co/integrations/data-extraction-service:$EXTRACTION_SERVICE_VERSION +---- ++ +4. Next, you can follow the instructions for {connectors-python}/docs/DOCKER.md[running the self-managed connector in docker^] until step `4. Update the configuration file for your self-managed connector`. +When setting up your configuration, be sure to add the following settings for the self-hosted content extraction service. +Note that the `host` will now refer to an internal docker endpoint instead of localhost. ++ +[source,yaml] +---- +# data-extraction-service settings +extraction_service: + host: http://host.docker.internal:8090 + use_file_pointers: true + shared_volume_dir: '/app/files' +---- ++ +5. Next, during step `5. Run the Docker image`, we only need to add our new shared volume in the run command using `-v extraction-service-volume:/app/files`. ++ +[source,bash] +---- +$ docker run \ + -v ~/connectors-config:/config \ + -v extraction-service-volume:/app/files \ + --network "elastic" \ + --tty \ + --rm \ + docker.elastic.co/enterprise-search/elastic-connectors:$CONNECTOR_CLIENT_VERSION \ + /app/bin/elastic-ingest \ + -c /config/config.yml +---- ++ +6. Now the self-managed connector and extraction service docker containers should be set up to share files. +Run a test sync to make sure everything is configured correctly. +If you encounter any unexpected errors, refer to <>. + +[discrete#es-connectors-content-extraction-local-logs] +===== Self-hosted extraction service logs + +The extraction service produces two different log files that may be informative when troubleshooting. +These are saved at the following file locations internally in the docker container: + +* `/var/log/openresty.log` for request traffic logs +* `/var/log/tika.log` for tikaserver jar logs + +Logs can be viewed from outside of docker by combining `docker exec` with the `tail` command. + +[source,bash] +---- +$ docker exec extraction-service /bin/sh -c "tail /var/log/openresty.log" +$ docker exec extraction-service /bin/sh -c "tail /var/log/tika.log" +---- + +[discrete#es-connectors-content-extraction-troubleshooting] +===== Troubleshooting the self-hosted extraction service + +The following warning logs may appear while using self-hosted extraction service. +Each log in this section is followed by a description of what may have happened, and suggested fixes. + +[source,bash] +---- +Extraction service is not configured, skipping its preflight check. +---- + +The configuration file is missing the `extraction_service.host` field. +If you want to use this service, check that the configuration is formatted correctly and that the required field is present. + +[source,bash] +---- +Data extraction service found at , but health-check returned . +---- + +The `/ping` endpoint returned a non-`200` response. +This could mean that the extraction service is unhealthy and may need to be restarted, or that the configured `extraction_service.host` is incorrect. +You can find more information about what happened in the <>. + +[source,bash] +---- +Expected to find a running instance of data extraction service at but failed. . +---- + +The health check returned either a timeout or client connection error. + +* A timeout may be caused by the extraction service server not running, or not being accessible from the configured `host` in the configuration file. +* A server connection error is an internal error on the extraction service. You will need to investigate the <>. + +[source,bash] +---- +Extraction service has been initialised but no extraction service configuration was found. No text will be extracted for this sync. +---- + +You have enabled self-hosted extraction service for the connector, but the configuration file is missing the `extraction_service.host` field. +Check that the configuration is formatted correctly and that the required field is present. + +[source,bash] +---- +Extraction service could not parse . Status: ; : . +---- + +This warning will appear every time a file is not extractable. +Generally the `` will provide an explanation for why extraction failed. +Contact support if the message is unclear. +When a file fails extraction, it will be indexed with an empty string in the `body` field. diff --git a/docs/reference/connector/docs/connectors-content-syncs.asciidoc b/docs/reference/connector/docs/connectors-content-syncs.asciidoc new file mode 100644 index 0000000000000..0a2eb54047170 --- /dev/null +++ b/docs/reference/connector/docs/connectors-content-syncs.asciidoc @@ -0,0 +1,64 @@ +[#es-connectors-sync-types] +== Content syncs + +Elastic connectors have two types of content syncs: + +* <> +* <> + +[discrete#es-connectors-sync-types-full] +=== Full syncs + +[NOTE] +==== +We recommend running a full sync whenever <> are modified +==== + +A full sync syncs all documents in the third-party data source into {es}. + +It also deletes any documents in {es}, which no longer exist in the third-party data source. + +A full sync, by definition, takes longer than an incremental sync but it ensures full data consistency. + +A full sync is available for all connectors. + +You can <> or <> a full sync job. + +[discrete#es-connectors-sync-types-incremental] +=== Incremental syncs + +An incremental sync only syncs data changes since the last full or incremental sync. + +Incremental syncs are only available after an initial full sync has successfully completed. +Otherwise the incremental sync will fail. + +You can <> or <> an incremental sync job. + +[discrete#es-connectors-sync-types-incremental-performance] +==== Incremental sync performance + +During an incremental sync your connector will still _fetch_ all data from the third-party data source. +If data contains timestamps, the connector framework compares document ids and timestamps. +If a document already exists in {es} with the same timestamp, then this document does not need updating and will not be sent to {es}. + +The determining factor in incremental sync performance is the raw volume of data ingested. +For small volumes of data, the performance improvement using incremental syncs will be negligible. +For large volumes of data, the performance impact can be huge. +Additionally, an incremental sync is less likely to be throttled by {es}, making it more performant than a full sync when {es} is under heavy load. + +A third-party data source that has throttling and low throughput, but stores very little data in Elasticsearch, such as GitHub, Jira, or Confluence, won't see a significant performance improvement from incremental syncs. + +However, a fast, accessible third-party data source that stores huge amounts of data in {es}, such as Azure Blob Storage, Google Drive, or S3, can lead to a significant performance improvement from incremental syncs. + +[NOTE] +==== +Incremental syncs for <> and <> connectors use specific logic. +All other connectors use the same shared connector framework logic for incremental syncs. +==== + +[discrete#es-connectors-sync-types-incremental-supported] +==== Incremental sync availability + +Incremental syncs are available for the following connectors: + +include::_connectors-list-incremental.asciidoc[] diff --git a/docs/reference/connector/docs/connectors-docker-compose-quickstart.asciidoc b/docs/reference/connector/docs/connectors-docker-compose-quickstart.asciidoc new file mode 100644 index 0000000000000..52a8921c90ec1 --- /dev/null +++ b/docs/reference/connector/docs/connectors-docker-compose-quickstart.asciidoc @@ -0,0 +1,10 @@ +[#es-connectors-docker-compose-quickstart] +=== Docker Compose quickstart + +Use our Docker Compose quickstart to easily set up a full self-managed stack and try out Elastic Connectors. + +We've provided a script to start up and run Elasticsearch, Kibana, and Connectors instances using Docker Compose. +The script prompts you to configure your Connectors before starting. +Additionally, you can use the same set of scripts to manually configure your Connectors and run the stack. + +Refer to the https://github.com/elastic/connectors/tree/main/scripts/stack#readme[README^] in the https://github.com/elastic/connectors[elastic/connectors^] source repository for more information. diff --git a/docs/reference/connector/docs/connectors-dropbox.asciidoc b/docs/reference/connector/docs/connectors-dropbox.asciidoc new file mode 100644 index 0000000000000..295b7e2936625 --- /dev/null +++ b/docs/reference/connector/docs/connectors-dropbox.asciidoc @@ -0,0 +1,580 @@ +[#es-connectors-dropbox] +=== Elastic Dropbox connector reference +++++ +Dropbox +++++ +// Attributes used in this file +:service-name: Dropbox +:service-name-stub: dropbox + +The _Elastic Dropbox connector_ is a <> for https://www.dropbox.com[Dropbox^]. +This connector is written in Python using the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/{service-name-stub}.py[*source code* for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + +.Choose your connector reference +******************************* +Are you using a managed connector on Elastic Cloud or a self-managed connector? Expand the documentation based on your deployment method. +******************************* + +// //////// //// //// //// //// //// //// //////// +// //////// NATIVE CONNECTOR REFERENCE /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-dropbox-native-connector-reference] +==== *Elastic managed connector reference* + +.View *Elastic managed connector* reference + +[%collapsible] +=============== + +[discrete#es-connectors-dropbox-availability-and-prerequisites] +===== Availability and prerequisites + +This connector is available as a *managed connector* in Elastic versions *8.10.0 and later*. + +To use this connector natively in Elastic Cloud, satisfy all <>. + +[discrete#es-connectors-dropbox-create-native-connector] +===== Create a {service-name} connector +include::_connectors-create-native.asciidoc[] + +[discrete#es-connectors-dropbox-usage] +===== Usage + +To use this connector as a *managed connector*, see <>. + +For additional operations, see <>. + +Before you can configure your connector, you'll need to: + +* <> +* <> + +[discrete#es-connectors-dropbox-dropbox-api-authorization] +===== Dropbox API Authorization + +[discrete#es-connectors-dropbox-create-dropbox-oauth-app] +====== Create Dropbox OAuth App + +You'll need to create an OAuth app in the Dropbox platform by following these steps: + +1. Register a new app in the https://www.dropbox.com/developers/apps[Dropbox App Console^]. +Select *Full Dropbox API app* and choose the following required permissions: +* `files.content.read` +* `sharing.read` ++ +To use document level security, you'll also need the following permissions: +* `team_info.read` +* `team_data.member` +* `team_data.content.read` +* `members.read` +2. Once the app is created, make note of the *app key* and *app secret* values which you'll need to configure the Dropbox connector on your Elastic deployment. + +[discrete#es-connectors-dropbox-refresh-token] +====== Generate a refresh Token + +To generate a refresh token, follow these steps: + +1. Go to the following URL, replacing `` with the *app key* value saved earlier: +`https://www.dropbox.com/oauth2/authorize?client_id=&response_type=code&token_access_type=offline` ++ +The HTTP response should contain an *authorization code* that you'll use to generate a refresh token. +An authorization code *can only be used once* to create a refresh token. ++ +2. In your terminal, run the following `cURL` command, replacing ``, `:` with the values you saved earlier: ++ +[source,shell] +---- +curl -X POST "https://api.dropboxapi.com/oauth2/token?code=&grant_type=authorization_code" -u ":" +---- +// NOTCONSOLE +Store the refresh token from the response to be used in the connector configuration. ++ +Make sure the response has a list of the following scopes: ++ +* `account_info.read` +* `files.content.read` +* `files.metadata.read` +* `sharing.read` +* `team_info.read` (if using document level security) +* `team_data.member` (if using document level security) +* `team_data.content.read` (if using document level security) +* `members.read` (if using document level security) + +[discrete#es-connectors-dropbox-configuration] +===== Configuration + +The following configuration fields are required to set up the connector: + +Path to fetch files/folders:: +The folder path to fetch files/folders from Dropbox. Default value is `/`. ++ +[Note] +==== +This field can be bypassed by advanced sync rules. +==== + +App key:: +The App Key to authenticate your Dropbox application. + +App secret:: +The App Secret to authenticate your Dropbox application. + +Refresh token:: +The refresh token to authenticate your Dropbox application. + +Enable document level security:: +Toggle to enable <>. +When enabled, full syncs will fetch access control lists for each document and store them in the `_allow_access_control` field. +Access control syncs will fetch users' access control lists and store them in a separate index. + +Include groups and inherited users:: +Appears when document level security is enabled. +Include groups and inherited users when indexing permissions. + +[WARNING] +==== +Enabling `Include groups and inherited users` will cause a signficant performance degradation. +==== + +[discrete#es-connectors-dropbox-documents-and-syncs] +===== Documents and syncs + +The connector syncs the following objects and entities: + +* *Files* +** Includes metadata such as file name, path, size, content, etc. +* *Folders* + +[NOTE] +==== +Due to a Dropbox issue, metadata updates to Paper files from Dropbox Paper are not immediately reflected in the Dropbox UI. +This delays the availability of updated results for the connector. +Once the metadata changes are visible in the Dropbox UI, the updates are available. +==== + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted. (Self-managed connectors can use the <> to handle larger binary files.) +* Currently, the connector doesn't retrieve files from shared Team folders. +* Permissions are not synced by default. If <> is not enabled *all documents* indexed to an Elastic deployment will be visible to *all users with access* to that Elastic Deployment. +==== + +[discrete#es-connectors-dropbox-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-dropbox-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +[discrete#es-connectors-dropbox-sync-rules-advanced] +====== Advanced sync rules + +[NOTE] +==== +A <> is required for advanced sync rules to take effect. +==== + +The following section describes <> for this connector. + +Advanced sync rules for Dropbox allow you to sync Dropbox files based on a query that matches strings in the filename. +You can optionally filter the results of the query by `file_extensions` or `file_categories`. +When both are provided, priority is given to `file_categories`. +We have some examples below for illustration. + +[discrete#es-connectors-dropbox-sync-rules-advanced-example-1] +*Example: Query only* + +[source,js] +---- +[ + { + "query": "confidential" + }, + { + "query": "dropbox" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-dropbox-sync-rules-advanced-example-2] +*Example: Query with file extension filter* + +[source,js] +---- +[ + { + "query": "dropbox", + "options": { + "file_extensions": [ + "txt", + "pdf" + ] + } + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-dropbox-sync-rules-advanced-example-3] +*Example: Query with file category filter* + +[source,js] +---- +[ + { + "query": "test", + "options": { + "file_categories": [ + { + ".tag": "paper" + }, + { + ".tag": "png" + } + ] + } + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-dropbox-sync-rules-advanced-limitations] +*Limitations* + +* Content extraction is not supported for Dropbox *Paper* files when advanced sync rules are enabled. + +[discrete#es-connectors-dropbox-known-issues] +===== Known issues + +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-dropbox-troubleshooting] +===== Troubleshooting + +See <> for a list of troubleshooting tips for all connectors. + +[discrete#es-connectors-dropbox-security] +===== Security + +See <> for a list of security tips for all connectors. + +[discrete#es-connectors-dropbox-content-extraction] +===== Content extraction + +See <>. +// Closing the collapsible section +=============== + + +// //////// //// //// //// //// //// //// //////// +// //////// CONNECTOR CLIENT REFERENCE /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-dropbox-connector-client-reference] +==== *Self-managed connector* + +.View *self-managed connector* reference + +[%collapsible] +=============== + +[discrete#es-connectors-dropbox-client-availability-and-prerequisites] +===== Availability and prerequisites + +This connector is available as a self-managed *self-managed connector*. + +This self-managed connector is compatible with Elastic versions *8.9.0*+. + +To use this connector, satisfy all <>. + +[discrete#es-connectors-dropbox-create-connector-client] +===== Create a {service-name} connector +include::_connectors-create-client.asciidoc[] + +[discrete#es-connectors-dropbox-client-usage] +===== Usage +Before you can configure your connector, you'll need to: + +* <> +* <> + + +To use this connector as a *self-managed connector*, see <> +Once set up, for additional usage operations, see <>. + +[discrete#es-connectors-dropbox-client-dropbox-api-authorization] +===== Dropbox API Authorization + +[discrete#es-connectors-dropbox-client-create-dropbox-oauth-app] +====== Create Dropbox OAuth App + +You'll need to create an OAuth app in the Dropbox platform by following these steps: + +1. Register a new app in the https://www.dropbox.com/developers/apps[Dropbox App Console^]. +Select *Full Dropbox API app* and choose the following required permissions: +* `files.content.read` +* `sharing.read` ++ +To use document level security, you'll also need the following permissions: +* `team_info.read` +* `team_data.member` +* `team_data.content.read` +* `members.read` +2. Once the app is created, make note of the *app key* and *app secret* values which you'll need to configure the Dropbox connector on your Elastic deployment. + +[discrete#es-connectors-dropbox-client-refresh-token] +====== Generate a refresh Token + +To generate a refresh token, follow these steps: + +1. Go to the following URL, replacing `` with the *app key* value saved earlier: +`https://www.dropbox.com/oauth2/authorize?client_id=&response_type=code&token_access_type=offline` ++ +The HTTP response should contain an *authorization code* that you'll use to generate a refresh token. +An authorization code *can only be used once* to create a refresh token. ++ +2. In your terminal, run the following `cURL` command, replacing ``, `:` with the values you saved earlier: ++ +[source,shell] +---- +curl -X POST "https://api.dropboxapi.com/oauth2/token?code=&grant_type=authorization_code" -u ":" +---- +// NOTCONSOLE +Store the refresh token from the response to be used in the connector configuration. ++ +Make sure the response has a list of the following scopes: ++ +* `account_info.read` +* `files.content.read` +* `files.metadata.read` +* `sharing.read` +* `team_info.read` (if using document level security) +* `team_data.member` (if using document level security) +* `team_data.content.read` (if using document level security) +* `members.read` (if using document level security) + +[discrete#es-connectors-dropbox-client-configuration] +===== Configuration + +[TIP] +==== +When using the <> workflow, initially these fields will use the default configuration set in the {connectors-python}/connectors/sources/confluence.py[connector source code^]. +These are set in the `get_default_configuration` function definition. + +These configurable fields will be rendered with their respective *labels* in the Kibana UI. +Once connected, you'll be able to update these values in Kibana. +==== + +The following configuration fields are required to set up the connector: + +`path`:: +The folder path to fetch files/folders from Dropbox. Default value is `/`. + +`app_key` (required):: +The App Key to authenticate your Dropbox application. + +`app_secret` (required):: +The App Secret to authenticate your Dropbox application. + +`refresh_token` (required):: +The refresh token to authenticate your Dropbox application. + +use_document_level_security:: +Toggle to enable <>. +When enabled, full syncs will fetch access control lists for each document and store them in the `_allow_access_control` field. +Access control syncs will fetch users' access control lists and store them in a separate index. + +`retry_count`:: +The number of retry attempts after a failed request to Dropbox. Default value is `3`. + +`concurrent_downloads`:: +The number of concurrent downloads for fetching attachment content. +This can help speed up content extraction of attachments. Defaults to `100`. + +`use_text_extraction_service`:: +Requires a separate deployment of the <>. +Requires that pipeline settings disable text extraction. +Default value is `False`. + +`use_document_level_security`:: +Toggle to enable <>. +When enabled, full syncs will fetch access control lists for each document and store them in the `_allow_access_control` field. Access control syncs will fetch users' access control lists and store them in a separate index. + +`include_inherited_users_and_groups`:: +Depends on document level security being enabled. +Include groups and inherited users when indexing permissions. + +[WARNING] +==== +Enabling `Include groups and inherited users` will cause a signficant performance degradation. +==== + +[discrete#es-connectors-dropbox-client-docker] +===== Deployment using Docker + +include::_connectors-docker-instructions.asciidoc[] + +[discrete#es-connectors-dropbox-client-documents-and-syncs] +===== Documents and syncs + +The connector syncs the following objects and entities: + +* *Files* +** Includes metadata such as file name, path, size, content, etc. +* *Folders* + +[NOTE] +==== +Due to a Dropbox issue, metadata updates to Paper files from Dropbox Paper are not immediately reflected in the Dropbox UI. +This delays the availability of updated results for the connector. +Once the metadata changes are visible in the Dropbox UI, the updates are available. +==== + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted by default. You can use the <> to handle larger binary files. +* Currently, the connector doesn't retrieve files from shared Team folders. +* Permissions are not synced by default. If <> is not enabled *all documents* indexed to an Elastic deployment will be visible to *all users with access* to that Elastic Deployment. +==== + +[discrete#es-connectors-dropbox-client-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-dropbox-client-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +[discrete#es-connectors-dropbox-client-sync-rules-advanced] +====== Advanced sync rules + +[NOTE] +==== +A <> is required for advanced sync rules to take effect. +==== + +The following section describes <> for this connector. + +Advanced sync rules for Dropbox allow you to sync Dropbox files based on a query that matches strings in the filename. +You can optionally filter the results of the query by `file_extensions` or `file_categories`. +When both are provided, priority is given to `file_categories`. +We have some examples below for illustration. + +[discrete#es-connectors-dropbox-client-sync-rules-advanced-example-1] +*Example: Query only* + +[source,js] +---- +[ + { + "query": "confidential" + }, + { + "query": "dropbox" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-dropbox-client-sync-rules-advanced-example-2] +*Example: Query with file extension filter* + +[source,js] +---- +[ + { + "query": "dropbox", + "options": { + "file_extensions": [ + "txt", + "pdf" + ] + } + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-dropbox-client-sync-rules-advanced-example-3] +*Example: Query with file category filter* + +[source,js] +---- +[ + { + "query": "test", + "options": { + "file_categories": [ + { + ".tag": "paper" + }, + { + ".tag": "png" + } + ] + } + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-dropbox-client-sync-rules-advanced-limitations] +*Limitations* + +* Content extraction is not supported for Dropbox *Paper* files when advanced sync rules are enabled. + +[discrete#es-connectors-dropbox-client-end-to-end-testing] +===== End-to-end Testing + +The connector framework enables operators to run functional tests against a real data source. +Refer to <> for more details. + +To perform E2E testing for the Dropbox connector, run the following command: + +[source,shell] +---- +$ make ftest NAME=dropbox +---- + +For faster tests, add the `DATA_SIZE=small` flag: + +[source,shell] +---- +make ftest NAME=dropbox DATA_SIZE=small +---- + +[discrete#es-connectors-dropbox-client-known-issues] +===== Known issues + +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-dropbox-client-troubleshooting] +===== Troubleshooting + +See <> for a list of troubleshooting tips for all connectors. + +[discrete#es-connectors-dropbox-client-security] +===== Security + +See <> for a list of security tips for all connectors. + +[discrete#es-connectors-dropbox-client-content-extraction] +===== Content extraction + +See <>. + +// Closing the collapsible section +=============== diff --git a/docs/reference/connector/docs/connectors-filter-extract-transform.asciidoc b/docs/reference/connector/docs/connectors-filter-extract-transform.asciidoc new file mode 100644 index 0000000000000..278478c908bf0 --- /dev/null +++ b/docs/reference/connector/docs/connectors-filter-extract-transform.asciidoc @@ -0,0 +1,53 @@ +[#es-connectors-filter-extract-transform] +== Extract, filter, and transform content +++++ +Extract and transform +++++ + +Elastic connectors offer a number of tools for extracting, filtering, and transforming content from your third-party data sources. +Each connector has its own default logic, specific to the data source, and every Elastic Search deployment uses a default ingest pipeline to extract and transform data. +Several tools are also available for more advanced use cases. + +The following diagram provides an overview of how content extraction, sync rules, and ingest pipelines can be orchestrated in your connector's data pipeline. + +[.screenshot] +image::images/pipelines-extraction-sync-rules.png[Architecture diagram of data pipeline with content extraction, sync rules, and ingest pipelines] + +By default, only the connector specific logic (2) and the default `ent-search-generic-ingestion` pipeline (6) extract and transform your data, as configured in your deployment. + +The following tools are available for more advanced use cases: + +* *Advanced sync rules* (1). Remote filtering at the data source level, before data reaches the connector. +* *Basic sync rules* (4) or *extraction service* (3). Integration filtering controlled by the connector. +* *Ingest pipelines* (6). Customized pipeline filtering where {es} filters data _before_ indexing. + +Learn more in the following documentation links. + +[discrete#es-connectors-filter-extract-transform-content-extraction] +=== Content extraction + +Connectors have a default content extraction service, plus the <> for advanced use cases. + +Refer to <> for details. + +[discrete#es-connectors-filter-extract-transform-sync-rules] +=== Sync rules + +Use sync rules to help control which documents are synced between the third-party data source and Elasticsearch. +Sync rules enable you to filter data early in the data pipeline, which is more efficient and secure. + +* *Basic* sync rules are identical for all connectors. +* *Advanced sync rules* are data source-specific. +They cover complex query-and-filter scenarios, defined in a DSL JSON snippet. + +Refer to <> for details. + +[discrete#es-connectors-filter-extract-transform-ingest-pipelines] +=== Ingest pipelines + +Ingest pipelines are a user-defined sequence of processors that modify documents before they are indexed into Elasticsearch. +Use ingest pipelines for data enrichment, normalization, and more. + +Elastic connectors use a default ingest pipeline, which you can copy and customize to meet your needs. + +Refer to {ref}/ingest-pipeline-search.html[ingest pipelines in Search] in the {es} documentation. \ No newline at end of file diff --git a/docs/reference/connector/docs/connectors-framework.asciidoc b/docs/reference/connector/docs/connectors-framework.asciidoc new file mode 100644 index 0000000000000..b0a037d9ef468 --- /dev/null +++ b/docs/reference/connector/docs/connectors-framework.asciidoc @@ -0,0 +1,27 @@ +[#es-connectors-framework] +== Elastic connector framework: build and customize connectors +++++ +Build and customize connectors +++++ + +The Elastic connector framework enables developers to build Elastic-supported self-managed connectors which sync third-party data sources to Elasticsearch. +The framework implements common functionalities out of the box, so developers can focus on the logic specific to integrating their chosen data source. + +The framework ensures compatibility, makes it easier for our team to review PRs, and help out in the development process. +When you build using our framework, we provide a pathway for the connector to be officially supported by Elastic. + +[discrete#es-connectors-framework-use-cases] +=== Use cases + +The framework serves two distinct, but related use cases: + +* Customizing an existing Elastic <> +* Building a new self-managed connector + +[discrete#es-connectors-framework-learn-more] +=== Learn more + +To learn how to contribute connectors using the framework, refer to our https://github.com/elastic/connectors/blob/main/docs/CONTRIBUTING.md[contributing guide] in the `connectors` repository. +This guide explains how to get started and includes a contribution checklist and pull request guidelines. + +This repo contains all the source code for existing Elastic connectors. \ No newline at end of file diff --git a/docs/reference/connector/docs/connectors-github.asciidoc b/docs/reference/connector/docs/connectors-github.asciidoc new file mode 100644 index 0000000000000..df577d83e8121 --- /dev/null +++ b/docs/reference/connector/docs/connectors-github.asciidoc @@ -0,0 +1,697 @@ +[#es-connectors-github] +=== Elastic GitHub connector reference +++++ +GitHub +++++ +// Attributes used in this file +:service-name: GitHub +:service-name-stub: github + +The _Elastic GitHub connector_ is a <> for https://www.github.com[GitHub^]. +This connector is written in Python using the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/{service-name-stub}.py[*source code* for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + +.Choose your connector reference +******************************* +Are you using a managed connector on Elastic Cloud or a self-managed connector? Expand the documentation based on your deployment method. +******************************* + +// //////// //// //// //// //// //// //// //////// +// //////// NATIVE CONNECTOR REFERENCE /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-github-native-connector-reference] +==== *Elastic managed connector reference* + +.View *Elastic managed connector* reference + +[%collapsible] +=============== + +[discrete#es-connectors-github-availability-prerequisites] +===== Availability and prerequisites + +This connector is available as a *managed connector* as of Elastic version *8.11.0*. + +To use this connector natively in Elastic Cloud, satisfy all <>. + +[discrete#es-connectors-github-create-native-connector] +===== Create a {service-name} connector +include::_connectors-create-native.asciidoc[] + +[discrete#es-connectors-github-usage] +===== Usage + +To use this connector as a *managed connector*, see <>. + +For additional operations, see <>. + +[discrete#es-connectors-github-personal-access-token] +====== GitHub personal access token + +Configure a GitHub personal access token to fetch data from GitHub. + +Follow these steps to generate a GitHub personal access token: + +* Go to *GitHub Settings → Developer settings → Personal access tokens → Tokens(classic)*. +* Select `Generate new token`. +* Add a note and select the following scopes: +** `repo` +** `user` +** `read:org` +* Select `Generate token` and copy the token. + +[discrete#es-connectors-github-github-app] +====== GitHub App + +Configure a GitHub App to fetch data from GitHub. + +Follow these steps to create a GitHub App: + +* Go to *GitHub Settings → Developer settings → GitHub Apps*. +* Select `New GitHub App`. +* Add a name and Homepage URL, deselect `Active` under `Webhook`. +* Under `Permissions`, select `Read-only` for `Commit statuses`, `Contents`, `Issues`, `Metadata` and `Pull requests` under `Repository permissions`, select `Read-only` for `Members` under `Organization permissions`. +* Select `Any account` for `Where can this GitHub App be installed?`. +* Click `Create GitHub App`. +* Scroll down to the section `Private keys`, and click `Generate a private key`. +* Click `Install App` in the upper-left corner, select the organizations/personal accounts you want to install the GitHub App on, click `Install`. +* You can choose to install it on all repositories or selected repositories, and click `Install`. + +[discrete#es-connectors-github-compatability] +===== Compatibility + +Both GitHub and GitHub Enterprise are supported. + +[discrete#es-connectors-github-configuration] +===== Configuration + +The following configuration fields are required: + +Data source:: +Toggle between GitHub Cloud or GitHub Server. + +Server URL:: +URL of the GitHub Server instance. (GitHub Server only) + +Authentication method:: +The method to authenticate the GitHub instance. Toggle between `Personal access token` and `GitHub App`. + +Token:: +GitHub personal access token to authenticate the GitHub instance. This field is only available for `Personal access token` authentication method. + +Repository Type:: +Toggle between `Organization` and `Other`. +Note that document level security (DLS) is only available for `Organization` repositories. + +Organization Name:: +Name of the organization to fetch data from. This field is only available when `Authentication method` is set to `Personal access token` and `Repository Type` is set to `Organization`. + +App ID:: +App ID of the GitHub App. This field is only available when `Authentication method` is set to `GitHub App`. + +App private key:: +Private key generated for the GitHub App. This field is only available when `Authentication method` is set to `GitHub App`. + +List of repositories:: +Comma-separated list of repositories to fetch data from GitHub instance. If the value is `*` the connector will fetch data from all repositories present in the configured user's account. ++ +Default value is `*`. ++ +Examples: ++ +* `elasticsearch`,`elastic/kibana` +* `*` +[TIP] +==== +*Repository ownership* + +If the "OWNER/" portion of the "OWNER/REPO" repository argument is omitted, it defaults to the name of the authenticating user. + +In the examples provided here: + +* the `elasticsearch` repo synced will be the `/elasticsearch` repo +* the `kibana` repo synced will be the Elastic owned repo + +The "OWNER/" portion of the "OWNER/REPO" repository argument must be provided when `GitHub App` is selected as the `Authentication method`. +==== +[NOTE] +==== +This field can be bypassed by advanced sync rules. +==== + +Enable SSL:: +Enable SSL for the GitHub instance. + +SSL certificate:: +SSL certificate for the GitHub instance. Example: ++ +[source, txt] +---- +-----BEGIN CERTIFICATE----- +MIID+jCCAuKgAwIBAgIGAJJMzlxLMA0GCSqGSIb3DQEBCwUAMHoxCzAJBgNVBAYT +... +7RhLQyWn2u00L7/9Omw= +-----END CERTIFICATE----- +---- + +Enable document level security:: +Toggle to enable <>. +When enabled, full syncs will fetch access control lists for each document and store them in the `_allow_access_control` field. +DLS is only available when `Repository Type` is set to `Organization`. + +[discrete#es-connectors-github-documents-syncs] +===== Documents and syncs + +The connector syncs the following objects and entities: + +* **Repositories** +* **Pull Requests** +* **Issues** +* **Files & Folder** + +Only the following file extensions are ingested: + +* `.markdown` +* `.md` +* `.rst` + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted. (Self-managed connectors can use the <> to handle larger binary files.) +* Permissions are not synced. **All documents** indexed to an Elastic deployment will be visible to **all users with access** to that Elasticsearch Index. +==== + +[discrete#es-connectors-github-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-github-sync-rules] +===== Sync rules + +_Basic_ sync rules are identical for all connectors and are available by default. +For more information read <>. + +[discrete#es-connectors-github-sync-rules-advanced] +====== Advanced sync rules + +[NOTE] +==== +A <> is required for advanced sync rules to take effect. +==== + +The following section describes *advanced sync rules* for this connector. +Advanced sync rules are defined through a source-specific DSL JSON snippet. + +The following sections provide examples of advanced sync rules for this connector. + +[discrete#es-connectors-github-sync-rules-advanced-branch] +*Indexing document and files based on branch name configured via branch key* + +[source,js] +---- +[ + { + "repository": "repo_name", + "filter": { + "branch": "sync-rules-feature" + } + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-github-sync-rules-advanced-issue-key] +*Indexing document based on issue query related to bugs via issue key* + +[source,js] +---- +[ + { + "repository": "repo_name", + "filter": { + "issue": "is:bug" + } + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-github-sync-rules-advanced-pr-key] +*Indexing document based on PR query related to open PR's via PR key* + +[source,js] +---- +[ + { + "repository": "repo_name", + "filter": { + "pr": "is:open" + } + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-github-sync-rules-advanced-issue-query-branch-name] +*Indexing document and files based on queries and branch name* + +[source,js] +---- +[ + { + "repository": "repo_name", + "filter": { + "issue": "is:bug", + "pr": "is:open", + "branch": "sync-rules-feature" + } + } +] +---- +// NOTCONSOLE + +[NOTE] +==== +All documents pulled by a given rule are indexed regardless of whether the document has already been indexed by a previous rule. +This can lead to document duplication, but the indexed documents count will differ in the logs. +Check the Elasticsearch index for the actual document count. +==== + +[discrete#es-connectors-github-sync-rules-advanced-overlapping] +*Advanced rules for overlapping* + +[source,js] +---- +[ + { + "filter": { + "pr": "is:pr is:merged label:auto-backport merged:>=2023-07-20" + }, + "repository": "repo_name" + }, + { + "filter": { + "pr": "is:pr is:merged label:auto-backport merged:>=2023-07-15" + }, + "repository": "repo_name" + } +] +---- +// NOTCONSOLE + +[NOTE] +==== +If `GitHub App` is selected as the authentication method, the "OWNER/" portion of the "OWNER/REPO" repository argument must be provided. +==== + +[discrete#es-connectors-github-content-extraction] +===== Content Extraction + +See <>. + +[discrete#es-connectors-github-known-issues] +===== Known issues + +There are currently no known issues for this connector. +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-github-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-github-security] +===== Security + +See <>. + +// Closing the collapsible section +=============== + + +// //////// //// //// //// //// //// //// //////// +// //////// CONNECTOR CLIENT REFERENCE /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-github-connector-client-reference] +==== *Self-managed connector* + +.View *self-managed connector* reference + +[%collapsible] +=============== + +[discrete#es-connectors-github-client-availability-prerequisites] +===== Availability and prerequisites + +This connector is available as a self-managed *self-managed connector*. + +This self-managed connector is compatible with Elastic versions *8.10.0+*. + +To use this connector, satisfy all <>. + +[discrete#es-connectors-github-create-connector-client] +===== Create a {service-name} connector +include::_connectors-create-client.asciidoc[] + +[discrete#es-connectors-github-client-usage] +===== Usage + +To use this connector as a *self-managed connector*, see <> +For additional usage operations, see <>. + +[discrete#es-connectors-github-client-personal-access-token] +====== GitHub personal access token + +Configure a GitHub personal access token to fetch data from GitHub. + +Follow these steps to generate a GitHub access token: + +* Go to *GitHub Settings → Developer settings → Personal access tokens → Tokens(classic)*. +* Select `Generate new token`. +* Add a note and select the following scopes: +** `repo` +** `user` +** `read:org` +* Select `Generate token` and copy the token. + +[discrete#es-connectors-github-client-github-app] +====== GitHub App + +Configure a GitHub App to fetch data from GitHub. + +Follow these steps to create a GitHub App: + +* Go to *GitHub Settings → Developer settings → GitHub Apps*. +* Select `New GitHub App`. +* Add a name and Homepage URL, deselect `Active` under `Webhook`. +* Under `Permissions`, select `Read-only` for `Commit statuses`, `Contents`, `Issues`, `Metadata` and `Pull requests` under `Repository permissions`, select `Read-only` for `Members` under `Organization permissions`. +* Select `Any account` for `Where can this GitHub App be installed?`. +* Click `Create GitHub App`. +* Scroll down to the section `Private keys`, and click `Generate a private key`. +* Click `Install App` in the upper-left corner, select the organizations/personal accounts you want to install the GitHub App on, click `Install`. +* You can choose to install it on all repositories or selected repositories, and click `Install`. + + +[discrete#es-connectors-github-client-compatability] +===== Compatibility + +Both GitHub and GitHub Enterprise are supported. + +[discrete#es-connectors-github-client-configuration] +===== Configuration + +[TIP] +==== +When using the <> workflow, initially these fields will use the default configuration set in the {connectors-python}/connectors/sources/github.py[connector source code^]. +These are set in the `get_default_configuration` function definition. + +These configurable fields will be rendered with their respective *labels* in the Kibana UI. +Once connected, you'll be able to update these values in Kibana. +==== + +The following configuration fields are required: + +`data_source`:: +GitHub Cloud or GitHub Server. + +`host`:: +URL of the GitHub Server instance. (GitHub Server only) + +`auth_method`:: +The method to authenticate the GitHub instance. Toggle between `Personal access token` and `GitHub App`. + +`token`:: +GitHub personal access token to authenticate the GitHub instance. This field is only available for `Personal access token` authentication method. + +`repo_type`:: +Toggle between `Organization` and `Other`. +Note that document level security (DLS) is only available for `Organization` repositories. + +`org_name`:: +Name of the organization to fetch data from. This field is only available when `Authentication method` is set to `Personal access token` and `Repository Type` is set to `Organization`. + +`app_id`:: +App ID of the GitHub App. This field is only available when `Authentication method` is set to `GitHub App`. + +`private_key`:: +Private key generated for the GitHub App. This field is only available when `Authentication method` is set to `GitHub App`. + +`repositories`:: +Comma-separated list of repositories to fetch data from GitHub instance. If the value is `*` the connector will fetch data from all repositories present in the configured user's account. ++ +Default value is `*`. ++ +Examples: ++ +* `elasticsearch`,`elastic/kibana` +* `*` +[TIP] +==== +*Repository ownership* + +If the "OWNER/" portion of the "OWNER/REPO" repository argument is omitted, it defaults to the name of the authenticating user. + +In the examples provided here: + +* the `elasticsearch` repo synced will be the `/elasticsearch` +* the `kibana` repo synced will be the Elastic owned repo + +The "OWNER/" portion of the "OWNER/REPO" repository argument must be provided when `GitHub App` is selected as the `Authentication method`. +==== +[NOTE] +==== +This field can be bypassed by advanced sync rules. +==== + +`ssl_enabled`:: +Whether SSL verification will be enabled. Default value is `False`. + +`ssl_ca`:: +Content of SSL certificate. Note: If `ssl_enabled` is `False`, the value in this field is ignored. Example certificate: ++ +[source, txt] +---- +-----BEGIN CERTIFICATE----- +MIID+jCCAuKgAwIBAgIGAJJMzlxLMA0GCSqGSIb3DQEBCwUAMHoxCzAJBgNVBAYT +... +7RhLQyWn2u00L7/9Omw= +-----END CERTIFICATE----- +---- + +`use_document_level_security`:: +Toggle to enable <>. +When enabled, full syncs will fetch access control lists for each document and store them in the `_allow_access_control` field. +DLS is only available when `Repository Type` is set to `Organization`. + +`retry_count`:: +The number of retry attempts after failed request to GitHub. Default value is `3`. + +`use_text_extraction_service`:: +Requires a separate deployment of the <>. Requires that pipeline settings disable text extraction. +Default value is `False`. + +[discrete#es-connectors-github-client-docker] +===== Deployment using Docker + +include::_connectors-docker-instructions.asciidoc[] + +[discrete#es-connectors-github-client-documents-syncs] +===== Documents and syncs + +The connector syncs the following objects and entities: + +* **Repositories** +* **Pull Requests** +* **Issues** +* **Files & Folder** + +Only the following file extensions are ingested: + +* `.markdown` +* `.md` +* `.rst` + +[NOTE] +==== +* Content of files bigger than 10 MB won't be extracted. +* Permissions are not synced. **All documents** indexed to an Elastic deployment will be visible to **all users with access** to that Elasticsearch Index. +==== + +[discrete#es-connectors-github-client-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-github-client-sync-rules] +===== Sync rules + +_Basic_ sync rules are identical for all connectors and are available by default. +For more information read <>. + +[discrete#es-connectors-github-client-sync-rules-advanced] +====== Advanced sync rules + +[NOTE] +==== +A <> is required for advanced sync rules to take effect. +==== + +The following section describes *advanced sync rules* for this connector. +Advanced sync rules are defined through a source-specific DSL JSON snippet. + +The following sections provide examples of advanced sync rules for this connector. + +[discrete#es-connectors-github-client-sync-rules-advanced-branch] +*Indexing document and files based on branch name configured via branch key* + +[source,js] +---- +[ + { + "repository": "repo_name", + "filter": { + "branch": "sync-rules-feature" + } + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-github-client-sync-rules-advanced-issue-key] +*Indexing document based on issue query related to bugs via issue key* + +[source,js] +---- +[ + { + "repository": "repo_name", + "filter": { + "issue": "is:bug" + } + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-github-client-sync-rules-advanced-pr-key] +*Indexing document based on PR query related to open PR's via PR key* + +[source,js] +---- +[ + { + "repository": "repo_name", + "filter": { + "pr": "is:open" + } + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-github-client-sync-rules-advanced-issue-query-branch-name] +*Indexing document and files based on queries and branch name* + +[source,js] +---- +[ + { + "repository": "repo_name", + "filter": { + "issue": "is:bug", + "pr": "is:open", + "branch": "sync-rules-feature" + } + } +] +---- +// NOTCONSOLE + +[NOTE] +==== +All documents pulled by a given rule are indexed regardless of whether the document has already been indexed by a previous rule. +This can lead to document duplication, but the indexed documents count will differ in the logs. +Check the Elasticsearch index for the actual document count. +==== + +[discrete#es-connectors-github-client-sync-rules-advanced-overlapping] +*Advanced rules for overlapping* + +[source,js] +---- +[ + { + "filter": { + "pr": "is:pr is:merged label:auto-backport merged:>=2023-07-20" + }, + "repository": "repo_name" + }, + { + "filter": { + "pr": "is:pr is:merged label:auto-backport merged:>=2023-07-15" + }, + "repository": "repo_name" + } +] +---- +// NOTCONSOLE + +[NOTE] +==== +If `GitHub App` is selected as the authentication method, the "OWNER/" portion of the "OWNER/REPO" repository argument must be provided. +==== + +[discrete#es-connectors-github-client-content-extraction] +===== Content Extraction + +See <>. + +[discrete#es-connectors-github-client-connector-client-operations] +===== Self-managed connector operations + +[discrete#es-connectors-github-client-testing] +===== End-to-end testing + +The connector framework enables operators to run functional tests against a real data source. +Refer to <> for more details. + +To perform E2E testing for the GitHub connector, run the following command: + +[source,shell] +---- +$ make ftest NAME=github +---- + +For faster tests, add the `DATA_SIZE=small` flag: + +[source,shell] +---- +make ftest NAME=github DATA_SIZE=small +---- + +[discrete#es-connectors-github-client-known-issues] +===== Known issues + +There are currently no known issues for this connector. +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-github-client-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-github-client-security] +===== Security + +See <>. + +// Closing the collapsible section +=============== diff --git a/docs/reference/connector/docs/connectors-gmail.asciidoc b/docs/reference/connector/docs/connectors-gmail.asciidoc new file mode 100644 index 0000000000000..594df7b9e681a --- /dev/null +++ b/docs/reference/connector/docs/connectors-gmail.asciidoc @@ -0,0 +1,366 @@ +[#es-connectors-gmail] +=== Elastic Gmail connector reference +++++ +Gmail +++++ +// Attributes used in this file +:service-name: Gmail +:service-name-stub: gmail + +The _Elastic GMail connector_ is a <> for GMail. + + +// //////// //// //// //// //// //// //// //////// +// //////// NATIVE CONNECTOR REFERENCE (MANAGED SERVICE) /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-gmail-native-connector-reference] +==== *Elastic managed connector reference* + +.View *Elastic managed connector* reference +[%collapsible] +=============== + +[discrete#es-connectors-gmail-availability] +===== Availability and prerequisites + +This connector is available as a *managed connector* (managed service) in Elastic Cloud. + +This connector is compatible with Elastic versions *8.13.0+*. + +To use this connector, satisfy all <>. + +[discrete#es-connectors-gmail-create-native-connector] +==== Create a {service-name} connector +include::_connectors-create-native.asciidoc[] + +[discrete#es-connectors-gmail-usage] +===== Usage + +To use this connector as a managed connector in Elastic Cloud, use the *Connectors* workflow in the Kibana UI. + +To create a new {service-name} connector: + +. Navigate to *Search -> Connectors* page in the Kibana UI. +. Select the *New Native Connector* button. +. Select the *{service-name}* connector. + +For additional operations, see <>. + +[discrete#es-connectors-gmail-connector-authentication-prerequisites] +===== Connector authentication prerequisites + +Before syncing any data from GMail, you need to create a https://cloud.google.com/iam/docs/service-account-overview[service account^] with appropriate access to the GMail and the Google Directory API, which is part of the Google Admin SDK API. +You also need to enable domain-wide delegation to impersonate the users you're fetching messages from. + +To get started, log into https://cloud.google.com[Google Cloud Platform^] and go to the `Console`. + +. *Create a Google Cloud Project.* Give your project a name, change the project ID and click the Create button. +. *Enable Google APIs.* Choose APIs & Services from the left menu and click on `Enable APIs and Services`. You need to enable *GMail API* and the *Google Admin SDK API*. +. *Create a Service Account.* In the `APIs & Services` section, click on `Credentials` and click on `Create credentials` to create a service account. Give your service account a name and a service account ID. This is like an email address and will be used to identify your service account in the future. Click `Done` to finish creating the service account. ++ +Your service account needs to have access to at least the following scope: ++ +* `https://www.googleapis.com/auth/gmail.readonly` +. *Create a Key File*. + * In the Cloud Console, go to `IAM and Admin` > `Service accounts` page. + * Click the email address of the service account that you want to create a key for. + * Click the `Keys` tab. Click the `Add key` drop-down menu, then select `Create new key`. + * Select JSON as the Key type and then click `Create`. This will download a JSON file that will contain the service account credentials. + +. *Google Workspace domain-wide delegation of authority*. ++ +To access user data like messages on a Google Workspace domain, the service account that you created needs to be granted access by a super administrator for the domain. You can follow https://developers.google.com/cloud-search/docs/guides/delegation[the official documentation^] to perform Google Workspace domain-wide delegation of authority. ++ +You need to grant the following *OAuth Scopes* to your service account: ++ +-- +* `https://www.googleapis.com/auth/admin.directory.user.readonly` +-- ++ +This step allows the connector to access user data and their group memberships in your Google Workspace organization. + +[discrete#es-connectors-gmail-configuration] +===== Configuration + +The following configuration fields are required: + +GMail service account JSON:: +The service account credentials generated from Google Cloud Platform (JSON string). +Refer to the https://developers.google.com/workspace/guides/create-credentials#create_credentials_for_a_service_account[Google Cloud documentation^] for more information. + +Google Workspace admin email:: +Google Workspace admin email. +Required to enable document level security (DLS). +A service account with delegated authority can impersonate an admin user with permissions to access Google Workspace user data and their group memberships. +Refer to the https://support.google.com/a/answer/162106?hl=en[Google Cloud documentation^] for more information. + +Google customer ID:: +Google customer id. +Required to fetch messages and to enable document level security (DLS). +Go to `Google Workspace Admin Console` -> `Account` and copy the value under `Customer Id`. + +Include spam and trash emails:: +Toggle to fetch spam and trash emails. +Also works with document level security (DLS). + +Enable document level security:: +Toggle to enable <>. +DLS is supported for the GMail connector. +When enabled: ++ +* Full syncs will fetch access control lists for each document and store them in the `_allow_access_control` field. +* Access control syncs will fetch users' access control lists and store them in a separate index. + +[discrete#es-connectors-gmail-documents-and-syncs] +===== Documents and syncs + +The connector will fetch all messages of all users the service account has access to. + +[discrete#es-connectors-gmail-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-gmail-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +Advanced sync rules are available for this connector. +The connector supports the https://support.google.com/mail/answer/7190[GMail advanced search syntax] under the `messages` field. + +For example: + +[source,js] +---- +{ + "messages": [ + "before:2021/10/10", + "from:amy" + ] +} +---- +// NOTCONSOLE + +[discrete#es-connectors-gmail-document-level-security] +===== Document level security + +Document level security (DLS) enables you to restrict access to documents based on a user’s permissions. +Refer to <> on this page for how to enable DLS for this connector. + +[NOTE] +==== +Refer to <> to learn how to ingest data from a connector with DLS enabled, when building a search application. +The example uses SharePoint Online as the data source, but the same steps apply to every connector. +==== + +[discrete#es-connectors-gmail-known-issues] +===== Known issues + +There are currently no known issues for this connector. + +[discrete#es-connectors-gmail-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-gmail-security] +===== Security + +See <>. + +[discrete#es-connectors-gmail-framework-and-source] +===== Framework and source + +This connector is built in Python with the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/gmail.py[source code for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + + +// Closing the collapsible section +=============== + + +// //////// //// //// //// //// //// //// //////// +// //////// CONNECTOR CLIENT REFERENCE (SELF-MANAGED) /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-gmail-connector-client-reference] +==== *Self-managed connector reference* + +.View *self-managed connector* reference +[%collapsible] +=============== + +[discrete#es-connectors-gmail-client-availability] +===== Availability and prerequisites + +This connector is available as a self-managed *self-managed connector* from the *Elastic connector framework*. + +This self-managed connector is compatible with Elastic versions *8.10.0+*. + +To use this connector, satisfy all <>. + +[discrete#es-connectors-gmail-create-connector-client] +===== Create a {service-name} connector +include::_connectors-create-client.asciidoc[] + +[discrete#es-connectors-gmail-client-usage] +===== Usage + +To use this connector as a **self-managed connector**, use the *Connector* workflow in the Kibana UI. + +For additional operations, see <>. + +[discrete#es-connectors-gmail-client-connector-authentication-prerequisites] +===== Connector authentication prerequisites + +Before syncing any data from GMail, you need to create a https://cloud.google.com/iam/docs/service-account-overview[service account^] with appropriate access to the GMail and the Google Directory API, which is part of the Google Admin SDK API. +You also need to enable domain-wide delegation to impersonate the users you're fetching messages from. + +To get started, log into https://cloud.google.com[Google Cloud Platform^] and go to the `Console`. + +. *Create a Google Cloud Project.* Give your project a name, change the project ID and click the Create button. +. *Enable Google APIs.* Choose APIs & Services from the left menu and click on `Enable APIs and Services`. You need to enable *GMail API* and the *Google Admin SDK API*. +. *Create a Service Account.* In the `APIs & Services` section, click on `Credentials` and click on `Create credentials` to create a service account. Give your service account a name and a service account ID. This is like an email address and will be used to identify your service account in the future. Click `Done` to finish creating the service account. ++ +Your service account needs to have access to at least the following scope: ++ +* `https://www.googleapis.com/auth/gmail.readonly` +. *Create a Key File*. + * In the Cloud Console, go to `IAM and Admin` > `Service accounts` page. + * Click the email address of the service account that you want to create a key for. + * Click the `Keys` tab. Click the `Add key` drop-down menu, then select `Create new key`. + * Select JSON as the Key type and then click `Create`. This will download a JSON file that will contain the service account credentials. + +. *Google Workspace domain-wide delegation of authority*. ++ +To access user data like messages on a Google Workspace domain, the service account that you created needs to be granted access by a super administrator for the domain. You can follow https://developers.google.com/cloud-search/docs/guides/delegation[the official documentation^] to perform Google Workspace domain-wide delegation of authority. ++ +You need to grant the following *OAuth Scopes* to your service account: ++ +-- +* `https://www.googleapis.com/auth/admin.directory.user.readonly` +-- ++ +This step allows the connector to access user data and their group memberships in your Google Workspace organization. + +[discrete#es-connectors-gmail-client-configuration] +===== Configuration + +[TIP] +==== +When using the <> workflow, initially these fields will use the default configuration set in the {connectors-python}/connectors/sources/gmail.py[connector source code^]. +These are set in the `get_default_configuration` function definition. + +These configurable fields will be rendered with their respective *labels* in the Kibana UI. +Once connected, you'll be able to update these values in Kibana. +==== + +The following configuration fields are required: + +`GMail service account JSON`:: +The service account credentials generated from Google Cloud Platform (JSON string). +Refer to the https://developers.google.com/workspace/guides/create-credentials#create_credentials_for_a_service_account[Google Cloud documentation^] for more information. + +`Google Workspace admin email`:: +Google Workspace admin email. +Required to enable document level security (DLS). +A service account with delegated authority can impersonate an admin user with permissions to access Google Workspace user data and their group memberships. +Refer to the https://support.google.com/a/answer/162106?hl=en[Google Cloud documentation^] for more information. + +`Google customer id`:: +Google customer id. +Required to fetch messages and to enable document level security (DLS). +Go to `Google Workspace Admin Console` -> `Account` and copy the value under `Customer Id`. + +`Include spam and trash emails`:: +Toggle to fetch spam and trash emails. +Also works with DLS. + +`Enable document level security`:: +Toggle to enable <>. +DLS is supported for the GMail connector. +When enabled: ++ +* Full syncs will fetch access control lists for each document and store them in the `_allow_access_control` field. +* Access control syncs will fetch users' access control lists and store them in a separate index. + +[discrete#es-connectors-gmail-client-deployment-using-docker] +===== Deployment using Docker + +include::_connectors-docker-instructions.asciidoc[] + +[discrete#es-connectors-gmail-client-documents-and-syncs] +===== Documents and syncs + +The connector will fetch all messages of all users the service account has access to. + +[discrete#es-connectors-gmail-client-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-gmail-client-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +Advanced sync rules are available for this connector. +The connector supports the https://support.google.com/mail/answer/7190[GMail advanced search syntax] under the `messages` field. + +For example: + +[source,js] +---- +{ + "messages": [ + "before:2021/10/10", + "from:amy" + ] +} +---- +// NOTCONSOLE + +[discrete#es-connectors-gmail-client-document-level-security] +===== Document level security + +Document level security (DLS) enables you to restrict access to documents based on a user’s permissions. +Refer to <> on this page for how to enable DLS for this connector. + +[NOTE] +==== +Refer to <> to learn how to ingest data from a connector with DLS enabled, when building a search application. +The example uses SharePoint Online as the data source, but the same steps apply to every connector. +==== + +[discrete#es-connectors-gmail-client-known-issues] +===== Known issues + +There are currently no known issues for this connector. + +[discrete#es-connectors-gmail-client-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-gmail-client-security] +===== Security + +See <>. + +[discrete#es-connectors-gmail-client-framework-and-source] +===== Framework and source + +This connector is built in Python with the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/gmail.py[source code for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + + +// Closing the collapsible section +=============== diff --git a/docs/reference/connector/docs/connectors-google-cloud.asciidoc b/docs/reference/connector/docs/connectors-google-cloud.asciidoc new file mode 100644 index 0000000000000..64fcb82b19ab7 --- /dev/null +++ b/docs/reference/connector/docs/connectors-google-cloud.asciidoc @@ -0,0 +1,266 @@ +[#es-connectors-google-cloud] +=== Google Cloud Storage Connector +++++ +Google Cloud Storage +++++ + +// Attributes used in this file +:service-name: Google Cloud Storage +:service-name-stub: google_cloud_storage + +The _Elastic Google Cloud Storage connector_ is a <> for https://cloud.google.com/storage[Google Cloud Storage^] data sources. + + +// //////// //// //// //// //// //// //// //////// +// //////// NATIVE CONNECTOR REFERENCE (MANAGED SERVICE) /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-google-cloud-native-connector-reference] +==== *Elastic managed connector reference* + +.View *Elastic managed connector* reference +[%collapsible] +=============== + +[discrete#es-connectors-google-cloud-availability-prerequisites] +===== Availability and prerequisites + +This connector is available natively in Elastic Cloud since *8.12.0.* +To use this connector in Elastic Cloud, satisfy all <>. + +[discrete#es-connectors-google-cloud-usage] +===== Usage + +The Google Cloud Storage service account must have (at least) the following scopes and roles: + +* `resourcemanager.projects.get` +* `serviceusage.services.use` +* `storage.buckets.list` +* `storage.objects.list` +* `storage.objects.get` + +Google Cloud Storage service account credentials are stored in a JSON file. + +[discrete#es-connectors-google-cloud-configuration] +===== Configuration + +The following configuration field is required to set up the connector: + +Buckets:: +List of buckets to index. +`*` will index all buckets. + +Google Cloud service account JSON:: +The service account credentials generated from Google Cloud Storage (JSON string). +Refer to the https://developers.google.com/workspace/guides/create-credentials#create_credentials_for_a_service_account[Google Cloud documentation^] for more information. + +[discrete#es-connectors-google-cloud-documents-syncs] +===== Documents and syncs + +The connector will fetch all buckets and paths the service account has access to. + +The `Owner` field is not fetched as `read_only` scope doesn’t allow the connector to fetch IAM information. + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted. (Self-managed connectors can use the <> to handle larger binary files.) +* Permission are not synced. All documents indexed to an Elastic deployment will be visible to all users with access to that Elastic Deployment. +==== + +[discrete#es-connectors-google-cloud-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-google-cloud-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +Advanced sync rules are not available for this connector in the present version. +Currently filtering is controlled by ingest pipelines. + +[discrete#es-connectors-google-cloud-content-extraction] +===== Content extraction + +See <>. + +[source,shell] +---- +$ make ftest NAME=google_cloud_storage +---- + +For faster tests, add the `DATA_SIZE=small` flag: + +[source,shell] +---- +make ftest NAME=google_cloud_storage DATA_SIZE=small +---- + +[discrete#es-connectors-google-cloud-known-issues] +===== Known issues + +There are currently no known issues for this connector. + +[discrete#es-connectors-google-cloud-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-google-cloud-security] +===== Security + +See <>. + +[discrete#es-connectors-google-cloud-source] +===== Framework and source + +This connector is built with the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/google_cloud_storage.py[source code for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + +// Closing the collapsible section +=============== + + +// //////// //// //// //// //// //// //// //////// +// //////// CONNECTOR CLIENT REFERENCE (SELF-MANAGED) /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-google-cloud-connector-client-reference] +==== *Self-managed connector reference* + +.View *self-managed connector* reference +[%collapsible] +=============== + +[discrete#es-connectors-google-cloud-client-availability-prerequisites] +===== Availability and prerequisites + +This connector is available as a self-managed *self-managed connector*. +This self-managed connector is compatible with Elastic versions *8.6.0+*. +To use this connector, satisfy all <>. + +[discrete#es-connectors-google-cloud-client-usage] +===== Usage + +The Google Cloud Storage service account must have (at least) the following scopes and roles: + +* `resourcemanager.projects.get` +* `serviceusage.services.use` +* `storage.buckets.list` +* `storage.objects.list` +* `storage.objects.get` + +Google Cloud Storage service account credentials are stored in a JSON file. + +[discrete#es-connectors-google-cloud-client-configuration] +===== Configuration + +[TIP] +==== +When using the <> workflow, initially these fields will use the default configuration set in the {connectors-python}/connectors/sources/google_cloud_storage.py[connector source code^]. +These are set in the `get_default_configuration` function definition. + +These configurable fields will be rendered with their respective *labels* in the Kibana UI. +Once connected, you'll be able to update these values in Kibana. +==== + +The following configuration fields are required to set up the connector: + +`buckets`:: +List of buckets to index. +`*` will index all buckets. + +`service_account_credentials`:: +The service account credentials generated from Google Cloud Storage (JSON string). +Refer to the https://developers.google.com/workspace/guides/create-credentials#create_credentials_for_a_service_account[Google Cloud documentation^] for more information. + +`retry_count`:: +The number of retry attempts after a failed call to Google Cloud Storage. +Default value is `3`. + +[discrete#es-connectors-google-cloud-client-docker] +===== Deployment using Docker + +include::_connectors-docker-instructions.asciidoc[] + +[discrete#es-connectors-google-cloud-client-documents-syncs] +===== Documents and syncs + +The connector will fetch all buckets and paths the service account has access to. + +The `Owner` field is not fetched as `read_only` scope doesn’t allow the connector to fetch IAM information. + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted by default. You can use the <> to handle larger binary files. +* Permission are not synced. All documents indexed to an Elastic deployment will be visible to all users with access to that Elastic Deployment. +==== + +[discrete#es-connectors-google-cloud-client-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-google-cloud-client-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +Advanced sync rules are not available for this connector in the present version. +Currently filtering is controlled by ingest pipelines. + +[discrete#es-connectors-google-cloud-client-content-extraction] +===== Content extraction + +See <>. + +[discrete#es-connectors-google-cloud-client-client-operations-testing] +===== End-to-end testing + +The connector framework enables operators to run functional tests against a real data source. +Refer to <> for more details. + +To perform E2E testing for the Google Cloud Storage connector, run the following command: + +[source,shell] +---- +$ make ftest NAME=google_cloud_storage +---- + +For faster tests, add the `DATA_SIZE=small` flag: + +[source,shell] +---- +make ftest NAME=google_cloud_storage DATA_SIZE=small +---- + +[discrete#es-connectors-google-cloud-client-known-issues] +===== Known issues + +There are currently no known issues for this connector. + +[discrete#es-connectors-google-cloud-client-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-google-cloud-client-security] +===== Security + +See <>. + +[discrete#es-connectors-google-cloud-client-source] +===== Framework and source + +This connector is built with the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/google_cloud_storage.py[source code for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + +// Closing the collapsible section +=============== diff --git a/docs/reference/connector/docs/connectors-google-drive.asciidoc b/docs/reference/connector/docs/connectors-google-drive.asciidoc new file mode 100644 index 0000000000000..d3c4a0886efc3 --- /dev/null +++ b/docs/reference/connector/docs/connectors-google-drive.asciidoc @@ -0,0 +1,409 @@ +[#es-connectors-google-drive] +=== Elastic Google Drive connector reference +++++ +Google Drive +++++ +// Attributes used in this file +:service-name: Google Drive +:service-name-stub: google_drive + +The _Elastic Google Drive connector_ is a <> for https://www.google.com/drive[Google Drive^]. +This connector is written in Python using the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/{service-name-stub}.py[*source code* for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + +.Choose your connector reference +******************************* +Are you using a managed connector on Elastic Cloud or a self-managed connector? Expand the documentation based on your deployment method. +******************************* + +// //////// //// //// //// //// //// //// //////// +// //////// NATIVE CONNECTOR REFERENCE /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-google-drive-native-connector-reference] +==== *Elastic managed connector reference* + +.View *Elastic managed connector* reference + +[%collapsible] +=============== + +[discrete#es-connectors-google-drive-availability-and-prerequisites] +===== Availability and prerequisites + +This connector is available as a *managed connector* as of Elastic version *8.11.0*. + +To use this connector natively in Elastic Cloud, satisfy all <>. + +[discrete#es-connectors-google-drive-usage] +===== Usage + +To use this connector natively in Elastic Cloud, see <>. + +For additional operations, see <>. + +[discrete#es-connectors-google-drive-connector-authentication-prerequisites] +===== Connector authentication prerequisites + +Before syncing any data from Google Drive, you need to create a https://cloud.google.com/iam/docs/service-account-overview[service account^] with appropriate access to Google Drive API. + +To get started, log into https://cloud.google.com[Google Cloud Platform^] and go to the `Console`. + +. *Create a Google Cloud Project.* Give your project a name, change the project ID and click the Create button. + +. *Enable Google APIs.* Choose APIs & Services from the left menu and click on `Enable APIs and Services`. You need to enable the *Drive API*. + +. *Create a Service Account.* In the `APIs & Services` section, click on `Credentials` and click on `Create credentials` to create a service account. Give your service account a name and a service account ID. This is like an email address and will be used to identify your service account in the future. Click `Done` to finish creating the service account. Your service account needs to have access to at least the following scope: +- `https://www.googleapis.com/auth/drive.readonly` + +. *Create a Key File*. + - In the Cloud Console, go to `IAM and Admin` > `Service accounts` page. + - Click the email address of the service account that you want to create a key for. + - Click the `Keys` tab. Click the `Add key` drop-down menu, then select `Create new key`. + - Select JSON as the Key type and then click `Create`. This will download a JSON file that will contain the service account credentials. + +. *[Optional] Share Google Drive Folders.* If you use domain-wide delegation for syncing data you can skip this step. Go to your Google Drive. Right-click the folder or shared drive, choose `Share` and add the email address of the service account you created in step 3. as a viewer to this folder. + +[NOTE] +==== +When you grant a service account access to a specific folder or shared drive in Google Drive, it's important to note that the permissions extend to all the children within that folder or drive. +This means that any folders or files contained within the granted folder or drive inherit the same access privileges as the parent. +==== + +[discrete#es-connectors-google-drive-additional-prerequisites-for-domain-wide-delegation] +====== Additional authentication prerequisites for domain-wide delegation + +This step is *required* when *Use domain-wide delegation for data sync* or *Enable document level security* configuration option is enabled. + +. *Enable Google APIs*. ++ +Choose APIs & Services from the left menu and click on `Enable APIs and Services`. You need to enable the *Admin SDK API* and *Drive API*. + +. *Google Workspace domain-wide delegation of authority*. ++ +To access drive and user data in a Google Workspace domain, the service account that you created needs to be granted access by a super administrator for the domain. You can follow https://developers.google.com/cloud-search/docs/guides/delegation[the official documentation^] to perform Google Workspace domain-wide delegation of authority. ++ +You need to grant the following *OAuth Scopes* to your service account: ++ +-- +* `https://www.googleapis.com/auth/admin.directory.group.readonly` +* `https://www.googleapis.com/auth/admin.directory.user.readonly` +* `https://www.googleapis.com/auth/drive.readonly` +* `https://www.googleapis.com/auth/drive.metadata.readonly` +-- ++ +This step allows the connector to: + +* access user data and their group memberships in a Google Workspace organization +* access Google Drive data in drives associated to Google Workspace members + +[discrete#es-connectors-google-drive-configuration] +===== Configuration + +The following configuration fields are required: + +Google Drive service account JSON:: +The service account credentials generated from Google Cloud Platform (JSON string). +Refer to the https://developers.google.com/workspace/guides/create-credentials#create_credentials_for_a_service_account[Google Cloud documentation^] for more information. + +Enable document level security:: +Toggle to enable <>. +DLS is supported for the Google Drive connector. +When enabled: ++ +* Full syncs will fetch access control lists for each document and store them in the `_allow_access_control` field. +* Access control syncs will fetch users' access control lists and store them in a separate index. + +Google Workspace admin email:: +Google Workspace admin email. +Required to enable document level security (DLS) or domain-wide delegation for data sync. +A service account with delegated authority can impersonate an admin user with permissions to access Google Workspace user data and their group memberships. +Refer to the https://support.google.com/a/answer/162106?hl=en[Google Cloud documentation^] for more information. + +[discrete#es-connectors-google-drive-documents-and-syncs] +===== Documents and syncs + +The connector will fetch all files and folders the service account has access to. + +It will attempt to extract the content from Google Suite documents (Google Docs, Google Sheets and Google Slides) and regular files. + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted. (Self-managed connectors can use the <> to handle larger binary files.) +* Permissions are not synced by default. +You must first enable <>. +Otherwise, *all documents* indexed to an Elastic deployment will be visible to *all users with access* to that Elastic Deployment. +==== + +[discrete#es-connectors-google-drive-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-google-drive-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +Advanced sync rules are not available for this connector in the present version. +Currently filtering is controlled via ingest pipelines. + +[discrete#es-connectors-google-drive-document-level-security] +===== Document level security + +Document level security (DLS) enables you to restrict access to documents based on a user’s permissions. +Refer to <> on this page for how to enable DLS for this connector. + +[NOTE] +==== +Refer to <> to learn how to ingest data from a connector with DLS enabled, when building a search application. +The example uses SharePoint Online as the data source, but the same steps apply to every connector. +==== + +[discrete#es-connectors-google-drive-content-extraction] +===== Content extraction + +See <> for more information. + +[discrete#es-connectors-google-drive-known-issues] +===== Known issues + +There are currently no known issues for this connector. + +[discrete#es-connectors-google-drive-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-google-drive-security] +===== Security + +See <>. + +// Closing the collapsible section +=============== + + +// //////// //// //// //// //// //// //// //////// +// //////// CONNECTOR CLIENT REFERENCE /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-google-drive-connector-client-reference] +==== *Self-managed connector* + +.View *self-managed connector* reference + +[%collapsible] +=============== + +[discrete#es-connectors-google-drive-client-availability-and-prerequisites] +===== Availability and prerequisites + +This connector is available as a self-managed *self-managed connector*. To use this connector, satisfy all <>. + +[discrete#es-connectors-google-drive-client-usage] +===== Usage + +To use this connector as a *self-managed connector*, see <> +For additional usage operations, see <>. + +[discrete#es-connectors-google-drive-client-connector-authentication-prerequisites] +===== Connector authentication prerequisites + +Before syncing any data from Google Drive, you need to create a https://cloud.google.com/iam/docs/service-account-overview[service account^] with appropriate access to Google Drive API. + +To get started, log into https://cloud.google.com[Google Cloud Platform^] and go to the `Console`. + +. *Create a Google Cloud Project.* Give your project a name, change the project ID and click the Create button. + +. *Enable Google APIs.* Choose APIs & Services from the left menu and click on `Enable APIs and Services`. You need to enable the *Drive API*. + +. *Create a Service Account.* In the `APIs & Services` section, click on `Credentials` and click on `Create credentials` to create a service account. Give your service account a name and a service account ID. This is like an email address and will be used to identify your service account in the future. Click `Done` to finish creating the service account. Your service account needs to have access to at least the following scope: +- `https://www.googleapis.com/auth/drive.readonly` + +. *Create a Key File*. + - In the Cloud Console, go to `IAM and Admin` > `Service accounts` page. + - Click the email address of the service account that you want to create a key for. + - Click the `Keys` tab. Click the `Add key` drop-down menu, then select `Create new key`. + - Select JSON as the Key type and then click `Create`. This will download a JSON file that will contain the service account credentials. + +. *[Optional] Share Google Drive Folders.* If you use domain-wide delegation for syncing data you can skip this step. Go to your Google Drive. Right-click the folder or shared drive, choose `Share` and add the email address of the service account you created in step 3. as a viewer to this folder. + +[NOTE] +==== +When you grant a service account access to a specific folder or shared drive in Google Drive, it's important to note that the permissions extend to all the children within that folder or drive. +This means that any folders or files contained within the granted folder or drive inherit the same access privileges as the parent. +==== + +[discrete#es-connectors-google-drive-client-additional-prerequisites-for-domain-wide-delegation] +====== Additional authentication prerequisites for domain-wide delegation + +This step is *required* when *Use domain-wide delegation for data sync* or *Enable document level security* configuration option is enabled. + +. *Enable Google APIs*. ++ +Choose APIs & Services from the left menu and click on `Enable APIs and Services`. You need to enable the *Admin SDK API* and *Drive API*. + +. *Google Workspace domain-wide delegation of authority*. ++ +To access drive and user data in a Google Workspace domain, the service account that you created needs to be granted access by a super administrator for the domain. You can follow https://developers.google.com/cloud-search/docs/guides/delegation[the official documentation^] to perform Google Workspace domain-wide delegation of authority. ++ +You need to grant the following *OAuth Scopes* to your service account: ++ +-- +* `https://www.googleapis.com/auth/admin.directory.group.readonly` +* `https://www.googleapis.com/auth/admin.directory.user.readonly` +* `https://www.googleapis.com/auth/drive.readonly` +* `https://www.googleapis.com/auth/drive.metadata.readonly` +-- ++ +This step allows the connector to: + +* access user data and their group memberships in a Google Workspace organization +* access Google Drive data in drives associated to Google Workspace members + +[discrete#es-connectors-google-drive-client-configuration] +===== Configuration + +[TIP] +==== +When using the <> workflow, initially these fields will use the default configuration set in the {connectors-python}/connectors/sources/google_drive.py[connector source code^]. +These are set in the `get_default_configuration` function definition. + +These configurable fields will be rendered with their respective *labels* in the Kibana UI. +Once connected, you'll be able to update these values in Kibana. +==== + +The following configuration fields are required: + +`service_account_credentials`:: +The service account credentials generated from Google Cloud Platform (JSON string). +Refer to the https://developers.google.com/workspace/guides/create-credentials#create_credentials_for_a_service_account[Google Cloud documentation^] for more information. + +`use_domain_wide_delegation_for_sync`:: +Use https://developers.google.com/cloud-search/docs/guides/delegation[domain-wide delegation] to automatically sync content from all shared and personal drives in the Google workspace. +This eliminates the need to manually share Google Drive data with your service account, though it may increase the sync time. +If disabled, only items and folders manually shared with the service account will be synced. + +`google_workspace_admin_email_for_data_sync`:: +Required when domain-wide delegation for data sync is enabled. +This email is used for discovery and syncing of shared drives. Only the shared drives this user has access to are synced. + +`google_workspace_email_for_shared_drives_sync`:: +Required when domain-wide delegation for data sync is enabled. +Provide the Google Workspace user email for discovery and syncing of shared drives. Only the shared drives this user has access to will be synced. + +`use_document_level_security`:: +Toggle to enable <>. +DLS is supported for the Google Drive connector. +When enabled: ++ +* Full syncs will fetch access control lists for each document and store them in the `_allow_access_control` field. +* Access control syncs will fetch users' access control lists and store them in a separate index. + +`google_workspace_admin_email`:: +Google Workspace admin email. +Required to enable document level security (DLS) or domain-wide delegation for data sync. +A service account with delegated authority can impersonate an admin user with permissions to access Google Workspace user data and their group memberships. +Refer to the https://support.google.com/a/answer/162106?hl=en[Google Cloud documentation^] for more information. + +`max_concurrency`:: +The maximum number of concurrent HTTP requests to the Google Drive API. +Increasing this value can improve data retrieval speed, but it may also place higher demands on system resources and network bandwidth. + +`use_text_extraction_service`:: +Requires a separate deployment of the <>. +Requires that pipeline settings disable text extraction. +Default value is `False`. + +[discrete#es-connectors-google-drive-client-deployment-using-docker] +===== Deployment using Docker + +include::_connectors-docker-instructions.asciidoc[] + +[discrete#es-connectors-google-drive-client-documents-and-syncs] +===== Documents and syncs + +The connector will fetch all files and folders the service account has access to. + +It will attempt to extract the content from Google Suite documents (Google Docs, Google Sheets and Google Slides) and regular files. + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted +* Permissions are not synced by default. +You must first enable <>. +Otherwise, *all documents* indexed to an Elastic deployment will be visible to *all users with access* to that Elastic Deployment. +==== + +[discrete#es-connectors-google-drive-client-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-google-drive-client-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +Advanced sync rules are not available for this connector in the present version. +Currently filtering is controlled via ingest pipelines. + +[discrete#es-connectors-google-drive-client-document-level-security] +===== Document level security + +Document level security (DLS) enables you to restrict access to documents based on a user’s permissions. +Refer to <> on this page for how to enable DLS for this connector. + +[NOTE] +==== +Refer to <> to learn how to ingest data from a connector with DLS enabled, when building a search application. +The example uses SharePoint Online as the data source, but the same steps apply to every connector. +==== + +[discrete#es-connectors-google-drive-client-content-extraction] +===== Content extraction + +See <> for more information. + +[discrete#es-connectors-google-drive-client-end-to-end-testing] +===== End-to-end testing + +The connector framework enables operators to run functional tests against a real data source. Refer to <> for more details. + +To perform E2E testing for the Google Drive connector, run the following command: + +[source,shell] +---- +make ftest NAME=google_drive +---- + +For faster tests, add the `DATA_SIZE=small` flag: + +[source,shell] +---- +make ftest NAME=google_drive DATA_SIZE=small +---- + +[discrete#es-connectors-google-drive-client-known-issues] +===== Known issues + +There are currently no known issues for this connector. + +[discrete#es-connectors-google-drive-client-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-google-drive-client-security] +===== Security + +See <>. + +// Closing the collapsible section +=============== diff --git a/docs/reference/connector/docs/connectors-graphql.asciidoc b/docs/reference/connector/docs/connectors-graphql.asciidoc new file mode 100644 index 0000000000000..bc7083d482e79 --- /dev/null +++ b/docs/reference/connector/docs/connectors-graphql.asciidoc @@ -0,0 +1,248 @@ +[#es-connectors-graphql] +=== Elastic GraphQL connector reference +++++ +GraphQL +++++ + +// Attributes used in this file +:service-name: GraphQL +:service-name-stub: graphql + +The Elastic GraphQL connector is written in Python using the https://github.com/elastic/connectors/tree/main[Elastic connector framework]. View the https://github.com/elastic/connectors/blob/main/connectors/sources/graphql.py[source code for this connector]. + +[discrete#es-connectors-graphql-connector-availability-and-prerequisites] +==== Availability and prerequisites + +This connector was introduced in Elastic *8.14.0*, available as a *self-managed* self-managed connector. + +To use this connector, satisfy all <>. +Importantly, you must deploy the connectors service on your own infrastructure. +You have two deployment options: + +* <>. Use this option if you're comfortable working with Python and want to iterate quickly locally. +* <>. Use this option if you want to deploy the connectors to a server, or use a container orchestration platform. + +[NOTE] +==== +This connector is in *technical preview* and is subject to change. The design and code is less mature than official GA features and is being provided as-is with no warranties. Technical preview features are not subject to the support SLA of official GA features. +==== + +[discrete#es-connectors-graphql-connector-usage] +==== Usage + +To set up this connector in the UI, select the *GraphQL* tile when creating a new connector under *Search -> Connectors*. + +If you're already familiar with how connectors work, you can also use the {ref}/connector-apis.html[Connector APIs]. + +For additional operations, see <>. + +[discrete#es-connectors-graphql-connector-docker] +==== Deploy with Docker + +include::_connectors-docker-instructions.asciidoc[] + +[discrete#es-connectors-graphql-connector-configuration] +==== Configuration + +[discrete#es-connectors-graphql-connector-configure-graphql-connector] +===== Configure GraphQL connector + +Note the following configuration fields: + +`http_endpoint` (required):: +Base URL of the GraphQL endpoint. +*Example*: `https://api.xyz.com/graphql` + +`http_method` (required):: +`GET` or `POST`. + +`authentication_method`(required):: +Select from `No Auth`, `Basic Auth`, and `Bearer Token`. + +`username`:: +Required when using basic authentication. + +`password`:: +Required when using basic authentication. + +`token`:: +Required when using bearer token authentication. + +`graphql_query` (required):: +Query used to fetch data from the source. +Can contain variables provided in the `graphql_variables` field. +The connector will substitute the variables in the query with values from `graphql_variables` and make a GraphQL query to the source. ++ +*Example*: ++ +[source,js] +---- +query getUser($id: ID!) { + user(id: $id) { + name + email + } +} +---- +// NOTCONSOLE + +`graphql_variables`:: +A JSON object of key/value pairs containing variables used in the GraphQL query. +The connector will substitute the variables in the query with the values provided here and make a GraphQL query to the source. ++ +*Example*: ++ +For the GraphQL query `query getUser($id: ID!) { user(id: $id) { name } }` ++ +* Where the value of `graphql_variables` is `{"id": "123"}` +* The connector will execute `query getUser { user(id: "123") { name } }` to fetch data from the source + +`graphql_object_to_id_map` (required):: +A JSON mapping between GraphQL response objects to index and their ID fields. +The connector will fetch data for each object (JSON key) and use the provided ID field (JSON value) to index the object into Elasticsearch. +The connector will index all fields for each object specified in the mapping. +Use dot `(.)` notation to specify the full path from the root of the GraphQL response to the desired object. ++ +*Example*: ++ +The GraphQL query `query getUser { organization { users{ user_id name email} } }` fetches all available users from the source. +To index every user as a separate document configure this field as below. ++ +[source,js] +---- +{ + "organization.users": "user_id" +} +---- +// NOTCONSOLE ++ +In this example `user_id` is unique in every user document. Therefore, we set `user_id` as the value for `organization.users`. ++ +[NOTE] +==== +The path provided in this field should only contain JSON objects and not lists. +==== + +`headers`:: +JSON object containing custom headers to be sent with each GraphQL request: ++ +[source,js] +---- +{ + "content-type": "Application/json" +} +---- +// NOTCONSOLE + +`pagination_model` (required):: +This field specifies the pagination model to be used by the connector. +The connector supports `No pagination` and `Cursor-based pagination` pagination models. ++ +For cursor-based pagination, add `pageInfo {endCursor hasNextPage}` and an `after` argument variable in your query at the desired node (`Pagination key`). +Use the `after` query argument with a variable to iterate through pages. +The default value for this field is `No pagination`. Example: ++ +For `Cursor-based pagination`, the query should look like this example: ++ +[source,js] +---- +query getUsers($cursor: String!) { + sampleData { + users(after: $cursor) { + pageInfo { + endCursor + hasNextPage + } + nodes { + first_name + last_name + address + } + } + } +} +---- +// NOTCONSOLE ++ +The value of `pagination_key` is `sampleData.users` so it must contain: ++ +* `pageInfo {endCursor hasNextPage}` +* the `after` argument with a variable when using cursor-based pagination + +`pagination_key` (required):: +Specifies which GraphQL object is used for pagination. +Use `.` to provide the full path of the object from the root of the response. ++ +*Example*: ++ +* `organization.users` + +`connection_timeout`:: +Specifies the maximum time in seconds to wait for a response from the GraphQL source. +Default value is *30 seconds*. + +[discrete#es-connectors-graphql-connector-documents-and-syncs] +==== Documents and syncs + +The connector syncs the objects and entities based on GraphQL Query and GraphQL Object List. + +[discrete#es-connectors-graphql-connector-sync-types] +==== Sync types + +<> are supported by default for all connectors. + +This connector currently does not support <>. + +[discrete#es-connectors-graphql-connector-sync-rules] +==== Sync rules + +<> are identical for all connectors and are available by default. + +[discrete#es-connectors-graphql-connector-advanced-sync-rules] +==== Advanced Sync Rules + +Advanced sync rules are not available for this connector in the present version. + +[discrete#es-connectors-graphql-connector-connector-client-operations] +==== Connector Client operations + +[discrete#es-connectors-graphql-connector-end-to-end-testing] +===== End-to-end Testing + +The connector framework enables operators to run functional tests against a real data source, using Docker Compose. +You don't need a running Elasticsearch instance or GraphQL source to run this test. + +Refer to <> for more details. + +To perform E2E testing for the GraphQL connector, run the following command: + +```shell +$ make ftest NAME=graphql +``` +For faster tests, add the `DATA_SIZE=small` flag: + +[source,shell] +---- +make ftest NAME=graphql DATA_SIZE=small +---- + +By default, `DATA_SIZE=MEDIUM`. + +[discrete#es-connectors-graphql-connector-known-issues] +==== Known issues + +* Every document will be updated in every sync. +* If the same field name exists with different types across different objects, the connector might raise a mapping parser exception. + +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-graphql-connector-troubleshooting] +==== Troubleshooting + +See <>. + +[discrete#es-connectors-graphql-connector-security] +==== Security + +See <>. + diff --git a/docs/reference/connector/docs/connectors-hosted-tutorial-mongo.asciidoc b/docs/reference/connector/docs/connectors-hosted-tutorial-mongo.asciidoc new file mode 100644 index 0000000000000..71ebec250dd9f --- /dev/null +++ b/docs/reference/connector/docs/connectors-hosted-tutorial-mongo.asciidoc @@ -0,0 +1,193 @@ +[#es-mongodb-start] +=== MongoDB managed connector tutorial +++++ +Managed connector tutorial (MongoDB) +++++ + +// Learn how to use the <> to sync data from https://www.mongodb.com/docs/atlas/[MongoDB Atlas^] to an Elastic Cloud deployment. + +This tutorial explains how to set up automatic, ongoing syncs from a MongoDB database to a search-optimized index on Elastic Cloud. +We'll use an Atlas deployment, together with built-in sample datasets in this tutorial, but you can also use your own data. + +This tutorial is an example of: + +* How to use the <>, which is compatible with MongoDB Atlas and on premises MongoDB servers. +See <>. +* How to use any connector available as a <>. +The workflow in this tutorial applies to all Elastic managed connectors. + +This tutorial has three main sections: + +* <>: First you'll need to get *MongoDB Atlas* up and running. +(*Skip this step* if you already have a MongoDB instance you'd like to use.) +** You'll create a free account, set up a free Atlas cluster, and load some sample data. +* <> Once that's done, you'll need to gather some details about your Atlas cluster, so you can connect it to an Elastic Cloud deployment. +* <>: Next, you'll need to get *Elastic Cloud* up and running. +** Then you'll need to create an Elasticsearch index and configure the Elastic connector to interface with your Atlas cluster. +We'll do all this in the Kibana UI. +** Once configured, you'll set a syncing schedule to start indexing your MongoDB data into Elasticsearch and ensure it stays up to date. + +[discrete#es-mongodb-start-atlas-setup] +== Set up MongoDB Atlas + +Follow the steps in the MongoDB documentation to https://www.mongodb.com/docs/atlas/getting-started[create a free Atlas account^]: + +* Create and deploy a free Atlas cluster. +* Under *Security > Network Access*, add IP access to `0.0.0.0/0`. +This CIDR-notation allows connections from any IP address. +This enables the Elastic connector, running on Elastic Cloud, to access Atlas. +See https://www.mongodb.com/docs/atlas/security/add-ip-address-to-list/[the Atlas documentation^] for complete instructions. +* Create a *database user*, with a username and password combination. +Select *Add new database user* in *Security > Database access*. +Keep these details handy, as you'll need them to configure the connector later. + +[discrete#es-mongodb-start-load-sample-data] +=== Load sample data into Atlas + +In this example we'll use the sample data available to MongoDB Atlas. +You can do this in the MongoDB Atlas UI. + +Use the *Load Sample Dataset* button in the Atlas UI, under *Database Deployments*. +Find this by selecting the *"..."* button next to your cluster name. + +.Loading sample data in Atlas UI +image::images/mongodb-load-sample-data.png[Load sample data in Atlas UI] + +Detailed instructions are available in the https://www.mongodb.com/docs/atlas/sample-data[Atlas documentation^]. + +[discrete#es-mongodb-start-view-sample-data] +=== Browse sample data in Atlas UI + +Once loaded, you can view your sample data in the Atlas UI, by selecting your database deployment's *Browse Collections* button. +Confirm that the sample databases have been added to your database deployment. + +In this example, we'll use the https://www.mongodb.com/docs/atlas/sample-data/sample-mflix/[`sample_mflix`^] dataset, which contains data on movies and movie theaters. +The database contains collections for certain metadata, including users and comments on specific movies. +We'll focus on the `comments` collection in this example. +Each document contains a comment, and information such as the commenter's name and email address. + +.A sample MongoDB document from the `comments` collection +image::images/mongodb-sample-document.png[Sample document from the comments collection] + +Later, once the connector transforms MongoDB documents into Elasticsearch documents, you can compare their structure. + +We've added data to our MongoDB Atlas cluster, and now we need to configure the Elastic MongoDB connector. + +[discrete#es-mongodb-start-gather-details] +== Gather details about your MongoDB instance + +Before we switch over to working in Elastic Cloud, we need to gather some details about our MongoDB Atlas cluster. +We'll need these details to configure the Elastic MongoDB connector. +You can find these details in the Atlas UI. + +Find the following details: + +* *Host*: The URI of your MongoDB Atlas cluster. +This should look like `mongodb+srv://.hjksqfc.mongodb.net`. +Find this by https://www.mongodb.com/docs/atlas/tutorial/connect-to-your-cluster/#connect-to-your-atlas-cluster[connecting to your cluster^] in the MongoDB Atlas UI. +** One way to find this URI is to select *Connect with MongoDB Shell* and copy the connection string from the CLI instructions. +* *Database*: The name of the database you want to sync. +In this example, we'll use the `sample_mflix` database. +* *Collection*: The name of the collection you want to sync. +In this example, we'll use the `comments` collection of the `sample_mflix` database. +* *Username*: The username you created earlier, in the the setup phase. +* *Password*: The password you created earlier. + +Keep these details handy! + +[discrete#es-mongodb-start-elastic-cloud] +== Set up Elastic Cloud + +Everything is set up in MongoDB Atlas and we have the details we need to configure the Elastic MongoDB connector. +First we'll need to get an Elastic Cloud deployment up and running. + +[discrete#es-mongodb-start-create-deployment] +=== Create an {ecloud} deployment + +[NOTE] +==== +This step is for users who are new to Elastic Cloud. +Skip this step if your team already has an Elastic Cloud deployment. +==== + +Log in to https://cloud.elastic.co/[Elastic Cloud^], and use the UI to create a deployment. +You'll need to run version *8.5.0+* or later. + +Read <> for full details. + +Once you're deployment is created, navigate to *Search*. + +[discrete#es-mongodb-start-create-index] +=== Create an Elasticsearch index + +The Elastic connector will sync your MongoDB data into a search-optimized Elasticsearch index. +The first step is to create your index in the Kibana UI. + +In the main menu, navigate to *Search > Content > Indices*, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. + +Follow these steps to create your index: + +* Select *Create an Elasticsearch index*. +* Choose *Connector* as your ingestion method. +* Select the *MongoDB* connector type. +* Name your new index, for example `search-mongo-sample`, then save. +This takes you to the *Configuration* tab of your index overview page. + +Next we need to input our Atlas details to configure the connector. + +[discrete#es-mongodb-start-configure-connector] +=== Configure the MongoDB connector + +Using the <>, configure the MongoDB connector. +Enter the details under the *Configuration* step. + +Set the *Direct connection* option to `false` for this example. +You must enable SSL/TLS for MongoDB Atlas, so toggle on the *SSL/TLS Connection* option. + +.Example configuration for the MongoDB connector +image::images/mongodb-connector-config.png[Example configuration for the MongoDB connector, width=350] + +Once you've entered these details, select *Save configuration*. + +[discrete#es-mongodb-start-launch-sync] +=== Begin syncing + +Once you've configured your MongoDB connector, it's time to schedule a sync. + +The UI will take you to the *Scheduling* tab of your index overview page. +We'll schedule a recurring sync for this example, which will run every day at midnight. + +In the *Scheduling* tab: + +* Toggle *Enable recurring syncs with the following schedule*. +* Select *Frequency*, "Every" `day`. +* Select *Time*, "At" `00:00`. +* *Save* this sync schedule. + +Once you save your sync schedule, the connector will start syncing your MongoDB Atlas data into Elasticsearch. + +[discrete#es-mongodb-start-verify-documents] +=== Verify documents + +[TIP] +==== +Our <> explains how documents in your MongoDB database and collection are extracted and transformed into documents in your Elasticsearch index. +==== + +If all the configuration details are correct, the sync will begin and documents will start to appear in your Elasticsearch index. + +As soon as your first documents are synced, you can view the documents and inspect the mapping for the index: + +* In Kibana, navigate to *Search* > *Content* > *Indices* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. +* Select your index, for example `search-mongo-sample`. +* Choose the *Documents* tab to view the synced documents. +Expand a document to view its fields. + +[discrete#es-mongodb-start-learn-more] +== Learn more + +* Refer to the <> for detailed information about the connector, including how *sync rules* work. +* For an overview of all Elastic managed connectors, see <>. +* Learn about <> for Elastic managed connectors. +* Learn about {ref}/ingest-pipeline-search.html[ingest pipelines for Search indices] +* Refer to the official https://www.mongodb.com/docs/atlas/[MongoDB Atlas documentation^] for MongoDB-specific questions. diff --git a/docs/reference/connector/docs/connectors-jira.asciidoc b/docs/reference/connector/docs/connectors-jira.asciidoc new file mode 100644 index 0000000000000..b11070b1a581a --- /dev/null +++ b/docs/reference/connector/docs/connectors-jira.asciidoc @@ -0,0 +1,544 @@ +[#es-connectors-jira] +=== Elastic Jira connector reference +++++ +Jira +++++ +// Attributes used in this file +:service-name: Jira +:service-name-stub: jira + +The _Elastic Jira connector_ is a <> for https://www.atlassian.com/software/jira[Atlassian Jira^]. +This connector is written in Python using the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/{service-name-stub}.py[*source code* for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + +.Choose your connector reference +******************************* +Are you using a managed connector on Elastic Cloud or a self-managed connector? Expand the documentation based on your deployment method. +******************************* + +// //////// //// //// //// //// //// //// //////// +// //////// NATIVE CONNECTOR REFERENCE /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-jira-native-connector-reference] +==== *Elastic managed connector reference* + +.View *Elastic managed connector* reference + +[%collapsible] +=============== + +[discrete#es-connectors-jira-availability-prerequisites] +===== Availability and prerequisites + +This connector is available as a *managed connector* on Elastic Cloud, as of *8.9.1*. + +[NOTE] +==== +Jira Data Center support was added in 8.13.0 in technical preview and is subject to change. The design and code is less mature than official GA features and is being provided as-is with no warranties. Technical preview features are not subject to the support SLA of official GA features. +==== + +To use this connector natively in Elastic Cloud, satisfy all <>. + +[discrete#es-connectors-jira-create-native-connector] +===== Create a {service-name} connector +include::_connectors-create-native.asciidoc[] + +[discrete#es-connectors-jira-usage] +===== Usage + +To use this connector natively in Elastic Cloud, see <>. + +For additional operations, see <>. + +[discrete#es-connectors-jira-compatability] +===== Compatibility + +* Jira Cloud, Jira Server, and Jira Data Center *versions 7 or later*. + +[discrete#es-connectors-jira-configuration] +===== Configuration + +The following configuration fields are required to set up the connector: + +Jira data source:: +Dropdown to determine the Jira platform type: `Jira Cloud`, `Jira Server`, or `Jira Data Center`. Default value is `Jira Cloud`. + +Jira Data Center username:: +The username of the account for Jira Data Center. + +Jira Data Center password:: +The password of the account to be used for Jira Data Center. + +Jira Cloud service account id:: +Email address to authenticate with Jira Cloud. Example: jane.doe@example.com + +Jira Cloud API token:: +The API Token to authenticate with Jira Cloud. + +Jira Server username:: +The username of the account for Jira Server. + +Jira Server password:: +The password of the account to be used for Jira Server. + +Jira Cloud service account id:: +The account email for Jira Cloud. + +Jira Cloud API token:: +The API Token to authenticate with Jira Cloud. + +Jira host url:: +The domain where Jira is hosted. Examples: + +* https://192.158.1.38:8080/ +* https://test_user.atlassian.net/ + +Jira project keys:: +Comma-separated list of https://support.atlassian.com/jira-software-cloud/docs/what-is-an-issue/#Workingwithissues-Projectkeys[Project Keys^] to fetch data from Jira server or cloud. If the value is `*` the connector will fetch data from all projects present in the configured projects. Default value is `*`. Examples: + +* `EC`, `TP` +* `*` + +Enable SSL:: +Whether SSL verification will be enabled. Default value is `False`. + +SSL certificate:: +Content of SSL certificate. Note: In case of `ssl_enabled` is `False`, the `ssl_ca` value will be ignored. Example certificate: ++ +[source, txt] +---- +-----BEGIN CERTIFICATE----- +MIID+jCCAuKgAwIBAgIGAJJMzlxLMA0GCSqGSIb3DQEBCwUAMHoxCzAJBgNVBAYT +... +7RhLQyWn2u00L7/9Omw= +-----END CERTIFICATE----- +---- + +Enable document level security:: +Toggle to enable <>. +When enabled, full syncs will fetch access control lists for each document and store them in the `_allow_access_control` field. +Access control syncs fetch users' access control lists and store them in a separate index. ++ +[NOTE] +==== +To access user data in Jira Administration, the account you created must be granted *Product Access* for Jira Administration. +This access needs to be provided by an administrator from the http://admin.atlassian.com/[Atlassian Admin], and the access level granted should be `Product Admin`. +==== + +[discrete#es-connectors-jira-documents-syncs] +===== Documents and syncs + +The connector syncs the following objects and entities: + +* **Projects** +** Includes metadata such as description, project key, project type, lead name, etc. +* **Issues** +** All types of issues including Task, Bug, Sub-task, Enhancement, Story, etc. +** Includes metadata such as issue type, parent issue details, fix versions, affected versions, resolution, attachments, comments, sub-task details, priority, custom fields, etc. +* **Attachments** + +**Note:** Archived projects and issues are not indexed. + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted. (Self-managed connectors can use the <> to handle larger binary files.) +* Permissions are not synced by default. +You must first enable <>. +Otherwise, *all documents* indexed to an Elastic deployment will be visible to *all users with access* to that Elastic Deployment. +==== + +[discrete#es-connectors-jira-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-jira-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +This connector supports <> for remote filtering. +These rules cover complex query-and-filter scenarios that cannot be expressed with basic sync rules. +Advanced sync rules are defined through a source-specific DSL JSON snippet. + +[discrete#es-connectors-jira-sync-rules-examples] +====== Advanced sync rules example + +*Example 1*: Queries to index content based on status of Jira issues. + +[source,js] +---- +[ + { + "query": "project = Collaboration AND status = 'In Progress'" + }, + { + "query": "status IN ('To Do', 'In Progress', 'Closed')" + } +] +---- +// NOTCONSOLE + +*Example 2*: Query to index data based on priority of issues for given projects. + +[source,js] +---- +[ + { + "query": "priority in (Blocker, Critical) AND project in (ProjA, ProjB, ProjC)" + } +] +---- +// NOTCONSOLE + +*Example 3*: Query to index data based on assignee and created time. + +[source,js] +---- +[ + { + "query": "assignee is EMPTY and created < -1d" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-jira-document-level-security] +===== Document level security + +Document level security (DLS) enables you to restrict access to documents based on a user’s permissions. +Refer to <> on this page for how to enable DLS for this connector. + +[WARNING] +==== +Enabling DLS for your connector will cause a significant performance degradation, as the API calls to the data source required for this functionality are rate limited. This impacts the speed at which your content can be retrieved. +==== + +[WARNING] +==== +When the `data_source` is set to Confluence Data Center or Server, the connector will only fetch 1000 users for access control syncs, due a https://auth0.com/docs/manage-users/user-search/retrieve-users-with-get-users-endpoint#limitations[limitation in the API used^]. +==== + +[NOTE] +==== +Refer to <> to learn how to ingest data from a connector with DLS enabled, when building a search application. +The example uses SharePoint Online as the data source, but the same steps apply to every connector. +==== + +[discrete#es-connectors-jira-content-extraction] +===== Content Extraction + +See <>. + +[discrete#es-connectors-jira-known-issues] +===== Known issues + +* *Enabling document-level security impacts performance.* ++ +Enabling DLS for your connector will cause a significant performance degradation, as the API calls to the data source required for this functionality are rate limited. This impacts the speed at which your content can be retrieved. + +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-jira-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-jira-security] +===== Security + +See <>. + +// Closing the collapsible section +=============== + + +// //////// //// //// //// //// //// //// //////// +// //////// CONNECTOR CLIENT REFERENCE /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-jira-connector-client-reference] +==== *Self-managed connector* + +.View *self-managed connector* reference + +[%collapsible] +=============== + +[discrete#es-connectors-jira-client-availability-prerequisites] +===== Availability and prerequisites + +This connector is available as a self-managed *self-managed connector*. +This self-managed connector is compatible with Elastic versions *8.7.0+*. + +[NOTE] +==== +Jira Data Center support was added in 8.13.0 in technical preview and is subject to change. The design and code is less mature than official GA features and is being provided as-is with no warranties. Technical preview features are not subject to the support SLA of official GA features. +==== + +To use this connector, satisfy all <>. + +[discrete#es-connectors-jira-create-connector-client] +===== Create a {service-name} connector +include::_connectors-create-client.asciidoc[] + +[discrete#es-connectors-jira-client-usage] +===== Usage + +To use this connector as a *self-managed connector*, see <> +For additional usage operations, see <>. + +[discrete#es-connectors-jira-client-compatability] +===== Compatibility + +* Jira Cloud, Jira Server, and Jira Data Center *versions 7 or later*. + +[discrete#es-connectors-jira-client-configuration] +===== Configuration + +[TIP] +==== +When using the <> workflow, initially these fields will use the default configuration set in the {connectors-python}/connectors/sources/jira.py[connector source code^]. +These are set in the `get_default_configuration` function definition. + +These configurable fields will be rendered with their respective *labels* in the Kibana UI. +Once connected, you'll be able to update these values in Kibana. +==== + +The following configuration fields are required to set up the connector: + +`data_source`:: +Dropdown to determine the Jira platform type: `Jira Cloud`, `Jira Server`, or `Jira Data Center`. Default value is `Jira Cloud`. + +`data_center_username`:: +The username of the account for Jira Data Center. + +`data_center_password`:: +The password of the account to be used for Jira Data Center. + +`username`:: +The username of the account for Jira Server. + +`password`:: +The password of the account to be used for Jira Server. + +`account_email`:: +Email address to authenticate with Jira Cloud. Example: jane.doe@example.com + +`api_token`:: +The API Token to authenticate with Jira Cloud. + +`jira_url`:: +The domain where Jira is hosted. Examples: + +* https://192.158.1.38:8080/ +* https://test_user.atlassian.net/ + +`projects`:: +Comma-separated list of https://support.atlassian.com/jira-software-cloud/docs/what-is-an-issue/#Workingwithissues-Projectkeys[Project Keys^] to fetch data from Jira server or cloud. If the value is `*` the connector will fetch data from all projects present in the configured projects. Default value is `*`. Examples: ++ +* `EC`, `TP` +* `*` ++ +[WARNING] +==== +This field can be bypassed by advanced sync rules. +==== + +`ssl_enabled`:: +Whether SSL verification will be enabled. Default value is `False`. + +`ssl_ca`:: +Content of SSL certificate. Note: In case of `ssl_enabled` is `False`, the `ssl_ca` value will be ignored. Example certificate: ++ +[source, txt] +---- +-----BEGIN CERTIFICATE----- +MIID+jCCAuKgAwIBAgIGAJJMzlxLMA0GCSqGSIb3DQEBCwUAMHoxCzAJBgNVBAYT +... +7RhLQyWn2u00L7/9Omw= +-----END CERTIFICATE----- +---- + +`retry_count`:: +The number of retry attempts after failed request to Jira. Default value is 3. + +`concurrent_downloads`:: +The number of concurrent downloads for fetching the attachment content. This speeds up the content extraction of attachments. Defaults to 100. + +`use_document_level_security`:: +Toggle to enable <>. +When enabled, full syncs will fetch access control lists for each document and store them in the `_allow_access_control` field. +Access control syncs fetch users' access control lists and store them in a separate index. ++ +[NOTE] +==== +To access user data in Jira Administration, the account you created must be granted *Product Access* for Jira Administration. +This access needs to be provided by an administrator from the http://admin.atlassian.com/[Atlassian Admin], and the access level granted should be `Product Admin`. +==== + +`use_text_extraction_service`:: +Requires a separate deployment of the <>. +Requires that ingest pipeline settings disable text extraction. +Default value is `False`. + +[discrete#es-connectors-jira-client-docker] +===== Deployment using Docker + +include::_connectors-docker-instructions.asciidoc[] + +[discrete#es-connectors-jira-client-documents-syncs] +===== Documents and syncs + +The connector syncs the following objects and entities: + +* **Projects** +** Includes metadata such as description, project key, project type, lead name, etc. +* **Issues** +** All types of issues including Task, Bug, Sub-task, Enhancement, Story, etc. +** Includes metadata such as issue type, parent issue details, fix versions, affected versions, resolution, attachments, comments, sub-task details, priority, custom fields, etc. +* **Attachments** + +**Note:** Archived projects and issues are not indexed. + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted +* Permissions are not synced by default. +You must first enable <>. +Otherwise, *all documents* indexed to an Elastic deployment will be visible to *all users with access* to that Elastic Deployment. +==== + +[discrete#es-connectors-jira-client-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-jira-client-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +This connector supports <> for remote filtering. +These rules cover complex query-and-filter scenarios that cannot be expressed with basic sync rules. +Advanced sync rules are defined through a source-specific DSL JSON snippet. + +[discrete#es-connectors-jira-client-sync-rules-examples] +====== Advanced sync rules example + +*Example 1*: Queries to index content based on status of Jira issues. + +[source,js] +---- +[ + { + "query": "project = Collaboration AND status = 'In Progress'" + }, + { + "query": "status IN ('To Do', 'In Progress', 'Closed')" + } +] +---- +// NOTCONSOLE + +*Example 2*: Query to index data based on priority of issues for given projects. + +[source,js] +---- +[ + { + "query": "priority in (Blocker, Critical) AND project in (ProjA, ProjB, ProjC)" + } +] +---- +// NOTCONSOLE + +*Example 3*: Query to index data based on assignee and created time. + +[source,js] +---- +[ + { + "query": "assignee is EMPTY and created < -1d" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-jira-client-document-level-security] +===== Document level security + +Document level security (DLS) enables you to restrict access to documents based on a user’s permissions. +Refer to <> on this page for how to enable DLS for this connector. + +[WARNING] +==== +Enabling DLS for your connector will cause a significant performance degradation, as the API calls to the data source required for this functionality are rate limited. This impacts the speed at which your content can be retrieved. +==== + +[WARNING] +==== +When the `data_source` is set to Confluence Data Center or Server, the connector will only fetch 1000 users for access control syncs, due a https://auth0.com/docs/manage-users/user-search/retrieve-users-with-get-users-endpoint#limitations[limitation in the API used^]. +==== + +[NOTE] +==== +Refer to <> to learn how to ingest data from a connector with DLS enabled, when building a search application. +The example uses SharePoint Online as the data source, but the same steps apply to every connector. +==== + +[discrete#es-connectors-jira-client-content-extraction] +===== Content Extraction + +See <>. + +[discrete#es-connectors-jira-client-connector-client-operations] +===== Self-managed connector operations + +[discrete#es-connectors-jira-client-testing] +===== End-to-end testing + +The connector framework enables operators to run functional tests against a real data source. +Refer to <> for more details. + +To perform E2E testing for the Jira connector, run the following command: + +[source,shell] +---- +$ make ftest NAME=jira +---- + +For faster tests, add the `DATA_SIZE=small` flag: + +[source,shell] +---- +make ftest NAME=jira DATA_SIZE=small +---- + +[discrete#es-connectors-jira-client-known-issues] +===== Known issues + +* *Enabling document-level security impacts performance.* ++ +Enabling DLS for your connector will cause a significant performance degradation, as the API calls to the data source required for this functionality are rate limited. This impacts the speed at which your content can be retrieved. + +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-jira-client-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-jira-client-security] +===== Security + +See <>. + +// Closing the collapsible section +=============== diff --git a/docs/reference/connector/docs/connectors-known-issues.asciidoc b/docs/reference/connector/docs/connectors-known-issues.asciidoc new file mode 100644 index 0000000000000..e8074df9526c6 --- /dev/null +++ b/docs/reference/connector/docs/connectors-known-issues.asciidoc @@ -0,0 +1,166 @@ +[#es-connectors-known-issues] +== Connector known issues +++++ +Known issues +++++ + +[discrete#es-connectors-known-issues-enterprie-search-service] +=== Enterprise Search service: self-managed connectors + +As of *8.10.0* self-managed connectors no longer require the Enterprise Search service to be running on your Elastic deployment. +However, if you are upgrading connectors from versions _earlier than 8.9_, you'll need to run Enterprise Search once to migrate your connectors to the new format. + +Some points to note about this migration: + +* This involves updating system indices that store configuration and sync history for your connectors. +* This is an in-place operation, meaning no temporary or backup indices will be created. +* Therefore, it is important to take a snapshot of the Elasticsearch cluster before upgrading— in the unlikely event of an index migration failure. + +If you have trouble with this migration, please contact support. + +[WARNING] +==== +To run self-managed connectors your self-deployed connector service version must match your Elasticsearch version. +For example, if you're running Elasticsearch 8.10.1, your connector service should be version 8.10.1.x. +Elastic does not support deployments running mismatched versions (except during upgrades). +==== + +[discrete#es-connectors-known-issues-connector-service] +=== Connector service + +The connector service has the following known issues: + +* *OOM errors when syncing large database tables* ++ +Syncs after the initial sync can cause out-of-memory (OOM) errors when syncing large database tables. +This occurs because database connectors load and store IDs in memory. +For tables with millions of records, this can lead to memory exhaustion if the connector service has insufficient RAM. ++ +To mitigate this issue, you can: ++ +** *Increase RAM allocation*: +*** *Elastic Cloud*: Upgrade the Enterprise Search instance to a larger size. Note that for Elastic managed connectors running on Elastic Cloud, the connector service runs on the Enterprise Search node. It only has access to up to 40% of the node’s RAM allocation. +*** *Self-managed*: Increase RAM allocation for the machine/container running the connector service. ++ +.RAM *sizing guidelines* +[%collapsible] +============== +The following table shows the estimated RAM usage for loading IDs into memory. +|=== +| *Number of IDs* | *Memory Usage in MB (2X buffer)* +| 1,000,000 +| ≈ 45.78 MB +| 10,000,000 +| ≈ 457.76 MB +| 50,000,000 +| ≈ 2288.82 MB (≈ 2.29 GB) +| 100,000,000 +| ≈ 4577.64 MB (≈ 4.58 GB) +|=== +============== ++ +** *Optimize* <>: +*** Review and optimize sync rules to filter and reduce data retrieved from the source before syncing. ++ +** *Use a self-managed connector* instead of a managed connector: +*** Because self-managed connectors run on your infrastructure, they are not subject to the same RAM limitations of the Enterprise Search node. + +* *Upgrades from deployments running on versions earlier than 8.9.0 can cause sync job failures* ++ +Due to a bug, the `job_type` field mapping will be missing after upgrading from deployments running on versions earlier than 8.9.0. +Sync jobs won't be displayed in the Kibana UI (job history) and the connector service won't be able to start new sync jobs. +*This will only occur if you have previously scheduled sync jobs.* ++ +To resolve this issue, you can manually add the missing field with the following command and trigger a sync job: ++ +[source,console] +---- +PUT .elastic-connectors-sync-jobs-v1/_mapping +{ + "properties": { + "job_type": { + "type": "keyword" + } + } +} +---- +// TEST[skip:TODO] + +* *The connector service will fail to sync when the connector tries to fetch more more than 2,147,483,647 (_2^31-1_) documents from a data source* ++ +A workaround is to manually partition the data to be synced using multiple search indices. ++ +* *Custom scheduling might break when upgrading from version 8.6 or earlier.* ++ +If you encounter the error `'custom_schedule_triggered': undefined method 'each' for nil:NilClass (NoMethodError)`, it means the custom scheduling feature migration failed. +You can use the following manual workaround: ++ +[source,console] +---- +POST /.elastic-connectors/_update/connector-id +{ + "doc": { + "custom_scheduling": {} + } +} +---- +// TEST[skip:TODO] ++ +This error can appear on Connectors or Crawlers that aren't the cause of the issue. +If the error continues, try running the above command for every document in the `.elastic-connectors` index. ++ +* *Connectors upgrading from 8.7 or earlier can be missing configuration fields* ++ +A connector that was created prior to 8.8 can sometimes be missing configuration fields. +This is a known issue for the MySQL connector but could also affect other connectors. ++ +If the self-managed connector raises the error `Connector for has missing configuration fields: , ...`, you can resolve the error by manually adding the missing configuration fields via the Dev Tools. +Only the following two field properties are required, as the rest will be autopopulated by the self-managed connector: ++ +** `type`: one of `str`, `int`, `bool`, or `list` +** `value`: any value, as long as it is of the correct `type` (`list` type values should be saved as comma-separated strings) ++ +[source,console] +---- +POST /.elastic-connectors/_update/connector_id +{ + "doc" : { + "configuration": { + "field_a": { + "type": "str", + "value": "" + }, + "field_b": { + "type": "bool", + "value": false + }, + "field_c": { + "type": "int", + "value": 1 + }, + "field_d": { + "type": "list", + "value": "a,b" + } + } + } +} +---- +// TEST[skip:TODO] ++ +* *Python connectors that upgraded from 8.7.1 will report document volumes in gigabytes (GB) instead of megabytes (MB)* ++ +As a result, true document volume will be under-reported by a factor of 1024. ++ +* *The following Elastic managed connectors will not run correctly on Elastic Cloud in 8.9.0.* +They are still available as self-managed connectors. +** Azure Blob Storage +** Confluence Cloud & Server +** Jira Cloud & Server +** Network drives + +[discrete#es-connectors-known-issues-specific] +=== Individual connector known issues + +Individual connectors may have additional known issues. +Refer to <> for connector-specific known issues. diff --git a/docs/reference/connector/docs/connectors-logs.asciidoc b/docs/reference/connector/docs/connectors-logs.asciidoc new file mode 100644 index 0000000000000..9e0000d442120 --- /dev/null +++ b/docs/reference/connector/docs/connectors-logs.asciidoc @@ -0,0 +1,63 @@ +[#es-connectors-logs] +=== Connector logs +++++ +Logs +++++ + +This document describes logs for <> and <>. + +[discrete#es-connectors-logs-enable] +==== Enable logs + +[discrete#es-connectors-logs-enable-cloud] +===== Elastic Cloud + +Elastic Cloud users need to {cloud}/ec-enable-logging-and-monitoring.html#ec-enable-logging-and-monitoring-steps[enable logging^] to view connector logs. +Go to *Cloud > Deployment > _your-deployment_ > Logs and metrics* to enable logs. + +Once enabled, <>. + +Configure the `log_level` user setting for Enterprise Search. See {cloud}/ec-manage-enterprise-search-settings.html[Add Enterprise Search user settings^] in the Elastic Cloud documentation. + +[discrete#es-connectors-logs-enable-self-managed] +===== self-managed connectors + +Note that self-managed deployments and self-managed connector logs are written to `STDOUT`. + +Self-managed connectors have the following logging options: + +* Use the `service.log_level` setting in your connector service configuration file to specify the log level for the service. +** Enable `elasticsearch.bulk.enable_operations_logging` to log the result of sending documents to Elasticsearch from connectors, for auditing and debugging. This setting depends on the `service.log_level` and will be logged at `DEBUG` level . +* Use the `elasticsearch.log_level` setting to specify the log level for the Elasticsearch _client_ used by the connector service. + +[discrete#es-connectors-logs-view] +==== View connector logs + +You can view logs in Kibana. + +You can filter by `service.type`: + +- `enterprise-search` +- `connectors` + +[discrete#es-connectors-logs-reference] +==== Logs reference + +Logs use Elastic Common Schema (ECS), without extensions. +See {ecs-ref}[the ECS Reference^] for more information. + +The fields logged are: + +* `@timestamp` +* `log.level` +* `ecs.version` +* `labels.index_date` +* `tags` +* `log.logger` +* `service.type` +* `service.version` +* `process.name` +* `process.pid` +* `process.thread.id` + +See {ref}/logging.html[Logging^] in the Elasticsearch documentation for more information. diff --git a/docs/reference/connector/docs/connectors-managed-service.asciidoc b/docs/reference/connector/docs/connectors-managed-service.asciidoc new file mode 100644 index 0000000000000..98144ed74bcfa --- /dev/null +++ b/docs/reference/connector/docs/connectors-managed-service.asciidoc @@ -0,0 +1,207 @@ +[#es-native-connectors] +== Elastic managed connectors + +.Naming history +**** +Elastic managed connectors were initially known as "native connectors". +You might find this term in older documentation. +**** + +Managed <> are available directly within your Elastic Cloud deployment. +No additional infrastructure is required. + +Managed connectors sync data sources directly to Elasticsearch indices. +Create these indices using the *Connector* workflow within {kib}. + +The following connectors are available as Elastic managed connectors. +Refer to each connector reference for additional information specific to each connector. + +include::_connectors-list-native.asciidoc[] + +[discrete#es-native-connectors-prerequisites] +=== Availability and prerequisites + +Managed connectors were introduced in Elastic version *8.5.0*. + +Your Elastic Cloud deployment must include the following Elastic services: + +* *Elasticsearch* +* *{kib}* +* {enterprise-search-ref}/server.html[*Enterprise Search*] + +Refer to _Native Integrations_ on the https://www.elastic.co/subscriptions/cloud[Elastic subscriptions page], in the *Elastic Search* section for managed connector licensing requirements. + +.Elastic Cloud IPs +[sidebar] +-- +Using our Elastic managed connectors involves outbound data transfer (egress) from your Elastic Cloud deployment. +If you have IP/firewall rules on your third party service, you'll need to add the Elastic Cloud egress static IP ranges to your service’s configuration. + +Refer to {cloud}/ec-static-ips.html[static IP ranges] in the Elastic Cloud documentation for the latest list of IP addresses. +-- + +[discrete#es-native-connectors-usage] +=== Usage in {kib} UI + +Follow the *Connector* workflow in {kib} to select the *Connector* ingestion method. +Choose a data source, create an Elasticsearch index, and configure a managed connector to manage the index. + +[.screenshot] +image::images/use-a-connector-workflow.png[] + +[discrete#es-native-connectors-select-connector] +==== Select a connector + +Choose the data source to sync from the available options and select *Continue*. + +[discrete#es-native-connectors-index] +==== Create index + +Create a new index to be managed by the connector: + +. Name your index and optionally change the language analyzer to match the human language of your data source. +(The index name will be automatically prefixed with `search-`.) +. Select *Create index*. + +The index is created and ready to <>. + +This operation requires: + +. Access to {kib} +. Permission to create or manage the index +. `write` {ref}/security-privileges.html[indices privilege^] for the `.elastic-connectors` index +. `manage_api_key` {ref}/security-privileges.html[cluster privilege^] to allow the API key to be created +. `write_connector_secrets` {ref}/security-privileges.html[cluster privilege^] to allow the API key to be stored as a secret + +[discrete#es-native-connectors-configuration] +==== Configure connector + +Create a new index to be managed by the connector. + +Continue from above, or navigate to the following location within the {kib} UI: + +*Search > Content > Elasticsearch indices* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. + +Choose the index to configure, and then choose the *Configuration* tab. + +Configure the connector: + +. Edit the name and description for the connector. +Your team can use this information to differentiate this index from other connector indices. +(These fields describe the _connector_ and are independent of the Elasticsearch index name.) +. Save your changes. +. Edit the data source configuration. +The fields here vary by connector. +Refer to the documentation for each connector for details (refer to list of Elastic managed connectors, above). +Refer to <> for security considerations. +. Save your changes. + +Optionally choose *Edit sync schedule* to begin <>. + +This operation requires: + +. Access to {kib} +. Permission to create or manage the index +. `write` {ref}/security-privileges.html[indices privilege^] for the `.elastic-connectors` index +. `manage_api_key` {ref}/security-privileges.html[cluster privilege^] to allow the API key to be created +. `write_connector_secrets` {ref}/security-privileges.html[cluster privilege^] to allow the API key to be stored as a secret + +[discrete#es-native-connectors-management] +==== Manage connector + +To manage documents, syncs, sync rules, ingest pipelines, and other connector features, refer to <>. + +[discrete#es-native-connectors-manage-API-keys] +=== Manage API keys + +[NOTE] +==== +API keys for Elastic managed connectors were introduced in Elastic version *8.13.0*. +Managed connectors created in earlier versions will not automatically use API keys upon upgrading to *8.13.0*. +Refer to <> for more information. +==== + +Managed connectors communicate with Elasticsearch using API keys. +When managing a managed connector through the Connectors page in the {kib} UI, API key management is handled automatically. +API keys for Elastic managed connectors have the `manage` permission for three indices: + +. The attached index +. The access control (ACL) index used for document level security +. The internal `.elastic-connectors` index. + +Changing the attached index through {kib} will automatically invalidate the existing API key and generate a new one. +If you want to rotate an existing API key, navigate to the *Configuration* tab. +Scroll down to *Manage API key* and select *Generate API key*. +This action will invalidate the previous API key, create a new API key, and update the connector secret. + +API keys for Elastic managed connectors are stored on an internal system index called `.connector-secrets`. +This index can only be written to through API requests by users with the `write_connector-secrets` cluster privilege. +Only the Enterprise Search instance has permission to read from this index. + +Users managing Elastic managed connectors will need the `write_connector_secrets` cluster privilege assigned to their role. +Refer to <> for security considerations. + +[discrete#es-native-connectors-manage-API-keys-programmatically] +==== Manage API keys programmatically + +You can also create and store API keys programmatically. + +.*Expand* the following section for details. +[%collapsible] +=================================== +include::_connectors-create-native-api-key.asciidoc[] +=================================== + +[discrete#es-native-connectors-enabling-API-keys-for-upgraded-connectors] +==== Enabling API keys for upgraded connectors + +Managed connectors created before *8.13.0* do not initially have API keys upon upgrading. +The attached indices of these connectors cannot be changed until the connector has been converted to use an API key. + +.*Expand* the following section for steps on enabling API keys for upgraded Elastic managed connectors. +[%collapsible] +=================================== +. Run the following command in *Dev Tools* to enable API keys for the connector, replacing values where indicated. ++ +[source, console,subs="+attributes"] +---- +POST .elastic-connectors/_update/connector_id +{ + "doc": { + "features": { + "native_connector_api_keys": { + "enabled": true + } + } + } +} +---- +// TEST[skip:requires connector_id] ++ +. Go back to the Connectors page and navigate to the *Configuration* tab. +. Scroll down to *Manage API key* and select *Generate API key*. + +Your managed connector is now using API keys to authorize ingestion into Elasticsearch. +=================================== + +[discrete#es-native-connectors-usage-api] +=== Usage via API + +In 8.12 we introduced a set of {ref}/connector-apis.html[Connector APIs] to create and manage Elastic connectors and sync jobs, along with a https://github.com/elastic/connectors/blob/main/docs/CLI.md[CLI tool]. +Use these tools if you'd like to work with connectors and sync jobs programmatically. + +[discrete#es-native-connectors-example] +=== End-to-end example + +The following example demonstrates how to use a managed connector on Elastic Cloud: <>. + +[discrete#es-native-connectors-convert] +=== Convert a managed connector + +You can convert a managed connector to a self-managed connector to be run on your own infrastructure. +You'll find instructions in the UI on the connector index's overview page. + +[WARNING] +==== +Converting a managed connector to a self-managed connector is an irreversible operation! +==== diff --git a/docs/reference/connector/docs/connectors-management.asciidoc b/docs/reference/connector/docs/connectors-management.asciidoc new file mode 100644 index 0000000000000..77f8b32cb0b05 --- /dev/null +++ b/docs/reference/connector/docs/connectors-management.asciidoc @@ -0,0 +1,9 @@ +[#es-connectors-management] +== Management topics + +Refer to the following sections: + +* <> +* <> +* <> +* <> \ No newline at end of file diff --git a/docs/reference/connector/docs/connectors-mongodb.asciidoc b/docs/reference/connector/docs/connectors-mongodb.asciidoc new file mode 100644 index 0000000000000..8e62437507009 --- /dev/null +++ b/docs/reference/connector/docs/connectors-mongodb.asciidoc @@ -0,0 +1,777 @@ +[#es-connectors-mongodb] +=== Elastic MongoDB connector reference +++++ +MongoDB +++++ +// Attributes used in this file +:service-name: MongoDB +:service-name-stub: mongodb + +The _Elastic MongoDB connector_ is a <> for https://www.mongodb.com[MongoDB^] data sources. +This connector is written in Python using the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/{service-name-stub}.py[*source code* for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + +.Choose your connector reference +******************************* +Are you using a managed connector on Elastic Cloud or a self-managed connector? Expand the documentation based on your deployment method. +******************************* + +// //////// //// //// //// //// //// //// //////// +// //////// NATIVE CONNECTOR REFERENCE /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-mongodb-native-connector-reference] +==== *Elastic managed connector reference* + +.View *Elastic managed connector* reference + +[%collapsible] +=============== + +[discrete#es-connectors-mongodb-prerequisites] +===== Availability and prerequisites + +This connector is available as a *managed connector* in Elastic versions *8.5.0 and later*. + +To use this connector natively in Elastic Cloud, satisfy all <>. + +[discrete#es-connectors-mongodb-compatibility] +===== Compatibility + +This connector is compatible with *MongoDB Atlas* and *MongoDB 3.6 and later*. + +The data source and your Elastic deployment must be able to communicate with each other over a network. + +[discrete#es-connectors-mongodb-configuration] +===== Configuration + +Each time you create an index to be managed by this connector, you will create a new connector configuration. +You will need some or all of the following information about the data source. + +Server hostname:: +The URI of the MongoDB host. +Examples: ++ +* `mongodb+srv://my_username:my_password@cluster0.mongodb.net/mydb?w=majority` +* `mongodb://127.0.0.1:27017` + +Username:: +The MongoDB username the connector will use. ++ +The user must have access to the configured database and collection. +You may want to create a dedicated, read-only user for each connector. + +Password:: +The MongoDB password the connector will use. + +Database:: +The MongoDB database to sync. +The database must be accessible using the configured username and password. + +Collection:: +The MongoDB collection to sync. +The collection must exist within the configured database. +The collection must be accessible using the configured username and password. + +Direct connection:: +Toggle to use the https://www.mongodb.com/docs/ruby-driver/current/reference/create-client/#direct-connection[direct connection option for the MongoDB client^]. +Disabled by default. + +SSL/TLS Connection:: +Toggle to establish a secure connection to the MongoDB server using SSL/TLS encryption. +Ensure that your MongoDB deployment supports SSL/TLS connections. +*Enable* if your MongoDB cluster uses DNS SRV records (namely MongoDB Atlas users). ++ +Disabled by default. + +Certificate Authority (.pem):: +Specifies the root certificate from the Certificate Authority. +The value of the certificate is used to validate the certificate presented by the MongoDB instance. +[TIP] +==== +Atlas users can leave this blank because https://www.mongodb.com/docs/atlas/reference/faq/security/#which-certificate-authority-signs-mongodb-atlas-tls-certificates-[Atlas uses a widely trusted root CA]. +==== + +Skip certificate verification:: +Skips various certificate validations (if SSL is enabled). +Disabled by default. +[NOTE] +==== +We strongly recommend leaving this option disabled in production environments. +==== + +[discrete#es-connectors-mongodb-create-native-connector] +===== Create a {service-name} connector +include::_connectors-create-native.asciidoc[] + +[discrete#es-connectors-mongodb-usage] +===== Usage + +To use this connector as a *managed connector*, use the *Connector* workflow. +See <>. + +For additional operations, see <>. + +[discrete#es-connectors-mongodb-example] +===== Example + +An example is available for this connector. +See <>. + +[discrete#es-connectors-mongodb-known-issues] +===== Known issues + +[discrete#es-connectors-mongodb-known-issues-ssl-tls-812] +====== SSL must be enabled for MongoDB Atlas + +* A bug introduced in *8.12.0* causes the connector to fail to sync Mongo *Atlas* urls (`mongo+srv`) unless SSL/TLS is enabled. +// https://github.com/elastic/sdh-enterprise-search/issues/1283#issuecomment-1919731668 + +[discrete#es-connectors-mongodb-known-issues-expressions-and-variables-in-aggregation-pipelines] +====== Expressions and variables in aggregation pipelines + +It's not possible to use expressions like `new Date()` inside an aggregation pipeline. +These expressions won't be evaluated by the underlying MongoDB client, but will be passed as a string to the MongoDB instance. +A possible workaround is to use https://www.mongodb.com/docs/manual/reference/aggregation-variables/[aggregation variables]. + +Incorrect (`new Date()` will be interpreted as string): +[source,js] +---- +{ + "aggregate": { + "pipeline": [ + { + "$match": { + "expiresAt": { + "$gte": "new Date()" + } + } + } + ] + } +} +---- +// NOTCONSOLE + +Correct (usage of https://www.mongodb.com/docs/manual/reference/aggregation-variables/#mongodb-variable-variable.NOW[$$NOW]): +[source,js] +---- +{ + "aggregate": { + "pipeline": [ + { + "$addFields": { + "current_date": { + "$toDate": "$$NOW" + } + } + }, + { + "$match": { + "$expr": { + "$gte": [ + "$expiresAt", + "$current_date" + ] + } + } + } + ] + } +} +---- +// NOTCONSOLE + +[discrete#es-connectors-mongodb-known-issues-tls-with-invalid-cert] +====== Connecting with self-signed or custom CA TLS Cert + +Currently, the MongoDB connector does not support working with self-signed or custom CA certs when connecting to your self-managed MongoDB host. + +[WARNING] +==== +The following workaround should not be used in production. +==== + +This can be worked around in development environments, by appending certain query parameters to the configured host. + +For example, if your host is `mongodb+srv://my.mongo.host.com`, appending `?tls=true&tlsAllowInvalidCertificates=true` will allow disabling TLS certificate verification. + +The full host in this example will look like this: + +`mongodb+srv://my.mongo.host.com/?tls=true&tlsAllowInvalidCertificates=true` + +See <> for any issues affecting all connectors. + +[discrete#es-connectors-mongodb-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-mongodb-security] +===== Security + +See <>. + +[discrete#es-connectors-mongodb-syncs] +===== Documents and syncs + +The following describes the default syncing behavior for this connector. +Use <> and {ref}/ingest-pipeline-search.html[ingest pipelines] to customize syncing for specific indices. + +All documents in the configured MongoDB database and collection are extracted and transformed into documents in your Elasticsearch index. + +* The connector creates one *Elasticsearch document* for each MongoDB document in the configured database and collection. +* For each document, the connector transforms each MongoDB field into an *Elasticsearch field*. +* For each field, Elasticsearch {ref}/dynamic-mapping.html[dynamically determines the *data type*^]. + +This results in Elasticsearch documents that closely match the original MongoDB documents. + +The Elasticsearch mapping is created when the first document is created. + +Each sync is a "full" sync. +For each MongoDB document discovered: + +* If it does not exist, the document is created in Elasticsearch. +* If it already exists in Elasticsearch, the Elasticsearch document is replaced and the version is incremented. +* If an existing Elasticsearch document no longer exists in the MongoDB collection, it is deleted from Elasticsearch. +* Embedded documents are stored as an `object` field in the parent document. + +This is recursive, because embedded documents can themselves contain embedded documents. + +[NOTE] +==== +* Files bigger than 10 MB won't be extracted +* Permissions are not synced. All documents indexed to an Elastic deployment will be visible to *all users with access* to that Elastic Deployment. +==== + +[discrete#es-connectors-mongodb-sync-rules] +===== Sync rules + +The following sections describe <> for this connector. + +<> are identical for all connectors and are available by default. + +<> for MongoDB can be used to express either `find` queries or aggregation pipelines. +They can also be used to tune options available when issuing these queries/pipelines. + +[discrete#es-connectors-mongodb-sync-rules-find] +====== `find` queries + +[NOTE] +==== +You must create a https://www.mongodb.com/docs/current/core/indexes/index-types/index-text/[text index^] on the MongoDB collection in order to perform text searches. +==== + +For `find` queries, the structure of this JSON DSL should look like: + +[source,js] +---- +{ + "find":{ + "filter": { + // find query goes here + }, + "options":{ + // query options go here + } + } +} + +---- +// NOTCONSOLE + +For example: + +[source,js] +---- +{ + "find": { + "filter": { + "$text": { + "$search": "garden", + "$caseSensitive": false + } + }, + "skip": 10, + "limit": 1000 + } +} +---- +// NOTCONSOLE + +`find` queries also support additional options, for example the `projection` object: + +[source,js] +---- +{ + "find": { + "filter": { + "languages": [ + "English" + ], + "runtime": { + "$gt":90 + } + }, + "projection":{ + "tomatoes": 1 + } + } +} +---- +// NOTCONSOLE +Where the available options are: + +* `allow_disk_use` (true, false) — When set to true, the server can write temporary data to disk while executing the find operation. This option is only available on MongoDB server versions 4.4 and newer. +* `allow_partial_results` (true, false) — Allows the query to get partial results if some shards are down. +* `batch_size` (Integer) — The number of documents returned in each batch of results from MongoDB. +* `filter` (Object) — The filter criteria for the query. +* `limit` (Integer) — The max number of docs to return from the query. +* `max_time_ms` (Integer) — The maximum amount of time to allow the query to run, in milliseconds. +* `no_cursor_timeout` (true, false) — The server normally times out idle cursors after an inactivity period (10 minutes) to prevent excess memory use. Set this option to prevent that. +* `projection` (Array, Object) — The fields to include or exclude from each doc in the result set. If an array, it should have at least one item. +* `return_key` (true, false) — Return index keys rather than the documents. +* `show_record_id` (true, false) — Return the `$recordId` for each doc in the result set. +* `skip` (Integer) — The number of docs to skip before returning results. + +[discrete#es-connectors-mongodb-sync-rules-aggregation] +====== Aggregation pipelines + +Similarly, for aggregation pipelines, the structure of the JSON DSL should look like: + +[source,js] +---- +{ + "aggregate":{ + "pipeline": [ + // pipeline elements go here + ], + "options": { + // pipeline options go here + } + } +} +---- +// NOTCONSOLE + +Where the available options are: + +* `allowDiskUse` (true, false) — Set to true if disk usage is allowed during the aggregation. +* `batchSize` (Integer) — The number of documents to return per batch. +* `bypassDocumentValidation` (true, false) — Whether or not to skip document level validation. +* `collation` (Object) — The collation to use. +* `comment` (String) — A user-provided comment to attach to this command. +* `hint` (String) — The index to use for the aggregation. +* `let` (Object) — Mapping of variables to use in the pipeline. See the server documentation for details. +* `maxTimeMs` (Integer) — The maximum amount of time in milliseconds to allow the aggregation to run. + +[discrete#es-connectors-mongodb-migration-from-ruby] +===== Migrating from the Ruby connector framework + +As part of the 8.8.0 release the MongoDB connector was moved from the {connectors-python}[Ruby connectors framework^] to the {connectors-python}[Elastic connector framework^]. + +This change introduces minor formatting modifications to data ingested from MongoDB: + +1. Nested object id field name has changed from "_id" to "id". For example, if you had a field "customer._id", this will now be named "customer.id". +2. Date format has changed from `YYYY-MM-DD'T'HH:mm:ss.fff'Z'` to `YYYY-MM-DD'T'HH:mm:ss` + +If your MongoDB connector stopped working after migrating from 8.7.x to 8.8.x, read the workaround outlined in <>. +If that does not work, we recommend deleting the search index attached to this connector and re-creating a MongoDB connector from scratch. + + +// Closing the collapsible section +=============== + + +// //////// //// //// //// //// //// //// //////// +// //////// CONNECTOR CLIENT REFERENCE /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-mongodb-connector-client-reference] +==== *Self-managed connector* + +.View *self-managed connector* reference + +[%collapsible] +=============== + +[discrete#es-connectors-mongodb-client-prerequisites] +===== Availability and prerequisites + +This connector is also available as a *self-managed connector* from the *Elastic connector framework*. +To use this connector as a self-managed connector, satisfy all <>. + +[discrete#es-connectors-mongodb-client-compatibility] +===== Compatibility + +This connector is compatible with *MongoDB Atlas* and *MongoDB 3.6 and later*. + +The data source and your Elastic deployment must be able to communicate with each other over a network. + +[discrete#es-connectors-mongodb-client-configuration] +===== Configuration + +[TIP] +==== +When using the <> workflow, initially these fields will use the default configuration set in the {connectors-python}/connectors/sources/jira.py[connector source code^]. +These are set in the `get_default_configuration` function definition. + +These configurable fields will be rendered with their respective *labels* in the Kibana UI. +Once connected, you'll be able to update these values in Kibana. +==== + +The following configuration fields are required to set up the connector: + +`host`:: +The URI of the MongoDB host. +Examples: ++ +* `mongodb+srv://my_username:my_password@cluster0.mongodb.net/mydb?w=majority` +* `mongodb://127.0.0.1:27017` + + +`user`:: +The MongoDB username the connector will use. ++ +The user must have access to the configured database and collection. +You may want to create a dedicated, read-only user for each connector. + +`password`:: +The MongoDB password the connector will use. + +[NOTE] +==== +Anonymous authentication is supported for _testing purposes only_, but should not be used in production. +Omit the username and password, to use default values. +==== + +`database`:: +The MongoDB database to sync. +The database must be accessible using the configured username and password. + +`collection`:: +The MongoDB collection to sync. +The collection must exist within the configured database. +The collection must be accessible using the configured username and password. + +`direct_connection`:: +Whether to use the https://www.mongodb.com/docs/ruby-driver/current/reference/create-client/#direct-connection[direct connection option for the MongoDB client^]. +Default value is `False`. + +`ssl_enabled`:: +Whether to establish a secure connection to the MongoDB server using SSL/TLS encryption. +Ensure that your MongoDB deployment supports SSL/TLS connections. +*Enable* if your MongoDB cluster uses DNS SRV records (namely MongoDB Atlas users). ++ +Default value is `False`. + +`ssl_ca`:: +Specifies the root certificate from the Certificate Authority. +The value of the certificate is used to validate the certificate presented by the MongoDB instance. +[TIP] +==== +Atlas users can leave this blank because https://www.mongodb.com/docs/atlas/reference/faq/security/#which-certificate-authority-signs-mongodb-atlas-tls-certificates-[Atlas uses a widely trusted root CA]. +==== + +`tls_insecure`:: +Skips various certificate validations (if SSL is enabled). +Default value is `False`. +[NOTE] +==== +We strongly recommend leaving this option disabled in production environments. +==== + +[discrete#es-connectors-mongodb-create-connector-client] +===== Create a {service-name} connector +include::_connectors-create-client.asciidoc[] + +[discrete#es-connectors-mongodb-client-usage] +===== Usage + +To use this connector as a *self-managed connector*, see <> +For additional usage operations, see <>. + +[discrete#es-connectors-mongodb-client-example] +===== Example + +An example is available for this connector. +See <>. + +[discrete#es-connectors-mongodb-client-known-issues] +===== Known issues + +[discrete#es-connectors-mongodb-client-known-issues-ssl-tls-812] +====== SSL must be enabled for MongoDB Atlas + +* A bug introduced in *8.12.0* causes the connector to fail to sync Mongo *Atlas* urls (`mongo+srv`) unless SSL/TLS is enabled. +// https://github.com/elastic/sdh-enterprise-search/issues/1283#issuecomment-1919731668 + + +[discrete#es-connectors-mongodb-client-known-issues-expressions-and-variables-in-aggregation-pipelines] +====== Expressions and variables in aggregation pipelines + +It's not possible to use expressions like `new Date()` inside an aggregation pipeline. +These expressions won't be evaluated by the underlying MongoDB client, but will be passed as a string to the MongoDB instance. +A possible workaround is to use https://www.mongodb.com/docs/manual/reference/aggregation-variables/[aggregation variables]. + +Incorrect (`new Date()` will be interpreted as string): +[source,js] +---- +{ + "aggregate": { + "pipeline": [ + { + "$match": { + "expiresAt": { + "$gte": "new Date()" + } + } + } + ] + } +} +---- +// NOTCONSOLE + +Correct (usage of https://www.mongodb.com/docs/manual/reference/aggregation-variables/#mongodb-variable-variable.NOW[$$NOW]): +[source,js] +---- +{ + "aggregate": { + "pipeline": [ + { + "$addFields": { + "current_date": { + "$toDate": "$$NOW" + } + } + }, + { + "$match": { + "$expr": { + "$gte": [ + "$expiresAt", + "$current_date" + ] + } + } + } + ] + } +} +---- +// NOTCONSOLE + +[discrete#es-connectors-mongodb-client-known-issues-tls-with-invalid-cert] +====== Connecting with self-signed or custom CA TLS Cert + +Currently, the MongoDB connector does not support working with self-signed or custom CA certs when connecting to your self-managed MongoDB host. + +[WARNING] +==== +The following workaround should not be used in production. +==== + +This can be worked around in development environments, by appending certain query parameters to the configured host. + +For example, if your host is `mongodb+srv://my.mongo.host.com`, appending `?tls=true&tlsAllowInvalidCertificates=true` will allow disabling TLS certificate verification. + +The full host in this example will look like this: + +`mongodb+srv://my.mongo.host.com/?tls=true&tlsAllowInvalidCertificates=true` + +[discrete#es-connectors-mongodb-known-issues-docker-image-fails] +====== Docker image errors out for versions 8.12.0 and 8.12.1 + +A bug introduced in *8.12.0* causes the Connectors docker image to error out if run using MongoDB as its source. +The command line will output the error `cannot import name 'coroutine' from 'asyncio'`. +** This issue is fixed in versions *8.12.2* and *8.13.0*. +** This bug does not affect Elastic managed connectors. + +See <> for any issues affecting all connectors. + +[discrete#es-connectors-mongodb-client-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-mongodb-client-security] +===== Security + +See <>. + +[discrete#es-connectors-mongodb-client-docker] +===== Deployment using Docker + +include::_connectors-docker-instructions.asciidoc[] + +[discrete#es-connectors-mongodb-client-syncs] +===== Documents and syncs + +The following describes the default syncing behavior for this connector. +Use <> and {ref}/ingest-pipeline-search.html[ingest pipelines] to customize syncing for specific indices. + +All documents in the configured MongoDB database and collection are extracted and transformed into documents in your Elasticsearch index. + +* The connector creates one *Elasticsearch document* for each MongoDB document in the configured database and collection. +* For each document, the connector transforms each MongoDB field into an *Elasticsearch field*. +* For each field, Elasticsearch {ref}/dynamic-mapping.html[dynamically determines the *data type*^]. + +This results in Elasticsearch documents that closely match the original MongoDB documents. + +The Elasticsearch mapping is created when the first document is created. + +Each sync is a "full" sync. +For each MongoDB document discovered: + +* If it does not exist, the document is created in Elasticsearch. +* If it already exists in Elasticsearch, the Elasticsearch document is replaced and the version is incremented. +* If an existing Elasticsearch document no longer exists in the MongoDB collection, it is deleted from Elasticsearch. +* Embedded documents are stored as an `object` field in the parent document. + +This is recursive, because embedded documents can themselves contain embedded documents. + +[NOTE] +==== +* Files bigger than 10 MB won't be extracted +* Permissions are not synced. All documents indexed to an Elastic deployment will be visible to *all users with access* to that Elastic Deployment. +==== + +[discrete#es-connectors-mongodb-client-sync-rules] +===== Sync rules + +The following sections describe <> for this connector. + +<> are identical for all connectors and are available by default. + +<> for MongoDB can be used to express either `find` queries or aggregation pipelines. +They can also be used to tune options available when issuing these queries/pipelines. + +[discrete#es-connectors-mongodb-client-sync-rules-find] +====== `find` queries + +[NOTE] +==== +You must create a https://www.mongodb.com/docs/current/core/indexes/index-types/index-text/[text index^] on the MongoDB collection in order to perform text searches. +==== + +For `find` queries, the structure of this JSON DSL should look like: + +[source,js] +---- +{ + "find":{ + "filter": { + // find query goes here + }, + "options":{ + // query options go here + } + } +} + +---- +// NOTCONSOLE + +For example: + +[source,js] +---- +{ + "find": { + "filter": { + "$text": { + "$search": "garden", + "$caseSensitive": false + } + }, + "skip": 10, + "limit": 1000 + } +} +---- +// NOTCONSOLE + +`find` queries also support additional options, for example the `projection` object: + +[source,js] +---- +{ + "find": { + "filter": { + "languages": [ + "English" + ], + "runtime": { + "$gt":90 + } + }, + "projection":{ + "tomatoes": 1 + } + } +} +---- +// NOTCONSOLE +Where the available options are: + +* `allow_disk_use` (true, false) — When set to true, the server can write temporary data to disk while executing the find operation. This option is only available on MongoDB server versions 4.4 and newer. +* `allow_partial_results` (true, false) — Allows the query to get partial results if some shards are down. +* `batch_size` (Integer) — The number of documents returned in each batch of results from MongoDB. +* `filter` (Object) — The filter criteria for the query. +* `limit` (Integer) — The max number of docs to return from the query. +* `max_time_ms` (Integer) — The maximum amount of time to allow the query to run, in milliseconds. +* `no_cursor_timeout` (true, false) — The server normally times out idle cursors after an inactivity period (10 minutes) to prevent excess memory use. Set this option to prevent that. +* `projection` (Array, Object) — The fields to include or exclude from each doc in the result set. If an array, it should have at least one item. +* `return_key` (true, false) — Return index keys rather than the documents. +* `show_record_id` (true, false) — Return the `$recordId` for each doc in the result set. +* `skip` (Integer) — The number of docs to skip before returning results. + +[discrete#es-connectors-mongodb-client-sync-rules-aggregation] +====== Aggregation pipelines + +Similarly, for aggregation pipelines, the structure of the JSON DSL should look like: + +[source,js] +---- +{ + "aggregate":{ + "pipeline": [ + // pipeline elements go here + ], + "options": { + // pipeline options go here + } + } +} +---- +// NOTCONSOLE + +Where the available options are: + +* `allowDiskUse` (true, false) — Set to true if disk usage is allowed during the aggregation. +* `batchSize` (Integer) — The number of documents to return per batch. +* `bypassDocumentValidation` (true, false) — Whether or not to skip document level validation. +* `collation` (Object) — The collation to use. +* `comment` (String) — A user-provided comment to attach to this command. +* `hint` (String) — The index to use for the aggregation. +* `let` (Object) — Mapping of variables to use in the pipeline. See the server documentation for details. +* `maxTimeMs` (Integer) — The maximum amount of time in milliseconds to allow the aggregation to run. + +[discrete#es-connectors-mongodb-client-migration-from-ruby] +===== Migrating from the Ruby connector framework + +As part of the 8.8.0 release the MongoDB connector was moved from the {connectors-python}[Ruby connectors framework^] to the {connectors-python}[Elastic connector framework^]. + +This change introduces minor formatting modifications to data ingested from MongoDB: + +1. Nested object id field name has changed from "_id" to "id". For example, if you had a field "customer._id", this will now be named "customer.id". +2. Date format has changed from `YYYY-MM-DD'T'HH:mm:ss.fff'Z'` to `YYYY-MM-DD'T'HH:mm:ss` + +If your MongoDB connector stopped working after migrating from 8.7.x to 8.8.x, read the workaround outlined in <>. +If that does not work, we recommend deleting the search index attached to this connector and re-creating a MongoDB connector from scratch. + + +// Closing the collapsible section +=============== diff --git a/docs/reference/connector/docs/connectors-ms-sql.asciidoc b/docs/reference/connector/docs/connectors-ms-sql.asciidoc new file mode 100644 index 0000000000000..d706af8ca8043 --- /dev/null +++ b/docs/reference/connector/docs/connectors-ms-sql.asciidoc @@ -0,0 +1,594 @@ +[#es-connectors-ms-sql] +=== Elastic Microsoft SQL connector reference +++++ +Microsoft SQL +++++ +// Attributes used in this file: +:service-name: Microsoft SQL +:service-name-stub: mssql + +The _Elastic Microsoft SQL connector_ is a <> for https://learn.microsoft.com/en-us/sql/[Microsoft SQL^] databases. +This connector is written in Python using the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/{service-name-stub}.py[*source code* for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + +.Choose your connector reference +******************************* +Are you using a managed connector on Elastic Cloud or a self-managed connector? Expand the documentation based on your deployment method. +******************************* + +// //////// //// //// //// //// //// //// //////// +// //////// NATIVE CONNECTOR REFERENCE /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-ms-sql-native-connector-reference] +==== *Elastic managed connector reference* + +.View *Elastic managed connector* reference + +[%collapsible] +=============== + +[discrete#es-connectors-ms-sql-availability-prerequisites] +===== Availability and prerequisites + +This connector is available as a *managed connector* in Elastic versions *8.8.0 and later*. +To use this connector natively in Elastic Cloud, satisfy all <>. + +[discrete#es-connectors-{service_type}-create-native-connector] +===== Create a {service-name} connector +include::_connectors-create-native.asciidoc[] + +[discrete#es-connectors-ms-sql-usage] +===== Usage + +To use this connector as a *managed connector*, use the *Connector* workflow. +See <>. + +Users require the `sysadmin` SQL Server role. +Note that SQL Server Authentication is required. +Windows Authentication is not supported. + +For additional operations, see <>. + +[discrete#es-connectors-ms-sql-compatability] +===== Compatibility + +The following are compatible with Elastic connector frameworks: + +* Microsoft SQL Server versions 2017, 2019 +* Azure SQL +* Amazon RDS for SQL Server + +[discrete#es-connectors-ms-sql-configuration] +===== Configuration + +The following configuration fields are required to set up the connector: + +Host:: +The server host address where the Microsoft SQL Server is hosted. +Default value is `127.0.0.1`. +Examples: ++ +* `192.158.1.38` +* `demo.instance.demo-region.demo.service.com` + +Port:: +The port where the Microsoft SQL Server is hosted. Default value is `1433`. + +Username:: +The username of the account for Microsoft SQL Server (SQL Server Authentication only). + +Password:: +The password of the account to be used for the Microsoft SQL Server (SQL Server Authentication only). + +Database:: +Name of the Microsoft SQL Server database. +Examples: ++ +* `employee_database` +* `customer_database` + +Comma-separated list of tables:: +List of tables, separated by commas. +The Microsoft SQL connector will fetch data from all tables present in the configured database, if the value is `*` . +Default value is `*`. +Examples: ++ +* `table_1, table_2` +* `*` ++ +[WARNING] +==== +This field can be bypassed by advanced sync rules. +==== + +Schema:: +Name of the Microsoft SQL Server schema. +Default value is `dbo`. ++ +Examples: ++ +* `dbo` +* `custom_schema` + +Enable SSL:: +Toggle to enable SSL verification. +Default value is `False`. + +SSL certificate:: +Content of SSL certificate. +If SSL is disabled, the `ssl_ca` value will be ignored. ++ +.*Expand* to see an example certificate +[%collapsible] +==== +``` +-----BEGIN CERTIFICATE----- +MIID+jCCAuKgAwIBAgIGAJJMzlxLMA0GCSqGSIb3DQEBCwUAMHoxCzAJBgNVBAYT +... +7RhLQyWn2u00L7/9Omw= +-----END CERTIFICATE----- +``` +==== + +Validate host:: +Toggle to enable host validation. +Default value is `False`. + +[discrete#es-connectors-ms-sql-documents-syncs] +===== Documents and syncs + +* Tables with no primary key defined are skipped. +* If the `last_user_update` of `sys.dm_db_index_usage_stats` table is not available for a specific table and database then all data in that table will be synced. + +[NOTE] +==== +* Files bigger than 10 MB won't be extracted. +* Permissions are not synced. +**All documents** indexed to an Elastic deployment will be visible to **all users with access** to that Elastic Deployment. +==== + +[discrete#es-connectors-ms-sql-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. +For more information read <>. + +[discrete#es-connectors-ms-sql-sync-rules-advanced] +====== Advanced sync rules + +This connector supports <> for remote filtering. +These rules cover complex query-and-filter scenarios that cannot be expressed with basic sync rules. +Advanced sync rules are defined through a source-specific DSL JSON snippet. + +[NOTE] +==== +A <> is required for advanced sync rules to take effect. +==== + +Here are a few examples of advanced sync rules for this connector. + +.*Expand* to see example data +[%collapsible] +==== + +*`employee` table* + +[cols="3*", options="header"] +|=== +| emp_id | name | age +| 3 | John | 28 +| 10 | Jane | 35 +| 14 | Alex | 22 +|=== + +* +*`customer` table* + +[cols="3*", options="header"] +|=== +| c_id | name | age +| 2 | Elm | 24 +| 6 | Pine | 30 +| 9 | Oak | 34 +|=== +==== + +[discrete#es-connectors-ms-sql-sync-rules-advanced-queries] +*Example: Two queries* + +These rules fetch all records from both the `employee` and `customer` tables. The data from these tables will be synced separately to Elasticsearch. + +[source,js] +---- +[ + { + "tables": [ + "employee" + ], + "query": "SELECT * FROM employee" + }, + { + "tables": [ + "customer" + ], + "query": "SELECT * FROM customer" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-ms-sql-sync-rules-example-one-where] +*Example: One WHERE query* + +This rule fetches only the records from the `employee` table where the `emp_id` is greater than 5. Only these filtered records will be synced to Elasticsearch. + +[source,js] +---- +[ + { + "tables": ["employee"], + "query": "SELECT * FROM employee WHERE emp_id > 5" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-ms-sql-sync-rules-example-one-join] +*Example: One JOIN query* + +This rule fetches records by performing an INNER JOIN between the `employee` and `customer` tables on the condition that the `emp_id` in `employee` matches the `c_id` in `customer`. The result of this combined data will be synced to Elasticsearch. + +[source,js] +---- +[ + { + "tables": ["employee", "customer"], + "query": "SELECT * FROM employee INNER JOIN customer ON employee.emp_id = customer.c_id" + } +] +---- +// NOTCONSOLE + +[WARNING] +==== +When using advanced rules, a query can bypass the configuration field `tables`. +This will happen if the query specifies a table that doesn't appear in the configuration. +This can also happen if the configuration specifies `*` to fetch all tables while the advanced sync rule requests for only a subset of tables. +==== + +[discrete#es-connectors-ms-sql-known-issues] +===== Known issues + +There are no known issues for this connector. +See <> for any issues affecting all connectors. + +[discrete#es-connectors-ms-sql-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-ms-sql-security] +===== Security + +See <>. + + + + +This connector uses the https://github.com/elastic/connectors/blob/{branch}/connectors/sources/generic_database.py[generic database connector source code^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + +View {connectors-python}/connectors/sources/mssql.py[additional code specific to this data source^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + + +// Closing the collapsible section +=============== + + +// //////// //// //// //// //// //// //// //////// +// //////// CONNECTOR CLIENT REFERENCE /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-ms-sql-connector-client-reference] +==== *Self-managed connector* + +.View *self-managed connector* reference + +[%collapsible] +=============== + +[discrete#es-connectors-ms-sql-client-availability-prerequisites] +===== Availability and prerequisites + +This connector is available as a self-managed *self-managed connector*. +To use this connector, satisfy all <>. + +[discrete#es-connectors-{service_type}-create-connector-client] +===== Create a {service-name} connector +include::_connectors-create-client.asciidoc[] + +[discrete#es-connectors-ms-sql-client-usage] +===== Usage + +Users require the `sysadmin` server role. +Note that SQL Server Authentication is required. +Windows Authentication is not supported. + +To use this connector as a *self-managed connector*, see <> +For additional usage operations, see <>. + +[discrete#es-connectors-ms-sql-client-compatability] +===== Compatibility + +The following are compatible with Elastic connector frameworks: + +* Microsoft SQL Server versions 2017, 2019 +* Azure SQL +* Amazon RDS for SQL Server + +[discrete#es-connectors-ms-sql-client-configuration] +===== Configuration + +[TIP] +==== +When using the <> workflow, initially these fields will use the default configuration set in the {connectors-python}/connectors/sources/generic_database.py[connector source code^]. +Note that this data source uses the `generic_database.py` connector source code. + +Refer to {connectors-python}/connectors/sources/mssql.py[`mssql.py`^] for additional code, specific to this data source. +These configurable fields will be rendered with their respective *labels* in the Kibana UI. +Once connected, users will be able to update these values in Kibana. +==== + +The following configuration fields are required to set up the connector: + +`host`:: +The server host address where the Microsoft SQL Server is hosted. +Default value is `127.0.0.1`. +Examples: ++ +* `192.158.1.38` +* `demo.instance.demo-region.demo.service.com` + +`port`:: +The port where the Microsoft SQL Server is hosted. Default value is `9090`. + +`username`:: +The username of the account for Microsoft SQL Server. (SQL Server Authentication only) + +`password`:: +The password of the account to be used for the Microsoft SQL Server. (SQL Server Authentication only) + +`database`:: +Name of the Microsoft SQL Server database. +Examples: ++ +* `employee_database` +* `customer_database` + +`tables`:: +Comma-separated list of tables. +The Microsoft SQL connector will fetch data from all tables present in the configured database, if the value is `*` . +Default value is `*`. +Examples: ++ +* `table_1, table_2` +* `*` ++ +[WARNING] +==== +This field can be bypassed by advanced sync rules. +==== + +`fetch_size`:: +Rows fetched per request. + +`retry_count`:: +The number of retry attempts per failed request. + +`schema`:: +Name of the Microsoft SQL Server schema. +Default value is `dbo`. ++ +Examples: ++ +* `dbo` +* `custom_schema` + +`ssl_enabled`:: +SSL verification enablement. +Default value is `False`. + +`ssl_ca`:: +Content of SSL certificate. +If SSL is disabled, the `ssl_ca` value will be ignored. ++ +.*Expand* to see an example certificate +[%collapsible] +==== +``` +-----BEGIN CERTIFICATE----- +MIID+jCCAuKgAwIBAgIGAJJMzlxLMA0GCSqGSIb3DQEBCwUAMHoxCzAJBgNVBAYT +... +7RhLQyWn2u00L7/9Omw= +-----END CERTIFICATE----- +``` +==== + +`validate_host`:: +Host validation enablement. +Default value is `False`. + +[discrete#es-connectors-ms-sql-client-docker] +===== Deployment using Docker + +include::_connectors-docker-instructions.asciidoc[] + +[discrete#es-connectors-ms-sql-client-documents-syncs] +===== Documents and syncs + +* Tables with no primary key defined are skipped. +* If the `last_user_update` of `sys.dm_db_index_usage_stats` table is not available for a specific table and database then all data in that table will be synced. + +[NOTE] +==== +* Files bigger than 10 MB won't be extracted. +* Permissions are not synced. +**All documents** indexed to an Elastic deployment will be visible to **all users with access** to that Elastic Deployment. +==== + +[discrete#es-connectors-ms-sql-client-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. +For more information read <>. + +[discrete#es-connectors-ms-sql-client-sync-rules-advanced] +====== Advanced sync rules + +This connector supports <> for remote filtering. +These rules cover complex query-and-filter scenarios that cannot be expressed with basic sync rules. +Advanced sync rules are defined through a source-specific DSL JSON snippet. + +[NOTE] +==== +A <> is required for advanced sync rules to take effect. +==== + +Here are a few examples of advanced sync rules for this connector. + +.*Expand* to see example data +[%collapsible] +==== + +*`employee` table* + +[cols="3*", options="header"] +|=== +| emp_id | name | age +| 3 | John | 28 +| 10 | Jane | 35 +| 14 | Alex | 22 +|=== + +* +*`customer` table* + +[cols="3*", options="header"] +|=== +| c_id | name | age +| 2 | Elm | 24 +| 6 | Pine | 30 +| 9 | Oak | 34 +|=== +==== + +[discrete#es-connectors-ms-sql-client-sync-rules-advanced-queries] +*Example: Two queries* + +These rules fetch all records from both the `employee` and `customer` tables. The data from these tables will be synced separately to Elasticsearch. + +[source,js] +---- +[ + { + "tables": [ + "employee" + ], + "query": "SELECT * FROM employee" + }, + { + "tables": [ + "customer" + ], + "query": "SELECT * FROM customer" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-ms-sql-client-sync-rules-example-one-where] +*Example: One WHERE query* + +This rule fetches only the records from the `employee` table where the `emp_id` is greater than 5. Only these filtered records will be synced to Elasticsearch. + +[source,js] +---- +[ + { + "tables": ["employee"], + "query": "SELECT * FROM employee WHERE emp_id > 5" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-ms-sql-client-sync-rules-example-one-join] +*Example: One JOIN query* + +This rule fetches records by performing an INNER JOIN between the `employee` and `customer` tables on the condition that the `emp_id` in `employee` matches the `c_id` in `customer`. The result of this combined data will be synced to Elasticsearch. + +[source,js] +---- +[ + { + "tables": ["employee", "customer"], + "query": "SELECT * FROM employee INNER JOIN customer ON employee.emp_id = customer.c_id" + } +] +---- +// NOTCONSOLE + +[WARNING] +==== +When using advanced rules, a query can bypass the configuration field `tables`. +This will happen if the query specifies a table that doesn't appear in the configuration. +This can also happen if the configuration specifies `*` to fetch all tables while the advanced sync rule requests for only a subset of tables. +==== + +[discrete#es-connectors-ms-sql-client-client-operations-testing] +===== End-to-end testing + +The connector framework enables operators to run functional tests against a real data source. +Refer to <> for more details. + +To perform E2E testing for the Microsoft SQL connector, run the following command: + +[source,shell] +---- +make ftest NAME=mssql +---- + +For faster tests, add the `DATA_SIZE=small` flag: + +[source,shell] +---- +make ftest NAME=mssql DATA_SIZE=small +---- + +[discrete#es-connectors-ms-sql-client-known-issues] +===== Known issues + +There are no known issues for this connector. +See <> for any issues affecting all connectors. + +[discrete#es-connectors-ms-sql-client-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-ms-sql-client-security] +===== Security + +See <>. + + + + +This connector uses the https://github.com/elastic/connectors-python/blob/{branch}/connectors/sources/generic_database.py[generic database connector source code^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + +View {connectors-python}/connectors/sources/mssql.py[additional code specific to this data source^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + + +// Closing the collapsible section +=============== diff --git a/docs/reference/connector/docs/connectors-mysql.asciidoc b/docs/reference/connector/docs/connectors-mysql.asciidoc new file mode 100644 index 0000000000000..59a9724f1ffe4 --- /dev/null +++ b/docs/reference/connector/docs/connectors-mysql.asciidoc @@ -0,0 +1,538 @@ +[#es-connectors-mysql] +=== Elastic MySQL connector reference +++++ +MySQL +++++ +// Attributes used in this file: +:service-name: MySQL +:service-name-stub: mysql + +The _Elastic MySQL connector_ is a <> for https://www.mysql.com[MySQL^] data sources. +This connector is written in Python using the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/{service-name-stub}.py[*source code* for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + +.Choose your connector reference +******************************* +Are you using a managed connector on Elastic Cloud or a self-managed connector? Expand the documentation based on your deployment method. +******************************* + +// //////// //// //// //// //// //// //// //////// +// //////// NATIVE CONNECTOR REFERENCE /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-mysql-native-connector-reference] +==== *Elastic managed connector reference* + +.View *Elastic managed connector* reference + +[%collapsible] +=============== + +[discrete#es-connectors-mysql-prerequisites] +===== Availability and prerequisites + +This connector is available as a *managed connector* in Elastic versions *8.5.0 and later*. +To use this connector natively in Elastic Cloud, satisfy all <>. + +This connector has no additional prerequisites beyond the shared requirements, linked above. + +[discrete#es-connectors-mysql-compatibility] +===== Compatibility + +This connector is compatible with *MySQL 5.6 and later*. + +The connector is also compatible with *MariaDB* databases compatible with the above. + +The data source and your Elastic deployment must be able to communicate with each other over a network. + +[discrete#es-connectors-mysql-create-native-connector] +===== Create a {service-name} connector +include::_connectors-create-native.asciidoc[] + +[discrete#es-connectors-mysql-usage] +===== Usage + +To use this connector natively in Elastic Cloud, see <>. + +For additional operations, see <>. + +[discrete#es-connectors-mysql-configuration] +===== Configuration + +Each time you create an index to be managed by this connector, you will create a new connector configuration. +You will need some or all of the following information about the data source. + +Host:: +The IP address or domain name of the MySQL host, excluding port. +Examples: ++ +* `192.158.1.38` +* `localhost` + +Port:: +The port of the MySQL host. +Examples: ++ +* `3306` +* `3307` + +Username:: +The MySQL username the connector will use. ++ +The user must have access to the configured database. +You may want to create a dedicated, read-only user for each connector. + +Password:: +The MySQL password the connector will use. + +Database:: +The MySQL database to sync. +The database must be accessible using the configured username and password. ++ +Examples: ++ +* `products` +* `orders` + +Comma-separated list of tables:: +The tables in the configured database to sync. +One or more table names, separated by commas. +The tables must be accessible using the configured username and password. ++ +Examples: ++ +* `furniture, food, toys` +* `laptops` ++ +[TIP] +==== +This field can be bypassed when using advanced sync rules. +==== + +Enable SSL:: +Whether SSL verification will be enabled. +Default value is `True`. + +SSL Certificate:: +Content of SSL certificate. +If SSL is disabled, the SSL certificate value will be ignored. ++ +.*Expand* to see an example certificate +[%collapsible] +==== +``` +-----BEGIN CERTIFICATE----- +MIID+jCCAuKgAwIBAgIGAJJMzlxLMA0GCSqGSIb3DQEBCwUAMHoxCzAJBgNVBAYT +AlVTMQwwCgYDVQQKEwNJQk0xFjAUBgNVBAsTDURlZmF1bHROb2RlMDExFjAUBgNV +BAsTDURlZmF1bHRDZWxsMDExGTAXBgNVBAsTEFJvb3QgQ2VydGlmaWNhdGUxEjAQ +BgNVBAMTCWxvY2FsaG9zdDAeFw0yMTEyMTQyMjA3MTZaFw0yMjEyMTQyMjA3MTZa +MF8xCzAJBgNVBAYTAlVTMQwwCgYDVQQKEwNJQk0xFjAUBgNVBAsTDURlZmF1bHRO +b2RlMDExFjAUBgNVBAsTDURlZmF1bHRDZWxsMDExEjAQBgNVBAMTCWxvY2FsaG9z +dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMv5HCsJZIpI5zCy+jXV +z6lmzNc9UcVSEEHn86h6zT6pxuY90TYeAhlZ9hZ+SCKn4OQ4GoDRZhLPTkYDt+wW +CV3NTIy9uCGUSJ6xjCKoxClJmgSQdg5m4HzwfY4ofoEZ5iZQ0Zmt62jGRWc0zuxj +hegnM+eO2reBJYu6Ypa9RPJdYJsmn1RNnC74IDY8Y95qn+WZj//UALCpYfX41hko +i7TWD9GKQO8SBmAxhjCDifOxVBokoxYrNdzESl0LXvnzEadeZTd9BfUtTaBHhx6t +njqqCPrbTY+3jAbZFd4RiERPnhLVKMytw5ot506BhPrUtpr2lusbN5svNXjuLeea +MMUCAwEAAaOBoDCBnTATBgNVHSMEDDAKgAhOatpLwvJFqjAdBgNVHSUEFjAUBggr +BgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4E+UHJvZmlsZVVVSUQ6QXBwU3J2 +MDEtQkFTRS05MDkzMzJjMC1iNmFiLTQ2OTMtYWI5NC01Mjc1ZDI1MmFmNDiCCWxv +Y2FsaG9zdDARBgNVHQ4ECgQITzqhA5sO8O4wDQYJKoZIhvcNAQELBQADggEBAKR0 +gY/BM69S6BDyWp5dxcpmZ9FS783FBbdUXjVtTkQno+oYURDrhCdsfTLYtqUlP4J4 +CHoskP+MwJjRIoKhPVQMv14Q4VC2J9coYXnePhFjE+6MaZbTjq9WaekGrpKkMaQA +iQt5b67jo7y63CZKIo9yBvs7sxODQzDn3wZwyux2vPegXSaTHR/rop/s/mPk3YTS +hQprs/IVtPoWU4/TsDN3gIlrAYGbcs29CAt5q9MfzkMmKsuDkTZD0ry42VjxjAmk +xw23l/k8RoD1wRWaDVbgpjwSzt+kl+vJE/ip2w3h69eEZ9wbo6scRO5lCO2JM4Pr +7RhLQyWn2u00L7/9Omw= +-----END CERTIFICATE----- +``` +==== + +[discrete#es-connectors-mysql-known-issues] +===== Known issues + +This connector has the following known issues: + +* *Upgrading from a tech preview connector (8.7 or earlier) to 8.8 will cause the MySQL connector configuration to be invalid.* ++ +MySQL connectors prior to 8.8 can be missing some configuration fields that are required for the connector to run. +If you would like to continue using your MySQL connector after upgrading from 8.7 or earlier, run the script below to fix your connector's configuration. +This will populate the configuration with the missing fields. +The auxilliary information needed for the configuration will then be automatically added by by the self-managed connector. ++ +[source,console] +---- +POST /.elastic-connectors/_update/connector_id +{ + "doc" : { + "configuration": { + "tables": { + "type": "list", + "value": "*" + }, + "ssl_enabled": { + "type": "bool", + "value": false + }, + "ssl_ca": { + "type": "str", + "value": "" + }, + "fetch_size": { + "type": "int", + "value": 50 + }, + "retry_count": { + "type": "int", + "value": 3 + } + } + } +} +---- +// TEST[skip:TODO] ++ +* *Upgrading to 8.8 does not migrate MySQL sync rules.* ++ +After upgrading, you must re-create your sync rules. + +See <> for any issues affecting all connectors. + +[discrete#es-connectors-mysql-syncs] +===== Documents and syncs + +The following describes the default syncing behavior for this connector. +Use <> and {ref}/ingest-pipeline-search.html[ingest pipelines] to customize syncing for specific indices. + +All records in the MySQL database included in your connector configuration are extracted and transformed into documents in your Elasticsearch index. + +* For each row in your MySQL database table, the connector creates one *Elasticsearch document*. +* For each column, the connector transforms the column into an *Elasticsearch field*. +* Elasticsearch {ref}/dynamic-mapping.html[dynamically maps^] MySQL data types to *Elasticsearch data types*. +* Tables with no primary key defined are skipped. +* Field values that represent other records are replaced with the primary key for that record (composite primary keys are joined with `_`). + +The Elasticsearch mapping is created when the first document is created. + +Each sync is a "full" sync. + +For each MySQL row discovered: + +* If it does not exist, the document is created in Elasticsearch. +* If it already exists in Elasticsearch, the Elasticsearch document is replaced and the version is incremented. +* If an existing Elasticsearch document no longer exists in the MySQL table, it is deleted from Elasticsearch. + +[NOTE] +==== +* Files bigger than 10 MB won't be extracted +* Permissions are not synced by default. +*All documents* indexed to an Elastic deployment will be visible to *all users with access* to that Elastic Deployment. +==== + +[discrete#es-connectors-mysql-sync-rules] +===== Sync rules + +The following sections describe <> for this connector. + +<> are identical for all connectors and are available by default. + +<> for MySQL can be used to pass arbitrary SQL statements to a MySQL instance. + +[IMPORTANT] +==== +You need to specify the tables used in your custom query in the "tables" field. +==== + +For example: + +[source,js] +---- +[ + { + "tables": ["table1", "table2"], + "query": "SELECT ... FROM ..." + } +] +---- +// NOTCONSOLE + +[WARNING] +==== +When using advanced rules, a query can bypass the configuration field `tables`. +This will happen if the query specifies a table that doesn't appear in the configuration. +This can also happen if the configuration specifies `*` to fetch all tables while the advanced sync rule requests for only a subset of tables. +==== + +[discrete#es-connectors-mysql-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-mysql-security] +===== Security + +See <>. + +// Closing the collapsible section +=============== + + +// //////// //// //// //// //// //// //// //////// +// //////// CONNECTOR CLIENT REFERENCE /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-mysql-connector-client-reference] +==== *Self-managed connector* + +.View *self-managed connector* reference + +[%collapsible] +=============== + +[discrete#es-connectors-mysql-client-prerequisites] +===== Availability and prerequisites + +This connector is available as a *managed connector* in Elastic versions *8.5.0 and later*. +To use this connector natively in Elastic Cloud, satisfy all <>. + +This connector is also available as a *self-managed connector* from the *Elastic connector framework*. +To use this connector as a self-managed connector, satisfy all <>. + +This connector has no additional prerequisites beyond the shared requirements, linked above. + +[discrete#es-connectors-mysql-create-connector-client] +===== Create a {service-name} connector +include::_connectors-create-client.asciidoc[] + +[discrete#es-connectors-mysql-client-usage] +===== Usage + +To use this connector as a *managed connector*, use the *Connector* workflow. +See <>. + +To use this connector as a *self-managed connector*, see <>. + +For additional operations, see <>. + +[discrete#es-connectors-mysql-client-compatibility] +===== Compatibility + +This connector is compatible with *MySQL 5.6 and later*. + +The connector is also compatible with *MariaDB* databases compatible with the above. + +The data source and your Elastic deployment must be able to communicate with each other over a network. + +[discrete#es-connectors-mysql-client-configuration] +===== Configuration + +Each time you create an index to be managed by this connector, you will create a new connector configuration. +You will need some or all of the following information about the data source. + +Host:: +The IP address or domain name of the MySQL host, excluding port. +Examples: ++ +* `192.158.1.38` +* `localhost` + +Port:: +The port of the MySQL host. +Examples: ++ +* `3306` +* `3307` + +Username:: +The MySQL username the connector will use. ++ +The user must have access to the configured database. +You may want to create a dedicated, read-only user for each connector. + +Password:: +The MySQL password the connector will use. + +Database:: +The MySQL database to sync. +The database must be accessible using the configured username and password. ++ +Examples: ++ +* `products` +* `orders` + +Tables:: +The tables in the configured database to sync. +One or more table names, separated by commas. +The tables must be accessible using the configured username and password. ++ +Examples: ++ +* `furniture, food, toys` +* `laptops` + +Enable SSL:: +Whether SSL verification will be enabled. +Default value is `True`. + +SSL Certificate:: +Content of SSL certificate. +If SSL is disabled, the SSL certificate value will be ignored. ++ +.*Expand* to see an example certificate +[%collapsible] +==== +``` +-----BEGIN CERTIFICATE----- +MIID+jCCAuKgAwIBAgIGAJJMzlxLMA0GCSqGSIb3DQEBCwUAMHoxCzAJBgNVBAYT +AlVTMQwwCgYDVQQKEwNJQk0xFjAUBgNVBAsTDURlZmF1bHROb2RlMDExFjAUBgNV +BAsTDURlZmF1bHRDZWxsMDExGTAXBgNVBAsTEFJvb3QgQ2VydGlmaWNhdGUxEjAQ +BgNVBAMTCWxvY2FsaG9zdDAeFw0yMTEyMTQyMjA3MTZaFw0yMjEyMTQyMjA3MTZa +MF8xCzAJBgNVBAYTAlVTMQwwCgYDVQQKEwNJQk0xFjAUBgNVBAsTDURlZmF1bHRO +b2RlMDExFjAUBgNVBAsTDURlZmF1bHRDZWxsMDExEjAQBgNVBAMTCWxvY2FsaG9z +dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMv5HCsJZIpI5zCy+jXV +z6lmzNc9UcVSEEHn86h6zT6pxuY90TYeAhlZ9hZ+SCKn4OQ4GoDRZhLPTkYDt+wW +CV3NTIy9uCGUSJ6xjCKoxClJmgSQdg5m4HzwfY4ofoEZ5iZQ0Zmt62jGRWc0zuxj +hegnM+eO2reBJYu6Ypa9RPJdYJsmn1RNnC74IDY8Y95qn+WZj//UALCpYfX41hko +i7TWD9GKQO8SBmAxhjCDifOxVBokoxYrNdzESl0LXvnzEadeZTd9BfUtTaBHhx6t +njqqCPrbTY+3jAbZFd4RiERPnhLVKMytw5ot506BhPrUtpr2lusbN5svNXjuLeea +MMUCAwEAAaOBoDCBnTATBgNVHSMEDDAKgAhOatpLwvJFqjAdBgNVHSUEFjAUBggr +BgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4E+UHJvZmlsZVVVSUQ6QXBwU3J2 +MDEtQkFTRS05MDkzMzJjMC1iNmFiLTQ2OTMtYWI5NC01Mjc1ZDI1MmFmNDiCCWxv +Y2FsaG9zdDARBgNVHQ4ECgQITzqhA5sO8O4wDQYJKoZIhvcNAQELBQADggEBAKR0 +gY/BM69S6BDyWp5dxcpmZ9FS783FBbdUXjVtTkQno+oYURDrhCdsfTLYtqUlP4J4 +CHoskP+MwJjRIoKhPVQMv14Q4VC2J9coYXnePhFjE+6MaZbTjq9WaekGrpKkMaQA +iQt5b67jo7y63CZKIo9yBvs7sxODQzDn3wZwyux2vPegXSaTHR/rop/s/mPk3YTS +hQprs/IVtPoWU4/TsDN3gIlrAYGbcs29CAt5q9MfzkMmKsuDkTZD0ry42VjxjAmk +xw23l/k8RoD1wRWaDVbgpjwSzt+kl+vJE/ip2w3h69eEZ9wbo6scRO5lCO2JM4Pr +7RhLQyWn2u00L7/9Omw= +-----END CERTIFICATE----- +``` +==== + +[discrete#es-connectors-mysql-client-known-issues] +===== Known issues + +This connector has the following known issues: + +* *Upgrading from a tech preview connector (8.7 or earlier) to 8.8 will cause the MySQL connector configuration to be invalid.* ++ +MySQL connectors prior to 8.8 can be missing some configuration fields that are required for the connector to run. +If you would like to continue using your MySQL connector after upgrading from 8.7 or earlier, run the script below to fix your connector's configuration. +This will populate the configuration with the missing fields. +The auxilliary information needed for the configuration will then be automatically added by by the self-managed connector. ++ +[source,console] +---- +POST /.elastic-connectors/_update/connector_id +{ + "doc" : { + "configuration": { + "tables": { + "type": "list", + "value": "*" + }, + "ssl_enabled": { + "type": "bool", + "value": false + }, + "ssl_ca": { + "type": "str", + "value": "" + }, + "fetch_size": { + "type": "int", + "value": 50 + }, + "retry_count": { + "type": "int", + "value": 3 + } + } + } +} +---- +// TEST[skip:TODO] ++ +* *Upgrading to 8.8 does not migrate MySQL sync rules.* ++ +After upgrading, you must re-create your sync rules. + +See <> for any issues affecting all connectors. + +[discrete#es-connectors-mysql-client-syncs] +===== Documents and syncs + +The following describes the default syncing behavior for this connector. +Use <> and {ref}/ingest-pipeline-search.html[ingest pipelines] to customize syncing for specific indices. + +All records in the MySQL database included in your connector configuration are extracted and transformed into documents in your Elasticsearch index. + +* For each row in your MySQL database table, the connector creates one *Elasticsearch document*. +* For each column, the connector transforms the column into an *Elasticsearch field*. +* Elasticsearch {ref}/dynamic-mapping.html[dynamically maps^] MySQL data types to *Elasticsearch data types*. +* Tables with no primary key defined are skipped. +* Field values that represent other records are replaced with the primary key for that record (composite primary keys are joined with `_`). + +The Elasticsearch mapping is created when the first document is created. + +Each sync is a "full" sync. + +For each MySQL row discovered: + +* If it does not exist, the document is created in Elasticsearch. +* If it already exists in Elasticsearch, the Elasticsearch document is replaced and the version is incremented. +* If an existing Elasticsearch document no longer exists in the MySQL table, it is deleted from Elasticsearch. + +[discrete#es-connectors-mysql-client-docker] +===== Deployment using Docker + +include::_connectors-docker-instructions.asciidoc[] + +[discrete#es-connectors-mysql-client-sync-rules] +===== Sync rules + +The following sections describe <> for this connector. + +<> are identical for all connectors and are available by default. + +<> for MySQL can be used to pass arbitrary SQL statements to a MySQL instance. + +[IMPORTANT] +==== +You need to specify the tables used in your custom query in the "tables" field. +==== + +For example: + +[source,js] +---- +[ + { + "tables": ["table1", "table2"], + "query": "SELECT ... FROM ..." + } +] +---- +// NOTCONSOLE + +[WARNING] +==== +When using advanced rules, a query can bypass the configuration field `tables`. +This will happen if the query specifies a table that doesn't appear in the configuration. +This can also happen if the configuration specifies `*` to fetch all tables while the advanced sync rule requests for only a subset of tables. +==== + +[discrete#es-connectors-mysql-client-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-mysql-client-security] +===== Security + +See <>. + +// Closing the collapsible section +=============== diff --git a/docs/reference/connector/docs/connectors-network-drive.asciidoc b/docs/reference/connector/docs/connectors-network-drive.asciidoc new file mode 100644 index 0000000000000..909e3440c9f02 --- /dev/null +++ b/docs/reference/connector/docs/connectors-network-drive.asciidoc @@ -0,0 +1,533 @@ +[#es-connectors-network-drive] +=== Elastic network drive connector reference +++++ +Network drive +++++ + +// Attributes used in this file: +:service-name: Network drive +:service-name-stub: network_drive + +The _Elastic network drive connector_ is a <> for network drive data sources. +This connector is written in Python using the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/{service-name-stub}.py[*source code* for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + +.Choose your connector reference +******************************* +Are you using a managed connector on Elastic Cloud or a self-managed connector? Expand the documentation based on your deployment method. +******************************* + +// //////// //// //// //// //// //// //// //////// +// //////// NATIVE CONNECTOR REFERENCE /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-network-drive-native-connector-reference] +==== *Elastic managed connector reference* + +.View *Elastic managed connector* reference + +[%collapsible] +=============== + +[discrete#es-connectors-network-drive-prerequisites] +===== Availability and prerequisites + +This connector is available as a *managed connector* on Elastic Cloud, as of *8.9.1*. + +To use this connector natively in Elastic Cloud, satisfy all <>. + +[discrete#es-connectors-network-drive-usage] +===== Usage + +To use this connector natively in Elastic Cloud, see <>. + +[discrete#es-connectors-network-drive-configuration] +===== Configuration + +The following configuration fields are required to set up the connector: + +Username:: +The username of the account for the network drive. +The user must have at least **read** permissions for the folder path provided. + +Password:: +The password of the account to be used for crawling the network drive. + +IP address:: +The server IP address where the network drive is hosted. +Default value is `127.0.0.1`. + +Port:: +The server port where the network drive service is available. +Default value is `445`. + +Path:: +** The network drive path the connector will crawl to fetch files. +This is the name of the folder shared via SMB. +The connector uses the Python https://github.com/jborean93/smbprotocol[`smbprotocol`^] library which supports both *SMB v2* and *v3*. +** Accepts only one path— parent folders can be specified to widen the scope. +** The drive path should use *forward slashes* as path separators. +Example: ++ +* `admin/bin` + +Enable document level security:: +Toggle to enable document level security (DLS). When enabled: +* Full syncs will fetch access control lists for each document and store them in the `_allow_access_control` field. +* Access control syncs will fetch users' access control lists and store them in a separate index. +Default value is `false`. ++ +[TIP] +==== +Refer to <> for more information, including prerequisites and limitations. +==== + +Identity mappings:: +Path to a CSV file containing user and group SIDs (For Linux Network Drive). ++ +File should be formatted as follows: ++ +* Fields separated by semicolons (`;`) +* Three fields per line: `Username;User-SID;Group-SIDs` +* Group-SIDs are comma-separated and optional. ++ +*Example* with one username, user-sid and no group: ++ +[source,text] +---- +user1;S-1; +---- ++ +*Example* with one username, user-sid and two groups: ++ +[source,text] +---- +user1;S-1;S-11,S-22 +---- + +[discrete#es-connectors-network-drive-documents-syncs] +===== Documents and syncs + +The connector syncs folders as separate documents in Elasticsearch. +The following fields will be added for the document type `folder`: + +* `create_time` +* `title` +* `path` +* `modified` +* `time` +* `id` + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted. (Self-managed connectors can use the <> to handle larger binary files.) +* Permissions are not synced by default. +You must first enable <>. +Otherwise, *all documents* indexed to an Elastic deployment will be visible to *all users with access* to that Elastic Deployment. +==== + +[discrete#es-connectors-network-drive-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-network-drive-dls] +===== Document level security + +Document Level Security (DLS) enables you to restrict access to documents based on a user's permissions. +DLS facilitates the syncing of folder and file permissions, including both user and group level permissions. + +[NOTE] +==== +**Note:** Refer to <> to learn how to search data with DLS enabled, when building a search application. +==== + +[discrete#es-connectors-network-drive-dls-availability] +====== Availability + +* The present version of Network Drive connector offers DLS support for Windows network drives only. +* To fetch users and groups in a Windows network drive, account credentials added in the connector configuration should have access to the Powershell of the Windows Server where the network drive is hosted. + +[discrete#es-connectors-network-drive-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +[discrete#es-connectors-network-drive-sync-rules-advanced] +====== Advanced sync rules + +[NOTE] +==== +A <> is required for advanced sync rules to take effect. +==== + +Advanced sync rules are defined through a source-specific DSL JSON snippet. +Advanced sync rules for this connector use *glob patterns*. + +1. Each rule must contains a glob pattern. This pattern is then matched against all the available folder paths inside the configured drive path. +2. The pattern must begin with the `drive_path` field configured in the connector. +3. If the pattern matches any available folder paths, the contents directly within those folders will be fetched. + +The following sections provide examples of advanced sync rules for this connector. + +[discrete#es-connectors-network-drive-indexing-files-and-folders-recursively-within-folders] +*Indexing files and folders recursively within folders* + +[source,js] +---- +[ + { + "pattern": "Folder-shared/a/mock/**" + }, + { + "pattern": "Folder-shared/b/alpha/**" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-network-drive-indexing-files-and-folders-directly-inside-folder] +*Indexing files and folders directly inside folder* + +[source,js] +---- +[ + { + "pattern": "Folder-shared/a/b/test" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-network-drive-indexing-files-and-folders-directly-inside-a-set-of-folders] +*Indexing files and folders directly inside a set of folders* + +[source,js] +---- +[ + { + "pattern": "Folder-shared/org/*/all-tests/test[135]" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-network-drive-excluding-files-and-folders-that-match-a-pattern] +*Excluding files and folders that match a pattern* + +[source,js] +---- +[ + { + "pattern": "Folder-shared/**/all-tests/test[!7]" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-network-drive-content-extraction] +===== Content extraction + +See <>. + +[discrete#es-connectors-network-drive-known-issues] +===== Known issues + +There are no known issues for this connector. + +See <> for any issues affecting all connectors. + +[discrete#es-connectors-network-drive-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-network-drive-security] +===== Security + +See <>. + +// Closing the collapsible section +=============== + + +// //////// //// //// //// //// //// //// //////// +// //////// CONNECTOR CLIENT REFERENCE /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-network-drive-connector-client-reference] +==== *Self-managed connector* + +.View *self-managed connector* reference + +[%collapsible] +=============== + +[discrete#es-connectors-network-drive-client-prerequisites] +===== Availability and prerequisites + +This connector is available as a self-managed *self-managed connector*. +This self-managed connector is compatible with Elastic versions *8.6.0+*. + +To use this connector, satisfy all <>. + +[discrete#es-connectors-network-drive-client-usage] +===== Usage + +To use this connector as a *self-managed connector*, see <>For additional usage operations, see <>. + +[discrete#es-connectors-network-drive-client-configuration] +===== Configuration + +[TIP] +==== +When using the <> workflow, initially these fields will use the default configuration set in the {connectors-python}/connectors/sources/network_drive.py[connector source code^]. +These are set in the `get_default_configuration` function definition. + +These configurable fields will be rendered with their respective *labels* in the Kibana UI. +Once connected, you'll be able to update these values in Kibana. +==== + +The following configuration fields are required to set up the connector: + +`username`:: +The username of the account for the network drive. +The user must have at least **read** permissions for the folder path provided. + +`password`:: +The password of the account to be used for crawling the network drive. + +`server_ip`:: +The server IP address where the network drive is hosted. +Default value is `127.0.0.1`. + +`server_port`:: +The server port where the network drive service is available. +Default value is `445`. + +`drive_path`:: +** The network drive path the connector will crawl to fetch files. +This is the name of the folder shared via SMB. +The connector uses the Python https://github.com/jborean93/smbprotocol[`smbprotocol`^] library which supports both *SMB v2* and *v3*. +** Accepts only one path— parent folders can be specified to widen the scope. +** The drive path should use *forward slashes* as path separators. +Example: ++ +* `admin/bin` + +`use_document_level_security`:: +Toggle to enable document level security (DLS). When enabled: +* Full syncs will fetch access control lists for each document and store them in the `_allow_access_control` field. +* Access control syncs will fetch users' access control lists and store them in a separate index. ++ +[TIP] +==== +Refer to <> for more information, including prerequisites and limitations. +==== + +`drive_type`:: +The type of network drive to be crawled. +The following options are available: ++ +* `Windows` +* `Linux` + +`identity_mappings`:: +Path to a CSV file containing user and group SIDs (For Linux Network Drive). ++ +File should be formatted as follows: ++ +* Fields separated by semicolons (`;`) +* Three fields per line: `Username;User-SID;Group-SIDs` +* Group-SIDs are comma-separated and optional. ++ +*Example* with one username, user-sid and no group: ++ +[source,text] +---- +user1;S-1; +---- ++ +*Example* with one username, user-sid and two groups: ++ +[source,text] +---- +user1;S-1;S-11,S-22 +---- + +[discrete#es-connectors-network-drive-client-docker] +===== Deployment using Docker + +include::_connectors-docker-instructions.asciidoc[] + +[discrete#es-connectors-network-drive-client-documents-syncs] +===== Documents and syncs + +The connector syncs folders as separate documents in Elasticsearch. +The following fields will be added for the document type `folder`: + +* `create_time` +* `title` +* `path` +* `modified` +* `time` +* `id` + + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted +* Permissions are not synced by default. +You must first enable <>. +Otherwise, *all documents* indexed to an Elastic deployment will be visible to *all users with access* to that Elastic Deployment. +==== + +[discrete#es-connectors-network-drive-client-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-network-drive-client-dls] +===== Document level security + +Document Level Security (DLS) enables you to restrict access to documents based on a user's permissions. +DLS facilitates the syncing of folder and file permissions, including both user and group level permissions. + +[NOTE] +==== +**Note:** Refer to <> to learn how to search data with DLS enabled, when building a search application. +==== + +[discrete#es-connectors-network-drive-client-dls-availability] +====== Availability + +* The Network Drive self-managed connector offers DLS support for both Windows and Linux network drives. +* To fetch users and groups in a Windows network drive, account credentials added in the connector configuration should have access to the Powershell of the Windows Server where the network drive is hosted. + +[discrete#es-connectors-network-drive-client-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +[discrete#es-connectors-network-drive-client-sync-rules-advanced] +====== Advanced sync rules + +[NOTE] +==== +A <> is required for advanced sync rules to take effect. +==== + +Advanced sync rules are defined through a source-specific DSL JSON snippet. +Advanced sync rules for this connector use *glob patterns*. + +1. Each rule must contains a glob pattern. This pattern is then matched against all the available folder paths inside the configured drive path. +2. The pattern must begin with the `drive_path` field configured in the connector. +3. If the pattern matches any available folder paths, the contents directly within those folders will be fetched. + +The following sections provide examples of advanced sync rules for this connector. + +[discrete#es-connectors-network-drive-client-indexing-files-and-folders-recursively-within-folders] +*Indexing files and folders recursively within folders* + +[source,js] +---- +[ + { + "pattern": "Folder-shared/a/mock/**" + }, + { + "pattern": "Folder-shared/b/alpha/**" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-network-drive-client-indexing-files-and-folders-directly-inside-folder] +*Indexing files and folders directly inside folder* + +[source,js] +---- +[ + { + "pattern": "Folder-shared/a/b/test" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-network-drive-client-indexing-files-and-folders-directly-inside-a-set-of-folders] +*Indexing files and folders directly inside a set of folders* + +[source,js] +---- +[ + { + "pattern": "Folder-shared/org/*/all-tests/test[135]" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-network-drive-client-excluding-files-and-folders-that-match-a-pattern] +*Excluding files and folders that match a pattern* + +[source,js] +---- +[ + { + "pattern": "Folder-shared/**/all-tests/test[!7]" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-network-drive-client-content-extraction] +===== Content extraction + +See <>. + +[discrete#es-connectors-network-drive-client-tests] +===== End-to-end tests + +The connector framework enables operators to run functional tests against a real data source. +Refer to <> for more details. + +To execute a functional test for the Network Drive self-managed connector, run the following command: + +[source,shell] +---- +$ make ftest NAME=network_drive +---- + +By default, this will use a medium-sized dataset. +For faster tests add the `DATA_SIZE=small` flag: + +[source,shell] +---- +make ftest NAME=network_drive DATA_SIZE=small +---- + +[discrete#es-connectors-network-drive-client-known-issues] +===== Known issues + +There are no known issues for this connector. + +See <> for any issues affecting all connectors. + +[discrete#es-connectors-network-drive-client-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-network-drive-client-security] +===== Security + +See <>. + +// Closing the collapsible section +=============== diff --git a/docs/reference/connector/docs/connectors-notion.asciidoc b/docs/reference/connector/docs/connectors-notion.asciidoc new file mode 100644 index 0000000000000..7c08c5d81e032 --- /dev/null +++ b/docs/reference/connector/docs/connectors-notion.asciidoc @@ -0,0 +1,747 @@ +[#es-connectors-notion] +=== Elastic Notion Connector reference +++++ +Notion +++++ + +// Attributes (AKA variables) used in this file +:service-name: Notion +:service-name-stub: notion + +The Notion connector is written in Python using the {connectors-python}[Elastic connector framework^]. +View the {connectors-python}/connectors/sources/{service-name-stub}.py[*source code* for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + +// //////// //// //// //// //// //// //// //////// +// //////// NATIVE CONNECTOR REFERENCE (MANAGED SERVICE) /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-notion-native-connector-reference] +==== *Elastic managed connector reference* + +.View *Elastic managed connector* reference +[%collapsible] +=============== + +[discrete#es-connectors-notion-connector-availability-and-prerequisites] +===== Availability and prerequisites + +This managed connector was introduced in Elastic *8.14.0* as a managed service on Elastic Cloud. + +To use this connector natively in Elastic Cloud, satisfy all <>. + +[NOTE] +==== +This connector is in *beta* and is subject to change. The design and code is less mature than official GA features and is being provided as-is with no warranties. Beta features are not subject to the support SLA of official GA features. +==== + +[discrete#es-connectors-notion-connector-usage] +===== Usage + +To use this connector in the UI, select the *Notion* tile when creating a new connector under *Search -> Connectors*. + +If you're already familiar with how connectors work, you can also use the {ref}/connector-apis.html[Connector APIs]. + +For additional operations, see <>. + +[discrete#es-connectors-notion-create-native-connector] +===== Create a {service-name} connector +include::_connectors-create-native.asciidoc[] + +[discrete#es-connectors-notion-connector-connecting-to-notion] +===== Connecting to Notion + +To connect to Notion, the user needs to https://www.notion.so/help/create-integrations-with-the-notion-api#create-an-internal-integration[create an internal integration] for their Notion workspace, which can access resources using the Internal Integration Secret Token. Configure the Integration with following settings: + +1. Users must grant `READ` permission for content, comment and user capabilities for that integration from the Capabilities tab. + +2. Users must manually https://www.notion.so/help/add-and-manage-connections-with-the-api#add-connections-to-pages[add the integration as a connection] to the top-level pages in a workspace. Sub-pages will inherit the connections of the parent page automatically. + +[discrete#es-connectors-notion-connector-configuration] +===== Configuration + +Note the following configuration fields: + +`Notion Secret Key`(required):: +Secret token assigned to your integration, for a particular workspace. Example: + +* `zyx-123453-12a2-100a-1123-93fd09d67394` + +`Databases`(required):: +Comma-separated list of database names to be fetched by the connector. If the value is `*`, connector will fetch all the databases available in the workspace. Example: + +* `database1, database2` +* `*` + +`Pages`(required):: +Comma-separated list of page names to be fetched by the connector. If the value is `*`, connector will fetch all the pages available in the workspace. Examples: + +* `*` +* `Page1, Page2` + +`Index Comments`:: + +Toggle to enable fetching and indexing of comments from the Notion workspace for the configured pages, databases and the corresponding child blocks. Default value is `False`. + +[NOTE] +==== +Enabling comment indexing could impact connector performance due to increased network calls. Therefore, by default this value is `False`. +==== + +[discrete#es-connectors-notion-connector-content-extraction] +====== Content Extraction + +Refer to <>. + +[discrete#es-connectors-notion-connector-documents-and-syncs] +===== Documents and syncs + +The connector syncs the following objects and entities: + +* *Pages* +** Includes metadata such as `page name`, `id`, `last updated time`, etc. +* *Blocks* +** Includes metadata such as `title`, `type`, `id`, `content` (in case of file block), etc. +* *Databases* +** Includes metadata such as `name`, `id`, `records`, `size`, etc. +* *Users* +** Includes metadata such as `name`, `id`, `email address`, etc. +* *Comments* +** Includes the content and metadata such as `id`, `last updated time`, `created by`, etc. +** *Note*: Comments are excluded by default. + +[NOTE] +==== +* Files bigger than 10 MB won't be extracted. +* Permissions are not synced. *All documents* indexed to an Elastic deployment will be visible to *all users with access* to the relevant Elasticsearch index. +==== + +[discrete#es-connectors-notion-connector-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +[discrete#es-connectors-notion-connector-advanced-sync-rules] +===== Advanced sync rules + +[NOTE] +==== +A <> is required for advanced sync rules to take effect. +==== + +The following section describes *advanced sync rules* for this connector, to filter data in Notion _before_ indexing into {es}. +Advanced sync rules are defined through a source-specific DSL JSON snippet. + +Advanced sync rules for Notion take the following parameters: + +1. `searches`: Notion's search filter to search by title. +2. `query`: Notion's database query filter to fetch a specific database. + +[discrete#es-connectors-notion-connector-advanced-sync-rules-examples] +====== Examples + +[discrete] +*Example 1* + +Indexing every page where the title contains `Demo Page`: + +[source,js] +---- + { + "searches": [ + { + "filter": { + "value": "page" + }, + "query": "Demo Page" + } + ] + } +---- +// NOTCONSOLE + +[discrete] +*Example 2* + +Indexing every database where the title contains `Demo Database`: + +[source,js] +---- +{ + "searches": [ + { + "filter": { + "value": "database" + }, + "query": "Demo Database" + } + ] +} +---- +// NOTCONSOLE + +[discrete] +*Example 3* + +Indexing every database where the title contains `Demo Database` and every page where the title contains `Demo Page`: + +[source,js] +---- +{ + "searches": [ + { + "filter": { + "value": "database" + }, + "query": "Demo Database" + }, + { + "filter": { + "value": "page" + }, + "query": "Demo Page" + } + ] +} +---- +// NOTCONSOLE + +[discrete] +*Example 4* + +Indexing all pages in the workspace: + +[source,js] +---- +{ + "searches": [ + { + "filter": { + "value": "page" + }, + "query": "" + } + ] +} +---- +// NOTCONSOLE + +[discrete] +*Example 5* + +Indexing all the pages and databases connected to the workspace: + +[source,js] +---- +{ + "searches":[ + { + "query":"" + } + ] +} +---- +// NOTCONSOLE + +[discrete] +*Example 6* + +Indexing all the rows of a database where the record is `true` for the column `Task completed` and its property(datatype) is a checkbox: + +[source,js] +---- +{ + "database_query_filters": [ + { + "filter": { + "property": "Task completed", + "checkbox": { + "equals": true + } + }, + "database_id": "database_id" + } + ] +} +---- +// NOTCONSOLE + +[discrete] +*Example 7* + +Indexing all rows of a specific database: + +[source,js] +---- +{ + "database_query_filters": [ + { + "database_id": "database_id" + } + ] +} +---- +// NOTCONSOLE + +[discrete] +*Example 8* + +Indexing all blocks defined in `searches` and `database_query_filters`: + +[source,js] +---- +{ + "searches":[ + { + "query":"External tasks", + "filter":{ + "value":"database" + } + }, + { + "query":"External tasks", + "filter":{ + "value":"page" + } + } + ], + "database_query_filters":[ + { + "database_id":"notion_database_id1", + "filter":{ + "property":"Task completed", + "checkbox":{ + "equals":true + } + } + } + ] +} +---- +// NOTCONSOLE + +[NOTE] +==== +In this example the `filter` object syntax for `database_query_filters` is defined per the https://developers.notion.com/reference/post-database-query-filter[Notion documentation]. +==== + +[discrete#es-connectors-notion-connector-known-issues] +===== Known issues + +* *Updates to new pages may not be reflected immediately in the Notion API.* ++ +This could lead to these pages not being indexed by the connector, if a sync is initiated immediately after their addition. +To ensure all pages are indexed, initiate syncs a few minutes after adding pages to Notion. + +* *Notion's Public API does not support linked databases.* ++ +Linked databases in Notion are copies of a database that can be filtered, sorted, and viewed differently. +To fetch the information in a linked database, you need to target the original *source* database. +For more details refer to the https://developers.notion.com/docs/working-with-databases#linked-databases[Notion documentation]. + +* *Documents' `properties` objects are serialized as strings under `details`*. ++ +Notion's schema for `properties` is not consistent, and can lead to `document_parsing_exceptions` if indexed to Elasticsearch as an object. +For this reason, the `properties` object is instead serialized as a JSON string, and stored under the `details` field. +If you need to search a sub-object from `properties`, you may need to post-process the `details` field in an ingest pipeline to extract your desired subfield(s). + +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-notion-connector-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-notion-connector-security] +===== Security + +See <>. + + +// Closing the collapsible section +=============== + + +// //////// //// //// //// //// //// //// //////// +// //////// CONNECTOR CLIENT REFERENCE (SELF-MANAGED) /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-notion-connector-client-reference] +==== *Self-managed connector reference* + +.View *self-managed connector* reference +[%collapsible] +=============== + +[discrete#es-connectors-notion-client-connector-availability-and-prerequisites] +===== Availability and prerequisites + +This connector was introduced in Elastic *8.13.0*, available as a *self-managed* self-managed connector. + +To use this connector, satisfy all <>. +Importantly, you must deploy the connectors service on your own infrastructure. +You have two deployment options: + +* <>. Use this option if you're comfortable working with Python and want to iterate quickly locally. +* <>. Use this option if you want to deploy the connectors to a server, or use a container orchestration platform. + +[NOTE] +==== +This connector is in *beta* and is subject to change. The design and code is less mature than official GA features and is being provided as-is with no warranties. Beta features are not subject to the support SLA of official GA features. +==== + +[discrete#es-connectors-notion-client-connector-usage] +===== Usage + +To use this connector in the UI, select the *Notion* tile when creating a new connector under *Search -> Connectors*. + +For additional operations, see <>. + +[discrete#es-connectors-notion-create-connector-client] +===== Create a {service-name} connector +include::_connectors-create-client.asciidoc[] + +[discrete#es-connectors-notion-client-connector-connecting-to-notion] +===== Connecting to Notion + +To connect to Notion, the user needs to https://www.notion.so/help/create-integrations-with-the-notion-api#create-an-internal-integration[create an internal integration] for their Notion workspace, which can access resources using the Internal Integration Secret Token. Configure the Integration with following settings: + +1. Users must grant `READ` permission for content, comment and user capabilities for that integration from the Capabilities tab. + +2. Users must manually https://www.notion.so/help/add-and-manage-connections-with-the-api#add-connections-to-pages[add the integration as a connection] to the top-level pages in a workspace. Sub-pages will inherit the connections of the parent page automatically. + +[discrete#es-connectors-notion-client-connector-docker] +===== Deploy with Docker + +include::_connectors-docker-instructions.asciidoc[] + +[discrete#es-connectors-notion-client-connector-configuration] +===== Configuration + +Note the following configuration fields: + +`Notion Secret Key`(required):: +Secret token assigned to your integration, for a particular workspace. Example: + +* `zyx-123453-12a2-100a-1123-93fd09d67394` + +`Databases`(required):: +Comma-separated list of database names to be fetched by the connector. If the value is `*`, connector will fetch all the databases available in the workspace. Example: + +* `database1, database2` +* `*` + +`Pages`(required):: +Comma-separated list of page names to be fetched by the connector. If the value is `*`, connector will fetch all the pages available in the workspace. Examples: + +* `*` +* `Page1, Page2` + +`Index Comments`:: + +Toggle to enable fetching and indexing of comments from the Notion workspace for the configured pages, databases and the corresponding child blocks. Default value is `False`. + +[NOTE] +==== +Enabling comment indexing could impact connector performance due to increased network calls. Therefore, by default this value is `False`. +==== + +[discrete#es-connectors-notion-client-connector-content-extraction] +====== Content Extraction + +Refer to <>. + +[discrete#es-connectors-notion-client-connector-documents-and-syncs] +===== Documents and syncs + +The connector syncs the following objects and entities: + +* *Pages* +** Includes metadata such as `page name`, `id`, `last updated time`, etc. +* *Blocks* +** Includes metadata such as `title`, `type`, `id`, `content` (in case of file block), etc. +* *Databases* +** Includes metadata such as `name`, `id`, `records`, `size`, etc. +* *Users* +** Includes metadata such as `name`, `id`, `email address`, etc. +* *Comments* +** Includes the content and metadata such as `id`, `last updated time`, `created by`, etc. +** *Note*: Comments are excluded by default. + +[NOTE] +==== +* Files bigger than 10 MB won't be extracted. +* Permissions are not synced. *All documents* indexed to an Elastic deployment will be visible to *all users with access* to the relevant Elasticsearch index. +==== + +[discrete#es-connectors-notion-client-connector-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +[discrete#es-connectors-notion-client-connector-advanced-sync-rules] +===== Advanced sync rules + +[NOTE] +==== +A <> is required for advanced sync rules to take effect. +==== + +The following section describes *advanced sync rules* for this connector, to filter data in Notion _before_ indexing into {es}. +Advanced sync rules are defined through a source-specific DSL JSON snippet. + +Advanced sync rules for Notion take the following parameters: + +1. `searches`: Notion's search filter to search by title. +2. `query`: Notion's database query filter to fetch a specific database. + +[discrete#es-connectors-notion-client-connector-advanced-sync-rules-examples] +====== Examples + +[discrete] +*Example 1* + +Indexing every page where the title contains `Demo Page`: + +[source,js] +---- + { + "searches": [ + { + "filter": { + "value": "page" + }, + "query": "Demo Page" + } + ] + } +---- +// NOTCONSOLE + +[discrete] +*Example 2* + +Indexing every database where the title contains `Demo Database`: + +[source,js] +---- +{ + "searches": [ + { + "filter": { + "value": "database" + }, + "query": "Demo Database" + } + ] +} +---- +// NOTCONSOLE + +[discrete] +*Example 3* + +Indexing every database where the title contains `Demo Database` and every page where the title contains `Demo Page`: + +[source,js] +---- +{ + "searches": [ + { + "filter": { + "value": "database" + }, + "query": "Demo Database" + }, + { + "filter": { + "value": "page" + }, + "query": "Demo Page" + } + ] +} +---- +// NOTCONSOLE + +[discrete] +*Example 4* + +Indexing all pages in the workspace: + +[source,js] +---- +{ + "searches": [ + { + "filter": { + "value": "page" + }, + "query": "" + } + ] +} +---- +// NOTCONSOLE + +[discrete] +*Example 5* + +Indexing all the pages and databases connected to the workspace: + +[source,js] +---- +{ + "searches":[ + { + "query":"" + } + ] +} +---- +// NOTCONSOLE + +[discrete] +*Example 6* + +Indexing all the rows of a database where the record is `true` for the column `Task completed` and its property(datatype) is a checkbox: + +[source,js] +---- +{ + "database_query_filters": [ + { + "filter": { + "property": "Task completed", + "checkbox": { + "equals": true + } + }, + "database_id": "database_id" + } + ] +} +---- +// NOTCONSOLE + +[discrete] +*Example 7* + +Indexing all rows of a specific database: + +[source,js] +---- +{ + "database_query_filters": [ + { + "database_id": "database_id" + } + ] +} +---- +// NOTCONSOLE + +[discrete] +*Example 8* + +Indexing all blocks defined in `searches` and `database_query_filters`: + +[source,js] +---- +{ + "searches":[ + { + "query":"External tasks", + "filter":{ + "value":"database" + } + }, + { + "query":"External tasks", + "filter":{ + "value":"page" + } + } + ], + "database_query_filters":[ + { + "database_id":"notion_database_id1", + "filter":{ + "property":"Task completed", + "checkbox":{ + "equals":true + } + } + } + ] +} +---- +// NOTCONSOLE + +[NOTE] +==== +In this example the `filter` object syntax for `database_query_filters` is defined per the https://developers.notion.com/reference/post-database-query-filter[Notion documentation]. +==== + +[discrete#es-connectors-notion-client-connector-connector-client-operations] +===== Connector Client operations + +[discrete#es-connectors-notion-client-connector-end-to-end-testing] +====== End-to-end Testing + +The connector framework enables operators to run functional tests against a real data source, using Docker Compose. +You don't need a running Elasticsearch instance or Notion source to run this test. + +Refer to <> for more details. + +To perform E2E testing for the Notion connector, run the following command: + + +[source,shell] +---- +$ make ftest NAME=notion +---- + +For faster tests, add the `DATA_SIZE=small` flag: + +[source,shell] +---- +make ftest NAME=notion DATA_SIZE=small +---- + +By default, `DATA_SIZE=MEDIUM`. + +[discrete#es-connectors-notion-client-connector-known-issues] +===== Known issues + +* *Updates to new pages may not be reflected immediately in the Notion API.* ++ +This could lead to these pages not being indexed by the connector, if a sync is initiated immediately after their addition. +To ensure all pages are indexed, initiate syncs a few minutes after adding pages to Notion. + +* *Notion's Public API does not support linked databases.* ++ +Linked databases in Notion are copies of a database that can be filtered, sorted, and viewed differently. +To fetch the information in a linked database, you need to target the original *source* database. +For more details refer to the https://developers.notion.com/docs/working-with-databases#linked-databases[Notion documentation]. + +* *Documents' `properties` objects are serialized as strings under `details`*. ++ +Notion's schema for `properties` is not consistent, and can lead to `document_parsing_exceptions` if indexed to Elasticsearch as an object. +For this reason, the `properties` object is instead serialized as a JSON string, and stored under the `details` field. +If you need to search a sub-object from `properties`, you may need to post-process the `details` field in an ingest pipeline to extract your desired subfield(s). + +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-notion-client-connector-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-notion-client-connector-security] +===== Security + +See <>. + + +// Closing the collapsible section +=============== diff --git a/docs/reference/connector/docs/connectors-onedrive.asciidoc b/docs/reference/connector/docs/connectors-onedrive.asciidoc new file mode 100644 index 0000000000000..44ac96e2ad99d --- /dev/null +++ b/docs/reference/connector/docs/connectors-onedrive.asciidoc @@ -0,0 +1,604 @@ +[#es-connectors-onedrive] +=== Elastic OneDrive connector reference +++++ +OneDrive +++++ +// Attributes used in this file +:service-name: OneDrive +:service-name-stub: onedrive + +The _Elastic OneDrive connector_ is a <> for OneDrive. +This connector is written in Python using the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/{service-name-stub}.py[*source code* for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + +.Choose your connector reference +******************************* +Are you using a managed connector on Elastic Cloud or a self-managed connector? Expand the documentation based on your deployment method. +******************************* + +// //////// //// //// //// //// //// //// //////// +// //////// NATIVE CONNECTOR REFERENCE /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-onedrive-native-connector-reference] +==== *Elastic managed connector reference* + +.View *Elastic managed connector* reference + +[%collapsible] +=============== + +[discrete#es-connectors-onedrive-availability-prerequisites] +===== Availability and prerequisites + +This connector is available as a *managed connector* as of Elastic version *8.11.0*. + +To use this connector natively in Elastic Cloud, satisfy all <>. + +[discrete#es-connectors-onedrive-create-native-connector] +===== Create a {service-name} connector +include::_connectors-create-native.asciidoc[] + +[discrete#es-connectors-onedrive-usage] +===== Usage + +To use this connector natively in Elastic Cloud, see <>. + +For additional operations, see <>. + +[discrete#es-connectors-onedrive-usage-connection] +====== Connecting to OneDrive + +To connect to OneDrive you need to https://learn.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal[create an Azure Active Directory application and service principal] that can access resources. + +Follow these steps: + +. Go to the https://portal.azure.com[Azure portal] and sign in with your Azure account. +. Navigate to the *Azure Active Directory* service. +. Select *App registrations* from the left-hand menu. +. Click on the *New registration* button to register a new application. +. Provide a name for your app, and optionally select the supported account types (e.g., single tenant, multi-tenant). +. Click on the *Register* button to create the app registration. +. After the registration is complete, you will be redirected to the app's overview page. Take note of the *Application (client) ID* value, as you'll need it later. +. Scroll down to the *API permissions* section and click on the *Add a permission* button. +. In the *Request API permissions* pane, select *Microsoft Graph* as the API. +. Choose the application permissions and select the following permissions under the *Application* tab: `User.Read.All`, `File.Read.All` +. Click on the *Add permissions* button to add the selected permissions to your app. +Finally, click on the *Grant admin consent* button to grant the required permissions to the app. This step requires administrative privileges. **NOTE**: If you are not an admin, you need to request the Admin to grant consent via their Azure Portal. +. Click on *Certificates & Secrets* tab. Go to Client Secrets. Generate a new client secret and keep a note of the string present under `Value` column. + +[discrete#es-connectors-onedrive-usage-configuration] +===== Configuration + +The following configuration fields are *required*: + +Azure application Client ID:: +Unique identifier for your Azure Application, found on the app's overview page. Example: +* `ab123453-12a2-100a-1123-93fd09d67394` + +Azure application Client Secret:: +String value that the application uses to prove its identity when requesting a token, available under the `Certificates & Secrets` tab of your Azure application menu. Example: +* `eyav1~12aBadIg6SL-STDfg102eBfCGkbKBq_Ddyu` + +Azure application Tenant ID:: +Unique identifier of your Azure Active Directory instance. Example: +* `123a1b23-12a3-45b6-7c8d-fc931cfb448d` + +Enable document level security:: +Toggle to enable <>. +When enabled: +* Full syncs will fetch access control lists for each document and store them in the `_allow_access_control` field. +* Access control syncs will fetch users' access control lists and store them in a separate index. + +[WARNING] +==== +Enabling DLS for your connector will cause a significant performance degradation, as the API calls to the data source required for this functionality are rate limited. +This impacts the speed at which your content can be retrieved. +==== + +[discrete#es-connectors-onedrive-usage-content-extraction] +===== Content Extraction + +Refer to <> for more details. + +[discrete#es-connectors-onedrive-documents-syncs] +===== Documents and syncs + +The connector syncs the following objects and entities: + +* *Files* +** Includes metadata such as file name, path, size, content, etc. +* *Folders* + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted. (Self-managed connectors can use the <> to handle larger binary files.) +* Permissions are not synced by default. +You must first enable <>. +Otherwise, *all documents* indexed to an Elastic deployment will be visible to *all users with access* to that Elastic Deployment. +==== + +[discrete#es-connectors-onedrive-connectors-onedrive-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-onedrive-dls] +===== Document level security + +Document level security (DLS) enables you to restrict access to documents based on a user’s permissions. This feature is available by default for the OneDrive connector. +See <> for how to enable DLS for this connector. + +Refer to <> for more details about this feature. + +[NOTE] +==== +Refer to <> to learn how to ingest data with DLS enabled, when building a search application. +==== + +[discrete#es-connectors-onedrive-documents-sync-rules] +===== Sync rules + +_Basic_ sync rules are identical for all connectors and are available by default. +For more information read <>. + +[discrete#es-connectors-onedrive-sync-rules-advanced] +====== Advanced sync rules + +This connector supports <> for remote filtering. +These rules cover complex query-and-filter scenarios that cannot be expressed with basic sync rules. +Advanced sync rules are defined through a source-specific DSL JSON snippet. + +[NOTE] +==== +A <> is required for advanced sync rules to take effect. +==== + +Here are a few examples of advanced sync rules for this connector. + +[discrete#es-connectors-onedrive-sync-rules-advanced-examples-1] +*Example 1* + +This rule skips indexing for files with `.xlsx` and `.docx` extensions. +All other files and folders will be indexed. + +[source,js] +---- +[ + { + "skipFilesWithExtensions": [".xlsx" , ".docx"] + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-onedrive-sync-rules-advanced-examples-2] +*Example 2* + +This rule focuses on indexing files and folders owned by `user1-domain@onmicrosoft.com` and `user2-domain@onmicrosoft.com` but excludes files with `.py` extension. + +[source,js] +---- +[ + { + "owners": ["user1-domain@onmicrosoft.com", "user2-domain@onmicrosoft.com"], + "skipFilesWithExtensions": [".py"] + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-onedrive-sync-rules-advanced-examples-3] +*Example 3* + +This rule indexes only the files and folders directly inside the root folder, excluding any `.md` files. + +[source,js] +---- +[ + { + "skipFilesWithExtensions": [".md"], + "parentPathPattern": "/drive/root:" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-onedrive-sync-rules-advanced-examples-4] +*Example 4* + +This rule indexes files and folders owned by `user1-domain@onmicrosoft.com` and `user3-domain@onmicrosoft.com` that are directly inside the `abc` folder, which is a subfolder of any folder under the `hello` directory in the root. Files with extensions `.pdf` and `.py` are excluded. + +[source,js] +---- +[ + { + "owners": ["user1-domain@onmicrosoft.com", "user3-domain@onmicrosoft.com"], + "skipFilesWithExtensions": [".pdf", ".py"], + "parentPathPattern": "/drive/root:/hello/**/abc" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-onedrive-sync-rules-advanced-examples-5] +*Example 5* + +This example contains two rules. +The first rule indexes all files and folders owned by `user1-domain@onmicrosoft.com` and `user2-domain@onmicrosoft.com`. +The second rule indexes files for all other users, but skips files with a `.py` extension. + +[source,js] +---- +[ + { + "owners": ["user1-domain@onmicrosoft.com", "user2-domain@onmicrosoft.com"] + }, + { + "skipFilesWithExtensions": [".py"] + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-onedrive-sync-rules-advanced-examples-6] +*Example 6* + +This example contains two rules. +The first rule indexes all files owned by `user1-domain@onmicrosoft.com` and `user2-domain@onmicrosoft.com`, excluding `.md` files. +The second rule indexes files and folders recursively inside the `abc` folder. + +[source,js] +---- +[ + { + "owners": ["user1-domain@onmicrosoft.com", "user2-domain@onmicrosoft.com"], + "skipFilesWithExtensions": [".md"] + }, + { + "parentPathPattern": "/drive/root:/abc/**" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-onedrive-content-extraction] +===== Content Extraction + +See <>. + +[discrete#es-connectors-onedrive-known-issues] +===== Known issues + +* *Enabling document-level security impacts performance.* ++ +Enabling DLS for your connector will cause a significant performance degradation, as the API calls to the data source required for this functionality are rate limited. This impacts the speed at which your content can be retrieved. + +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-onedrive-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-onedrive-security] +===== Security + +See <>. +// Closing the collapsible section +=============== + + +// //////// //// //// //// //// //// //// //////// +// //////// CONNECTOR CLIENT REFERENCE /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-onedrive-connector-client-reference] +==== *Self-managed connector* + +.View *self-managed connector* reference + +[%collapsible] +=============== + +[discrete#es-connectors-onedrive-client-availability-prerequisites] +===== Availability and prerequisites + +This connector is available as a self-managed *self-managed connector*. + +This self-managed connector is compatible with Elastic versions *8.10.0+*. + +To use this connector, satisfy all <>. + +[discrete#es-connectors-onedrive-create-connector-client] +===== Create a {service-name} connector +include::_connectors-create-client.asciidoc[] + +[discrete#es-connectors-onedrive-client-usage] +===== Usage +For additional operations, see <>. + +[discrete#es-connectors-onedrive-client-usage-connection] +====== Connecting to OneDrive + +To connect to OneDrive you need to https://learn.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal[create an Azure Active Directory application and service principal] that can access resources. + +Follow these steps: + +. Go to the https://portal.azure.com[Azure portal] and sign in with your Azure account. +. Navigate to the *Azure Active Directory* service. +. Select *App registrations* from the left-hand menu. +. Click on the *New registration* button to register a new application. +. Provide a name for your app, and optionally select the supported account types (e.g., single tenant, multi-tenant). +. Click on the *Register* button to create the app registration. +. After the registration is complete, you will be redirected to the app's overview page. Take note of the *Application (client) ID* value, as you'll need it later. +. Scroll down to the *API permissions* section and click on the *Add a permission* button. +. In the *Request API permissions* pane, select *Microsoft Graph* as the API. +. Choose the application permissions and select the following permissions under the *Application* tab: `User.Read.All`, `File.Read.All` +. Click on the *Add permissions* button to add the selected permissions to your app. +Finally, click on the *Grant admin consent* button to grant the required permissions to the app. This step requires administrative privileges. **NOTE**: If you are not an admin, you need to request the Admin to grant consent via their Azure Portal. +. Click on *Certificates & Secrets* tab. Go to Client Secrets. Generate a new client secret and keep a note of the string present under `Value` column. + +[discrete#es-connectors-onedrive-client-docker] +===== Deployment using Docker + +Self-managed connectors are run on your own infrastructure. + +include::_connectors-docker-instructions.asciidoc[] + +[discrete#es-connectors-onedrive-client-usage-configuration] +===== Configuration + +The following configuration fields are *required*: + +`client_id`:: +Azure application Client ID, unique identifier for your Azure Application, found on the app's overview page. Example: +* `ab123453-12a2-100a-1123-93fd09d67394` + +`client_secret`:: +Azure application Client Secret, string value that the application uses to prove its identity when requesting a token. Available under the `Certificates & Secrets` tab of your Azure application menu. Example: +* `eyav1~12aBadIg6SL-STDfg102eBfCGkbKBq_Ddyu` + +`tenant_id`:: +Azure application Tenant ID: unique identifier of your Azure Active Directory instance. Example: +* `123a1b23-12a3-45b6-7c8d-fc931cfb448d` + +`retry_count`:: +The number of retry attempts after failed request to OneDrive. Default value is `3`. + +`use_document_level_security`:: +Toggle to enable <>. +When enabled: +* Full syncs will fetch access control lists for each document and store them in the `_allow_access_control` field. +* Access control syncs will fetch users' access control lists and store them in a separate index. ++ +[WARNING] +==== +Enabling DLS for your connector will cause a significant performance degradation, as the API calls to the data source required for this functionality are rate limited. +This impacts the speed at which your content can be retrieved. +==== + +`use_text_extraction_service`:: +Requires a separate deployment of the <>. +Requires that ingest pipeline settings disable text extraction. +Default value is `False`. + +[discrete#es-connectors-onedrive-client-usage-content-extraction] +===== Content Extraction + +Refer to <> for more details. + +[discrete#es-connectors-onedrive-client-documents-syncs] +===== Documents and syncs + +The connector syncs the following objects and entities: + +* *Files* +** Includes metadata such as file name, path, size, content, etc. +* *Folders* + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted by default. You can use the <> to handle larger binary files. +* Permissions are not synced by default. +You must first enable <>. +Otherwise, *all documents* indexed to an Elastic deployment will be visible to *all users with access* to that Elastic Deployment. +==== + +[discrete#es-connectors-onedrive-client-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-onedrive-client-dls] +===== Document level security + +Document level security (DLS) enables you to restrict access to documents based on a user’s permissions. This feature is available by default for the OneDrive connector. +See <> for how to enable DLS for this connector. + +Refer to <> for more details about this feature. + +[NOTE] +==== +Refer to <> to learn how to ingest data with DLS enabled, when building a search application. +==== + +[discrete#es-connectors-onedrive-client-documents-sync-rules] +===== Sync rules + +_Basic_ sync rules are identical for all connectors and are available by default. +For more information read <>. + +[discrete#es-connectors-onedrive-client-sync-rules-advanced] +====== Advanced sync rules + +This connector supports <> for remote filtering. +These rules cover complex query-and-filter scenarios that cannot be expressed with basic sync rules. +Advanced sync rules are defined through a source-specific DSL JSON snippet. + +[NOTE] +==== +A <> is required for advanced sync rules to take effect. +==== + +Here are a few examples of advanced sync rules for this connector. + +[discrete#es-connectors-onedrive-client-sync-rules-advanced-examples-1] +*Example 1* + +This rule skips indexing for files with `.xlsx` and `.docx` extensions. +All other files and folders will be indexed. + +[source,js] +---- +[ + { + "skipFilesWithExtensions": [".xlsx" , ".docx"] + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-onedrive-client-sync-rules-advanced-examples-2] +*Example 2* + +This rule focuses on indexing files and folders owned by `user1-domain@onmicrosoft.com` and `user2-domain@onmicrosoft.com` but excludes files with `.py` extension. + +[source,js] +---- +[ + { + "owners": ["user1-domain@onmicrosoft.com", "user2-domain@onmicrosoft.com"], + "skipFilesWithExtensions": [".py"] + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-onedrive-client-sync-rules-advanced-examples-3] +*Example 3* + +This rule indexes only the files and folders directly inside the root folder, excluding any `.md` files. + +[source,js] +---- +[ + { + "skipFilesWithExtensions": [".md"], + "parentPathPattern": "/drive/root:" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-onedrive-client-sync-rules-advanced-examples-4] +*Example 4* + +This rule indexes files and folders owned by `user1-domain@onmicrosoft.com` and `user3-domain@onmicrosoft.com` that are directly inside the `abc` folder, which is a subfolder of any folder under the `hello` directory in the root. Files with extensions `.pdf` and `.py` are excluded. + +[source,js] +---- +[ + { + "owners": ["user1-domain@onmicrosoft.com", "user3-domain@onmicrosoft.com"], + "skipFilesWithExtensions": [".pdf", ".py"], + "parentPathPattern": "/drive/root:/hello/**/abc" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-onedrive-client-sync-rules-advanced-examples-5] +*Example 5* + +This example contains two rules. +The first rule indexes all files and folders owned by `user1-domain@onmicrosoft.com` and `user2-domain@onmicrosoft.com`. +The second rule indexes files for all other users, but skips files with a `.py` extension. + +[source,js] +---- +[ + { + "owners": ["user1-domain@onmicrosoft.com", "user2-domain@onmicrosoft.com"] + }, + { + "skipFilesWithExtensions": [".py"] + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-onedrive-client-sync-rules-advanced-examples-6] +*Example 6* + +This example contains two rules. +The first rule indexes all files owned by `user1-domain@onmicrosoft.com` and `user2-domain@onmicrosoft.com`, excluding `.md` files. +The second rule indexes files and folders recursively inside the `abc` folder. + +[source,js] +---- +[ + { + "owners": ["user1-domain@onmicrosoft.com", "user2-domain@onmicrosoft.com"], + "skipFilesWithExtensions": [".md"] + }, + { + "parentPathPattern": "/drive/root:/abc/**" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-onedrive-client-content-extraction] +===== Content Extraction + +See <>. + +[discrete#es-connectors-onedrive-client-connector-client-operations] +===== Self-managed connector operations + +[discrete#es-connectors-onedrive-client-testing] +===== End-to-end testing + +The connector framework enables operators to run functional tests against a real data source. +Refer to <> for more details. + +To perform E2E testing for the GitHub connector, run the following command: + +[source,shell] +---- +$ make ftest NAME=onedrive +---- + +For faster tests, add the `DATA_SIZE=small` flag: + +[source,shell] +---- +make ftest NAME=onedrive DATA_SIZE=small +---- + +[discrete#es-connectors-onedrive-client-known-issues] +===== Known issues + +* *Enabling document-level security impacts performance.* ++ +Enabling DLS for your connector will cause a significant performance degradation, as the API calls to the data source required for this functionality are rate limited. This impacts the speed at which your content can be retrieved. + +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-onedrive-client-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-onedrive-client-security] +===== Security + +See <>. +// Closing the collapsible section +=============== diff --git a/docs/reference/connector/docs/connectors-opentext-documentum.asciidoc b/docs/reference/connector/docs/connectors-opentext-documentum.asciidoc new file mode 100644 index 0000000000000..e320062240428 --- /dev/null +++ b/docs/reference/connector/docs/connectors-opentext-documentum.asciidoc @@ -0,0 +1,162 @@ +[#es-connectors-opentext] +=== Elastic OpenText Documentum connector reference +++++ +OpenText Documentum +++++ + +// Attributes used in this file +:service-name: OpenText Documentum +:service-name-stub: opentext_documentum + +[WARNING] +==== +This connector is an *example connector* that serves as a building block for customizations and is subject to change. +Its source code currently lives on a https://github.com/elastic/connectors/blob/opentext-connector-backup/connectors/sources/opentext_documentum.py[feature branch] and is yet not part of the main Elastic Connectors codebase. +The design and code is less mature than supported features and is being provided as-is with no warranties. +This connector is not subject to the support SLA of supported features. +==== + +The Elastic OpenText Documentum connector is written in Python using the https://github.com/elastic/connectors/tree/main?tab=readme-ov-file#connector-framework[Elastic connector framework]. View the https://github.com/elastic/connectors/blob/opentext-connector-backup/connectors/sources/opentext_documentum.py[source code] for this example connector. + +[discrete#es-connectors-opentext-documentum-connector-availability-and-prerequisites] +==== Availability and prerequisites + +This *example connector* was introduced in Elastic *8.14.0*, available as a *self-managed* self-managed connector on a feature branch, for testing and development purposes only. + +To use this connector, satisfy all <>. +Importantly, you must deploy the connectors service on your own infrastructure. +You have two deployment options: + +* <>. Use this option if you're comfortable working with Python and want to iterate quickly locally. +* <>. Use this option if you want to deploy the connectors to a server, or use a container orchestration platform. + +[discrete#es-connectors-opentext-documentum-connector-usage] +==== Usage + +To set up this connector in the UI, select the *OpenText Documentum* tile when creating a new connector under *Search -> Connectors*. + +If you're already familiar with how connectors work, you can also use the {ref}/connector-apis.html[Connector APIs]. + +For additional operations, see <>. + +[discrete#es-connectors-opentext-documentum-connector-connecting-to-opentext-documentum] +==== Connecting to OpenText Documentum + +Basic authentication is used to connect with OpenText Documentum. + +[discrete#es-connectors-opentext-documentum-connector-configuration] +==== Configuration + +[discrete#es-connectors-opentext-documentum-connector-configure-opentext-documentum-connector] +===== Configure OpenText Documentum connector + +Note the following configuration fields: + +`OpenText Documentum host url` (required):: +The domain where OpenText Documentum is hosted. +Example: `https://192.158.1.38:2099/` + +`Username` (required):: +The username of the account to connect to OpenText Documentum. + +`Password` (required):: +The password of the account to connect to OpenText Documentum. + +`Repositories` (optional):: +Comma-separated list of repositories to fetch data from OpenText Documentum. If the value is `*` the connector will fetch data from all repositories present in the configured user’s account. ++ +Default value is `*`. ++ +Examples: ++ +* `elastic`, `kibana` +* `*` + +`Enable SSL` (optional):: +Enable SSL for the OpenText Documentum instance. + +`SSL Certificate` (Required if SSL is enabled):: +SSL certificate for the OpenText Documentum instance. +Example: ++ +``` +-----BEGIN CERTIFICATE----- +MIID+jCCAuKgAwIBAgIGAJJMzlxLMA0GCSqGSIb3DQEBCwUAMHoxCzAJBgNVBAYT +... +7RhLQyWn2u00L7/9Omw= +-----END CERTIFICATE----- +``` + +[discrete#es-connectors-opentext-documentum-connector-content-extraction] +===== Content Extraction + +Refer to <>. + +[discrete#es-connectors-opentext-documentum-connector-documents-and-syncs] +==== Documents and syncs + +The connector syncs the following objects and entities: + +* *Repositories* +* *Cabinets* +* *Files & Folders* + +[NOTE] +==== +* Files bigger than 10 MB won't be extracted. +* Permissions are not synced. *All documents* indexed to an Elastic deployment will be visible to *all users with access* to the destination Elasticsearch index. +==== + +[discrete#es-connectors-opentext-documentum-connector-sync-types] +===== Sync types +<> are supported by default for all connectors. + +<> are not available for this connector in the present version. + +[discrete#es-connectors-opentext-documentum-connector-sync-rules] +==== Sync rules + +<> are identical for all connectors and are available by default. + +Advanced sync rules are not available for this connector in the present version. + +[discrete#es-connectors-opentext-documentum-connector-connector-client-operations] +==== Connector Client operations + +[discrete#es-connectors-opentext-documentum-connector-end-to-end-testing] +===== End-to-end Testing + +The connector framework enables operators to run functional tests against a real data source, using Docker Compose. +You don't need a running Elasticsearch instance or OpenText Documentum source to run this test. + +Refer to <> for more details. + +To perform E2E testing for the OpenText Documentum connector, run the following command: + +```shell +$ make ftest NAME=opentext_documentum +``` +For faster tests, add the `DATA_SIZE=small` flag: + +[source,shell] +---- +make ftest NAME=opentext_documentum DATA_SIZE=small +---- + +By default, `DATA_SIZE=MEDIUM`. + + +[discrete#es-connectors-opentext-documentum-connector-known-issues] +==== Known issues + +* There are no known issues for this connector. Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-opentext-documentum-connector-troubleshooting] +==== Troubleshooting + +See <>. + +[discrete#es-connectors-opentext-documentum-connector-security] +==== Security + +See <>. diff --git a/docs/reference/connector/docs/connectors-oracle.asciidoc b/docs/reference/connector/docs/connectors-oracle.asciidoc new file mode 100644 index 0000000000000..839a92985a7d5 --- /dev/null +++ b/docs/reference/connector/docs/connectors-oracle.asciidoc @@ -0,0 +1,395 @@ +[#es-connectors-oracle] +=== Elastic Oracle connector reference +++++ +Oracle +++++ + +// Attributes used in this file: +:service-name: Oracle +:service-name-stub: oracle + + +// //////// //// //// //// //// //// //// //////// +// //////// NATIVE CONNECTOR REFERENCE (MANAGED SERVICE) /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-oracle-native-connector-reference] +==== *Elastic managed connector reference* + +.View *Elastic managed connector* reference +[%collapsible] +=============== + +[discrete#es-connectors-oracle-availability-prerequisites] +===== Availability and prerequisites + +This connector is available natively in Elastic Cloud as of *8.12.0*. +To use this connector, satisfy all <>. + +[discrete#es-connectors-oracle-create-native-connector] +===== Create a {service-name} connector +include::_connectors-create-native.asciidoc[] + +[discrete#es-connectors-oracle-usage] +===== Usage + +To use this connector as a *managed connector*, see <>. + +The database user requires `CONNECT` and `DBA` privileges and must be the owner of the tables to be indexed. + +[discrete#es-connectors-oracle-usage-secure-connection] +====== Secure connection + +To set up a secure connection the Oracle service must be installed on the system where the connector is running. + +Follow these steps: + +. Set the `oracle_home` parameter to your Oracle home directory. +If configuration files are not at the default location, set the `wallet_configuration_path` parameter. +. Create a directory to store the wallet. ++ +[source,shell] +---- +$ mkdir $ORACLE_HOME/ssl_wallet +---- +. Create file named `sqlnet.ora` at `$ORACLE_HOME/network/admin` and add the following content: ++ +[source,shell] +---- +WALLET_LOCATION = (SOURCE = (METHOD = FILE) (METHOD_DATA = (DIRECTORY = $ORACLE_HOME/ssl_wallet))) +SSL_CLIENT_AUTHENTICATION = FALSE +SSL_VERSION = 1.0 +SSL_CIPHER_SUITES = (SSL_RSA_WITH_AES_256_CBC_SHA) +SSL_SERVER_DN_MATCH = ON +---- ++ +. Run the following commands to create a wallet and attach an SSL certificate. +Replace the file name with your file name. ++ +[source,shell] +---- +$ orapki wallet create -wallet path-to-oracle-home/ssl_wallet -auto_login_only +$ orapki wallet add -wallet path-to-oracle-home/ssl_wallet -trusted_cert -cert path-to-oracle-home/ssl_wallet/root_ca.pem -auto_login_only +---- + +For more information, refer to this https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.Oracle.Options.SSL.html[Amazon RDS documentation^] about Oracle SSL. +Oracle docs: https://docs.oracle.com/database/121/DBSEG/asossl.htm#DBSEG070. +// TODO: check whether this link is appropriate + +For additional operations, see <>. + +[discrete#es-connectors-oracle-compatability] +===== Compatibility + +This connector is compatible with Oracle Database versions *18c*, *19c* and *21c*. + +[discrete#es-connectors-oracle-configuration] +===== Configuration + +Use the following configuration fields to set up the connector: + +Connection source:: +Dropdown to determine the Oracle Source Connection: `Service Name` or `SID`. Default value is `SID`. Select 'Service Name' option if connecting to a pluggable database. + +SID:: +SID of the database. + +Service name:: +Service name for the database. + +Host:: +The IP address or hostname of the Oracle database server. +Default value is `127.0.0.1`. + +Port:: +Port number of the Oracle database server. + +Username:: +Username to use to connect to the Oracle database server. + +Password:: +Password to use to connect to the Oracle database server. + +Comma-separated list of tables:: +Comma-separated list of tables to monitor for changes. +Default value is `*`. +Examples: +* `TABLE_1, TABLE_2` +* `*` + +[discrete#es-connectors-oracle-documents-syncs] +===== Documents and syncs + +* Tables with no primary key defined are skipped. +* If the table's system change number (SCN) value is not between the `min(SCN)` and `max(SCN)` values of the `SMON_SCN_TIME` table, the connector will not be able to retrieve the most recently updated time. + Data will therefore index in every sync. + For more details refer to the following https://community.oracle.com/tech/apps-infra/discussion/4076446/show-error-about-ora-08181-specified-number-is-not-a-valid-system-change-number-when-using-scn-t[discussion thread^]. +* The `sys` user is not supported, as it contains 1000+ system tables. If you need to work with the `sys` user, use either `sysdba` or `sysoper` and configure this as the username. + +[NOTE] +==== +* Files bigger than 10 MB won't be extracted. +* Permissions are not synced. +**All documents** indexed to an Elastic deployment will be visible to **all users with access** to that Elastic Deployment. +==== + +[discrete#es-connectors-oracle-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +Advanced sync rules are not available for this connector in the present version. +Currently, filtering is controlled by ingest pipelines. + +[discrete#es-connectors-oracle-known-issues] +===== Known issues + +There are no known issues for this connector. + +See <> for any issues affecting all connectors. + +[discrete#es-connectors-oracle-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-oracle-security] +===== Security + +See <>. + +[discrete#es-connectors-oracle-source] +===== Framework and source + +This connector is built with the {connectors-python}[Elastic connector framework^]. + +This connector uses the https://github.com/elastic/connectors/blob/{branch}/connectors/sources/generic_database.py[generic database connector source code^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + +View {connectors-python}/connectors/sources/oracle.py[additional code specific to this data source^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + + +// Closing the collapsible section +=============== + + +// //////// //// //// //// //// //// //// //////// +// //////// CONNECTOR CLIENT REFERENCE (SELF-MANAGED) /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-oracle-connector-client-reference] +==== *Self-managed connector reference* + +.View *self-managed connector* reference +[%collapsible] +=============== + +[discrete#es-connectors-oracle-client-availability-prerequisites] +===== Availability and prerequisites + +This connector is available as a self-managed *self-managed connector*. +This self-managed connector is compatible with Elastic versions *8.6.0+*. +To use this connector, satisfy all <>. + +[discrete#es-connectors-oracle-create-connector-client] +===== Create a {service-name} connector +include::_connectors-create-client.asciidoc[] + +[discrete#es-connectors-oracle-client-usage] +===== Usage + +To use this connector as a *self-managed connector*, see <>. + +The database user requires `CONNECT` and `DBA` privileges and must be the owner of the tables to be indexed. + +[discrete#es-connectors-oracle-client-usage-secure-connection] +====== Secure connection + +To set up a secure connection the Oracle service must be installed on the system where the connector is running. + +Follow these steps: + +. Set the `oracle_home` parameter to your Oracle home directory. +If configuration files are not at the default location, set the `wallet_configuration_path` parameter. +. Create a directory to store the wallet. ++ +[source,shell] +---- +$ mkdir $ORACLE_HOME/ssl_wallet +---- +. Create file named `sqlnet.ora` at `$ORACLE_HOME/network/admin` and add the following content: ++ +[source,shell] +---- +WALLET_LOCATION = (SOURCE = (METHOD = FILE) (METHOD_DATA = (DIRECTORY = $ORACLE_HOME/ssl_wallet))) +SSL_CLIENT_AUTHENTICATION = FALSE +SSL_VERSION = 1.0 +SSL_CIPHER_SUITES = (SSL_RSA_WITH_AES_256_CBC_SHA) +SSL_SERVER_DN_MATCH = ON +---- ++ +. Run the following commands to create a wallet and attach an SSL certificate. +Replace the file name with your file name. ++ +[source,shell] +---- +$ orapki wallet create -wallet path-to-oracle-home/ssl_wallet -auto_login_only +$ orapki wallet add -wallet path-to-oracle-home/ssl_wallet -trusted_cert -cert path-to-oracle-home/ssl_wallet/root_ca.pem -auto_login_only +---- + +For more information, refer to this https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.Oracle.Options.SSL.html[Amazon RDS documentation^] about Oracle SSL. +Oracle docs: https://docs.oracle.com/database/121/DBSEG/asossl.htm#DBSEG070. +// TODO: check whether this link is appropriate + +For additional operations, see <>. + +[discrete#es-connectors-oracle-client-compatability] +===== Compatibility + +Oracle Database versions *18c*, *19c* and *21c* are compatible with Elastic connector frameworks. + +[discrete#es-connectors-oracle-client-configuration] +===== Configuration + +[TIP] +==== +When using the <> workflow, these fields will use the default configuration set in the https://github.com/elastic/connectors/blob/{branch}/connectors/sources/generic_database.py[connector source code^]. +Note that this data source uses the `generic_database.py` connector source code. +Refer to https://github.com/elastic/connectors/blob/{branch}/connectors/sources/oracle.py[`oracle.py`^] for additional code, specific to this data source. + +These configurable fields will be rendered with their respective *labels* in the Kibana UI. +Once connected, users will be able to update these values in Kibana. +==== + +Use the following configuration fields to set up the connector: + +`connection_source`:: +Determines the Oracle source: Service Name or SID. Default value is SID. Select 'Service Name' if connecting to a pluggable database. + +`sid`:: +SID of the database. + +`service_name`:: +Service name for the database. + +`host`:: +The IP address or hostname of the Oracle database server. +Default value is `127.0.0.1`. + +`port`:: +Port number of the Oracle database server. + +`username`:: +Username to use to connect to the Oracle database server. + +`password`:: +Password to use to connect to the Oracle database server. + +`tables`:: +Comma-separated list of tables to monitor for changes. +Default value is `*`. +Examples: +* `TABLE_1, TABLE_2` +* `*` + +`oracle_protocol`:: +Protocol which the connector uses to establish a connection. +Default value is `TCP`. +For secure connections, use `TCPS`. + +`oracle_home`:: +Path to Oracle home directory to run connector in thick mode for secured connection. +For unsecured connections, keep this field empty. + +`wallet_configuration_path`:: +Path to SSL Wallet configuration files. + +`fetch_size`:: +Number of rows to fetch per request. +Default value is `50`. + +`retry_count`:: +Number of retry attempts after failed request to Oracle Database. +Default value is `3`. + +[discrete#es-connectors-oracle-client-docker] +===== Deployment using Docker + +include::_connectors-docker-instructions.asciidoc[] + +[discrete#es-connectors-oracle-client-documents-syncs] +===== Documents and syncs + +* Tables with no primary key defined are skipped. +* If the table's system change number (SCN) value is not between the `min(SCN)` and `max(SCN)` values of the `SMON_SCN_TIME` table, the connector will not be able to retrieve the most recently updated time. + Data will therefore index in every sync. + For more details refer to the following https://community.oracle.com/tech/apps-infra/discussion/4076446/show-error-about-ora-08181-specified-number-is-not-a-valid-system-change-number-when-using-scn-t[discussion thread^]. +* The `sys` user is not supported, as it contains 1000+ system tables. If you need to work with the `sys` user, use either `sysdba` or `sysoper` and configure this as the username. + +[NOTE] +==== +* Files bigger than 10 MB won't be extracted. +* Permissions are not synced. +**All documents** indexed to an Elastic deployment will be visible to **all users with access** to that Elastic Deployment. +==== + +[discrete#es-connectors-oracle-client-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +Advanced sync rules are not available for this connector in the present version. +Currently, filtering is controlled by ingest pipelines. + +[discrete#es-connectors-oracle-client-operations] +===== Self-managed connector operations + +[discrete#es-connectors-oracle-client-operations-testing] +====== End-to-end testing + +The connector framework enables operators to run functional tests against a real data source. +Refer to <> for more details. + +To execute a functional test for the Oracle connector, run the following command: + +[source,shell] +---- +make ftest NAME=oracle +---- + +By default, this will use a medium-sized dataset. +To make the test faster add the `DATA_SIZE=small` argument: + +[source,shell] +---- +make ftest NAME=oracle DATA_SIZE=small +---- + +[discrete#es-connectors-oracle-client-known-issues] +===== Known issues + +There are no known issues for this connector. + +See <> for any issues affecting all connectors. + +[discrete#es-connectors-oracle-client-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-oracle-client-security] +===== Security + +See <>. + +[discrete#es-connectors-oracle-client-source] +===== Framework and source + +This connector is built with the {connectors-python}[Elastic connector framework^]. + +This connector uses the https://github.com/elastic/connectors/blob/{branch}/connectors/sources/generic_database.py[generic database connector source code^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + +View {connectors-python}/connectors/sources/oracle.py[additional code specific to this data source^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + + +// Closing the collapsible section +=============== diff --git a/docs/reference/connector/docs/connectors-outlook.asciidoc b/docs/reference/connector/docs/connectors-outlook.asciidoc new file mode 100644 index 0000000000000..e032fb243dfa9 --- /dev/null +++ b/docs/reference/connector/docs/connectors-outlook.asciidoc @@ -0,0 +1,446 @@ +[#es-connectors-outlook] +=== Elastic Outlook connector reference +++++ +Outlook +++++ +// Attributes used in this file +:service-name: Outlook +:service-name-stub: outlook + +The Elastic Outlook connector is built with the Elastic connector framework and is available as a self-managed <>. + + +// //////// //// //// //// //// //// //// //////// +// //////// NATIVE CONNECTOR REFERENCE (MANAGED SERVICE) /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-outlook-native-connector-reference] +==== *Elastic managed connector reference* + +.View *Elastic managed connector* reference +[%collapsible] +=============== + +[discrete#es-connectors-outlook-availability-and-prerequisites] +===== Availability and prerequisites + +This connector is available as a *managed connector* (managed service) in Elastic Cloud. + +This connector is compatible with Elastic versions *8.13.0+*. + +To use this connector, satisfy all <>. + +[discrete#es-connectors-outlook-create-native-connector] +===== Create a {service-name} connector +include::_connectors-create-native.asciidoc[] + +[discrete#es-connectors-outlook-usage] +===== Usage + +To use this connector as a managed connector in Elastic Cloud, use the *Connectors* workflow in the Kibana UI. + +To create a new {service-name} connector: + +. Navigate to *Search -> Connectors* page in the Kibana UI. +. Select the *New Native Connector* button. +. Select the *{service-name}* connector. + +For additional operations, see <>. + +[discrete#es-connectors-outlook-connecting-to-outlook] +===== Connecting to Outlook + +Outlook connector supports both cloud (Office365 Outlook) and on-premises (Exchange Server) platforms. + +[discrete#es-connectors-outlook-connect-to-exchange-server] +====== Connect to Exchange Server + +In order to connect to Exchange server, the connector fetches Active Directory users with the help of `ldap3` python library. + +[discrete#es-connectors-outlook-connect-to-office365-outlook-outlook-cloud] +====== Connect to Office365 Outlook (Outlook Cloud) + +To integrate with the Outlook connector using Azure, follow these steps to create and configure an Azure application: + +1. Navigate to the https://portal.azure.com/[Azure Portal] and log in using your credentials. +2. Click on *App registrations* to register a new application. +3. Navigate to the *Overview* tab. Make a note of the `Client ID` and `Tenant ID`. +4. Click on the *Certificates & secrets* tab and create a new client secret. Keep this secret handy. +5. Go to the *API permissions* tab. + * Click on "Add permissions." + * Choose "APIs my organization uses." + * Search for and select "Office 365 Exchange Online." + * Add the `full_access_as_app` application permission. + +You can now use the Client ID, Tenant ID, and Client Secret you've noted to configure the Outlook connector. + +[discrete#es-connectors-outlook-configuration] +===== Configuration + +Outlook data source (required):: +Dropdown to determine Outlook platform type: `outlook_cloud` or `outlook_server`. Default value is `outlook_cloud`. + +Tenant ID:: +Required if data source is `outlook_cloud`. +The Tenant ID for the Azure account hosting the Outlook instance. + +Client ID:: +Required if data source is `outlook_cloud`. +The Client ID to authenticate with Outlook instance. + +Client Secret Value:: +Required if data source is `outlook_cloud`. +The Client Secret value to authenticate with Outlook instance. + +Exchange Server:: +Required if data source is `outlook_server`. +IP address to connect with Exchange server. Example: `127.0.0.1` + +Active Directory Server:: +Required if data source is `outlook_server`. +IP address to fetch users from Exchange Active Directory to fetch data. Example: `127.0.0.1` + +Exchange server username:: +Required if data source is `outlook_server`. +Username to authenticate with Exchange server. + +Exchange server password:: +Required if data source is `outlook_server`. +Password to authenticate with Exchange server. + +Exchange server domain name:: +Required if data source is `outlook_server`. +Domain name for Exchange server users such as `gmail.com` or `exchange.local`. + +Enable SSL:: +Whether SSL verification will be enabled. Default value is `False`. +*Note:* This configuration is applicable for `Outlook Server` only. + +SSL certificate:: +Required if ssl is enabled. +Content of SSL certificate. Example certificate: ++ +[text] +[source, txt] +---- +-----BEGIN CERTIFICATE----- +MIID+jCCAuKgAwIBAgIGAJJMzlxLMA0GCSqGSIb3DQEBCwUAMHoxCzAJBgNVBAYT +... +7RhLQyWn2u00L7/9Omw= +-----END CERTIFICATE----- +---- ++ +*Note:* This configuration is applicable for `Outlook Server` only. + +Document level security:: +Toggle to enable <>. +When enabled: ++ +* Full syncs will fetch access control lists for each document and store them in the `_allow_access_control` field. +* Access control syncs fetch users' access control lists and store them in a separate index. + +[discrete#es-connectors-outlook-content-extraction] +===== Content Extraction + +Refer to <>. + +[discrete#es-connectors-outlook-documents-and-syncs] +===== Documents and syncs + +The connector syncs the following objects and entities: + +* *Mails* +** *Inbox Mails* +** *Sent Mails* +** *Archive Mails* +** *Junk Mails* +* *Contacts* +* *Calendar Events* +* *Tasks* +* *Attachments* +** *Mail Attachments* +** *Task Attachments* +** *Calendar Attachments* + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted. (Self-managed connectors can use the <> to handle larger binary files.) +* Permissions are not synced. *All documents* indexed to an Elastic deployment will be visible to *all users with access* to that Elastic Deployment. +==== + +[discrete#es-connectors-outlook-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-outlook-document-level-security] +===== Document level security + +Document level security (DLS) enables you to restrict access to documents based on a user’s permissions. +Refer to <> on this page for how to enable DLS for this connector. + +[NOTE] +==== +Refer to <> to learn how to ingest data from a connector with DLS enabled, when building a search application. +The example uses SharePoint Online as the data source, but the same steps apply to every connector. +==== + +[discrete#es-connectors-outlook-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +[discrete#es-connectors-outlook-advanced-sync-rules] +===== Advanced Sync Rules + +Advanced sync rules are not available for this connector in the present version. + +[discrete#es-connectors-outlook-connector-client-operations] +===== Connector Client operations + +[discrete#es-connectors-outlook-end-to-end-testing] +====== End-to-end Testing + +*Note:* End-to-end testing is not available in the current version of the connector. + +[discrete#es-connectors-outlook-known-issues] +====== Known issues + +There are currently no known issues for this connector. +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-outlook-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-outlook-security] +===== Security + +See <>. + +[discrete#es-connectors-outlook-source] +===== Framework and source + +This connector is written in Python using the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/outlook.py[source code for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + + +// Closing the collapsible section +=============== + + +// //////// //// //// //// //// //// //// //////// +// //////// CONNECTOR CLIENT REFERENCE (SELF-MANAGED) /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-outlook-connector-client-reference] +==== *Self-managed connector reference* + +.View *self-managed connector* reference +[%collapsible] +=============== + +[discrete#es-connectors-outlook-client-availability-and-prerequisites] +===== Availability and prerequisites + +This connector is available as a self-managed *self-managed connector*. To use this connector, satisfy all <>. + +[discrete#es-connectors-outlook-create-connector-client] +===== Create a {service-name} connector +include::_connectors-create-client.asciidoc[] + +[discrete#es-connectors-outlook-client-usage] +===== Usage + +To use this connector as a *self-managed connector*, use the *Outlook* tile from the connectors list OR *Customized connector* workflow. + +For additional operations, see <>. + +[discrete#es-connectors-outlook-client-connecting-to-outlook] +===== Connecting to Outlook + +Outlook connector supports both cloud (Office365 Outlook) and on-premises (Exchange Server) platforms. + +[discrete#es-connectors-outlook-client-connect-to-exchange-server] +====== Connect to Exchange Server + +In order to connect to Exchange server, the connector fetches Active Directory users with the help of `ldap3` python library. + +[discrete#es-connectors-outlook-client-connect-to-office365-outlook-outlook-cloud] +====== Connect to Office365 Outlook (Outlook Cloud) + +To integrate with the Outlook connector using Azure, follow these steps to create and configure an Azure application: + +1. Navigate to the https://portal.azure.com/[Azure Portal] and log in using your credentials. +2. Click on *App registrations* to register a new application. +3. Navigate to the *Overview* tab. Make a note of the `Client ID` and `Tenant ID`. +4. Click on the *Certificates & secrets* tab and create a new client secret. Keep this secret handy. +5. Go to the *API permissions* tab. + * Click on "Add permissions." + * Choose "APIs my organization uses." + * Search for and select "Office 365 Exchange Online." + * Add the `full_access_as_app` application permission. + +You can now use the Client ID, Tenant ID, and Client Secret you've noted to configure the Outlook connector. + +[discrete#es-connectors-outlook-client-configuration] +===== Configuration + +`data_source`:: (required) +Dropdown to determine Outlook platform type: `outlook_cloud` or `outlook_server`. Default value is `outlook_cloud`. + +`tenant_id`:: (required if data source is outlook_cloud) +The Tenant ID for the Azure account hosting the Outlook instance. + +`client_id`:: (required if data source is outlook_cloud) +The Client ID to authenticate with Outlook instance. + +`client_secret`:: (required if data source is outlook_cloud) +The Client Secret value to authenticate with Outlook instance. + +`exchange_server`:: (required if data source is outlook_server) +IP address to connect with Exchange server. Example: `127.0.0.1` + +`active_directory_server`:: (required if data source is outlook_server) +IP address to fetch users from Exchange Active Directory to fetch data. Example: `127.0.0.1` + +`username`:: (required if data source is outlook_server) +Username to authenticate with Exchange server. + +`password`:: (required if data source is outlook_server) +Password to authenticate with Exchange server. + +`domain`:: (required if data source is outlook_server) +Domain name for Exchange server users such as `gmail.com` or `exchange.local`. + +`ssl_enabled`:: +Whether SSL verification will be enabled. Default value is `False`. +*Note:* This configuration is applicable for `Outlook Server` only. + +`ssl_ca`:: (required if ssl is enabled) +Content of SSL certificate. Example certificate: ++ +[text] +[source, txt] +---- +-----BEGIN CERTIFICATE----- +MIID+jCCAuKgAwIBAgIGAJJMzlxLMA0GCSqGSIb3DQEBCwUAMHoxCzAJBgNVBAYT +... +7RhLQyWn2u00L7/9Omw= +-----END CERTIFICATE----- +---- + +`use_text_extraction_service`:: +Use <>. +Default value is `False`. + +`document_level_security`:: +Toggle to enable <>. +When enabled: ++ +* Full syncs will fetch access control lists for each document and store them in the `_allow_access_control` field. +* Access control syncs fetch users' access control lists and store them in a separate index. + +*Note:* This configuration is applicable for `Outlook Server` only. + +[discrete#es-connectors-outlook-client-client-docker] +===== Deployment using Docker + +include::_connectors-docker-instructions.asciidoc[] + +[discrete#es-connectors-outlook-client-content-extraction] +===== Content Extraction + +Refer to <>. + +[discrete#es-connectors-outlook-client-documents-and-syncs] +===== Documents and syncs + +The connector syncs the following objects and entities: + +* *Mails* +** *Inbox Mails* +** *Sent Mails* +** *Archive Mails* +** *Junk Mails* +* *Contacts* +* *Calendar Events* +* *Tasks* +* *Attachments* +** *Mail Attachments* +** *Task Attachments* +** *Calendar Attachments* + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted by default. You can use the <> to handle larger binary files. +* Permissions are not synced. *All documents* indexed to an Elastic deployment will be visible to *all users with access* to that Elastic Deployment. +==== + +[discrete#es-connectors-outlook-client-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-outlook-client-document-level-security] +===== Document level security + +Document level security (DLS) enables you to restrict access to documents based on a user’s permissions. +Refer to <> on this page for how to enable DLS for this connector. + +[NOTE] +==== +Refer to <> to learn how to ingest data from a connector with DLS enabled, when building a search application. +The example uses SharePoint Online as the data source, but the same steps apply to every connector. +==== + +[discrete#es-connectors-outlook-client-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +[discrete#es-connectors-outlook-client-advanced-sync-rules] +===== Advanced Sync Rules + +Advanced sync rules are not available for this connector in the present version. + +[discrete#es-connectors-outlook-client-connector-client-operations] +===== Connector Client operations + +[discrete#es-connectors-outlook-client-end-to-end-testing] +====== End-to-end Testing + +*Note:* End-to-end testing is not available in the current version of the connector. + +[discrete#es-connectors-outlook-client-known-issues] +====== Known issues + +There are currently no known issues for this connector. +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-outlook-client-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-outlook-client-security] +===== Security + +See <>. + +[discrete#es-connectors-outlook-client-source] +===== Framework and source + +This connector is included in the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/outlook.py[source code for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + + +// Closing the collapsible section +=============== diff --git a/docs/reference/connector/docs/connectors-postgresql.asciidoc b/docs/reference/connector/docs/connectors-postgresql.asciidoc new file mode 100644 index 0000000000000..aa6cb7f29e633 --- /dev/null +++ b/docs/reference/connector/docs/connectors-postgresql.asciidoc @@ -0,0 +1,644 @@ +[#es-connectors-postgresql] +=== Elastic PostgreSQL connector reference +++++ +PostgreSQL +++++ +// Attributes used in this file +:service-name: PostgreSQL +:service-name-stub: postgresql + +The _Elastic PostgreSQL connector_ is a connector for https://www.postgresql.org[PostgreSQL^]. +This connector is written in Python using the {connectors-python}[Elastic connector framework^]. + +This connector uses the https://github.com/elastic/connectors/blob/{branch}/connectors/sources/generic_database.py[generic database connector source code^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). +View the specific {connectors-python}/connectors/sources/{service-name-stub}.py[*source code* for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + + +.Choose your connector reference +******************************* +Are you using an Elastic managed connector on Elastic Cloud or a self-managed connector? Expand the documentation based on your deployment method. +******************************* + +// //////// //// //// //// //// //// //// //////// +// //////// NATIVE CONNECTOR REFERENCE /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#connectors-postgresql-native-connector-reference] +=== *Elastic managed connector (Elastic Cloud)* + +.View *Elastic managed connector* reference + +[%collapsible] +=============== + +[discrete#connectors-postgresql-availability-prerequisites] +==== Availability and prerequisites + +This connector is available as an *Elastic managed connector* in Elastic versions *8.8.0 and later*. +To use this connector natively in Elastic Cloud, satisfy all <>. + +[discrete#connectors-postgresql-create-native-connector] +==== Create a {service-name} connector +include::_connectors-create-native.asciidoc[] + +[discrete#connectors-postgresql-usage] +==== Usage + +To use this connector as an *Elastic managed connector*, use the *Connector* workflow. +See <>. + +[TIP] +==== +Users must set `track_commit_timestamp` to `on`. +To do this, run `ALTER SYSTEM SET track_commit_timestamp = on;` in PostgreSQL server. +==== + +For additional operations, see <<-esconnectors-usage>>. + +[NOTE] +==== +For an end-to-end example of the connector client workflow, see <>. +==== + +[discrete#connectors-postgresql-compatibility] +==== Compatibility + +PostgreSQL versions 11 to 15 are compatible with the Elastic connector. + +[discrete#connectors-postgresql-configuration] +==== Configuration + +Set the following configuration fields: + +Host:: +The server host address where the PostgreSQL instance is hosted. +Examples: ++ +* `192.158.1.38` +* `demo.instance.demo-region.demo.service.com` + +Port:: +The port where the PostgreSQL instance is hosted. +Examples: ++ +* `5432` (default) + +Username:: +The username of the PostgreSQL account. + +Password:: +The password of the PostgreSQL account. + +Database:: +Name of the PostgreSQL database. +Examples: ++ +* `employee_database` +* `customer_database` + +Schema:: +The schema of the PostgreSQL database. + +Comma-separated List of Tables:: +A list of tables separated by commas. +The PostgreSQL connector will fetch data from all tables present in the configured database, if the value is `*` . +Default value is `*`. +Examples: ++ +* `table_1, table_2` +* `*` ++ +[WARNING] +==== +This field can be bypassed when using advanced sync rules. +==== + +Enable SSL:: +Toggle to enable SSL verification. +Disabled by default. + +SSL Certificate:: +Content of SSL certificate. +If SSL is disabled, the `ssl_ca` value will be ignored. ++ +.*Expand* to see an example certificate +[%collapsible] +==== +``` +-----BEGIN CERTIFICATE----- +MIID+jCCAuKgAwIBAgIGAJJMzlxLMA0GCSqGSIb3DQEBCwUAMHoxCzAJBgNVBAYT +AlVTMQwwCgYDVQQKEwNJQk0xFjAUBgNVBAsTDURlZmF1bHROb2RlMDExFjAUBgNV +BAsTDURlZmF1bHRDZWxsMDExGTAXBgNVBAsTEFJvb3QgQ2VydGlmaWNhdGUxEjAQ +BgNVBAMTCWxvY2FsaG9zdDAeFw0yMTEyMTQyMjA3MTZaFw0yMjEyMTQyMjA3MTZa +MF8xCzAJBgNVBAYTAlVTMQwwCgYDVQQKEwNJQk0xFjAUBgNVBAsTDURlZmF1bHRO +b2RlMDExFjAUBgNVBAsTDURlZmF1bHRDZWxsMDExEjAQBgNVBAMTCWxvY2FsaG9z +dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMv5HCsJZIpI5zCy+jXV +z6lmzNc9UcVSEEHn86h6zT6pxuY90TYeAhlZ9hZ+SCKn4OQ4GoDRZhLPTkYDt+wW +CV3NTIy9uCGUSJ6xjCKoxClJmgSQdg5m4HzwfY4ofoEZ5iZQ0Zmt62jGRWc0zuxj +hegnM+eO2reBJYu6Ypa9RPJdYJsmn1RNnC74IDY8Y95qn+WZj//UALCpYfX41hko +i7TWD9GKQO8SBmAxhjCDifOxVBokoxYrNdzESl0LXvnzEadeZTd9BfUtTaBHhx6t +njqqCPrbTY+3jAbZFd4RiERPnhLVKMytw5ot506BhPrUtpr2lusbN5svNXjuLeea +MMUCAwEAAaOBoDCBnTATBgNVHSMEDDAKgAhOatpLwvJFqjAdBgNVHSUEFjAUBggr +BgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4E+UHJvZmlsZVVVSUQ6QXBwU3J2 +MDEtQkFTRS05MDkzMzJjMC1iNmFiLTQ2OTMtYWI5NC01Mjc1ZDI1MmFmNDiCCWxv +Y2FsaG9zdDARBgNVHQ4ECgQITzqhA5sO8O4wDQYJKoZIhvcNAQELBQADggEBAKR0 +gY/BM69S6BDyWp5dxcpmZ9FS783FBbdUXjVtTkQno+oYURDrhCdsfTLYtqUlP4J4 +CHoskP+MwJjRIoKhPVQMv14Q4VC2J9coYXnePhFjE+6MaZbTjq9WaekGrpKkMaQA +iQt5b67jo7y63CZKIo9yBvs7sxODQzDn3wZwyux2vPegXSaTHR/rop/s/mPk3YTS +hQprs/IVtPoWU4/TsDN3gIlrAYGbcs29CAt5q9MfzkMmKsuDkTZD0ry42VjxjAmk +xw23l/k8RoD1wRWaDVbgpjwSzt+kl+vJE/ip2w3h69eEZ9wbo6scRO5lCO2JM4Pr +7RhLQyWn2u00L7/9Omw= +-----END CERTIFICATE----- +``` +==== + +[discrete#connectors-postgresql-documents-syncs] +==== Documents and syncs + +* Tables must be owned by a PostgreSQL user. +* Tables with no primary key defined are skipped. +* To fetch the last updated time in PostgreSQL, `track_commit_timestamp` must be set to `on`. +Otherwise, all data will be indexed in every sync. + +[NOTE] +==== +* Files bigger than 10 MB won't be extracted. +* Permissions are not synced. +**All documents** indexed to an Elastic deployment will be visible to **all users with access** to that Elastic Deployment. +==== + +[discrete#connectors-postgresql-sync-rules] +==== Sync rules + +<> are identical for all connectors and are available by default. + +[discrete#connectors-postgresql-sync-rules-advanced] +===== Advanced sync rules + +[NOTE] +==== +A <> is required for advanced sync rules to take effect. +==== + +Advanced sync rules are defined through a source-specific DSL JSON snippet. + +[discrete#connectors-postgresql-sync-rules-advanced-example-data] +====== Example data + +Here is some example data that will be used in the following examples. + +[discrete#connectors-postgresql-sync-rules-advanced-example-data-1] +*`employee` table* + +[cols="3*", options="header"] +|=== +| emp_id | name | age +| 3 | John | 28 +| 10 | Jane | 35 +| 14 | Alex | 22 +|=== + +[discrete#connectors-postgresql-sync-rules-advanced-example-2] +*`customer` table* + +[cols="3*", options="header"] +|=== +| c_id | name | age +| 2 | Elm | 24 +| 6 | Pine | 30 +| 9 | Oak | 34 +|=== + +[discrete#connectors-postgresql-sync-rules-advanced-examples] +====== Advanced sync rules examples + +[discrete#connectors-postgresql-sync-rules-advanced-examples-1] +*Multiple table queries* + +[source,js] +---- +[ + { + "tables": [ + "employee" + ], + "query": "SELECT * FROM employee" + }, + { + "tables": [ + "customer" + ], + "query": "SELECT * FROM customer" + } +] +---- +// NOTCONSOLE + +[discrete#connectors-postgresql-sync-rules-advanced-examples-1-id-columns] +*Multiple table queries with `id_columns`* + +In 8.15.0, we added a new optional `id_columns` field in our advanced sync rules for the PostgreSQL connector. +Use the `id_columns` field to ingest tables which do not have a primary key. Include the names of unique fields so that the connector can use them to generate unique IDs for documents. + +[source,js] +---- +[ + { + "tables": [ + "employee" + ], + "query": "SELECT * FROM employee", + "id_columns": ["emp_id"] + }, + { + "tables": [ + "customer" + ], + "query": "SELECT * FROM customer", + "id_columns": ["c_id"] + } +] +---- +// NOTCONSOLE + +This example uses the `id_columns` field to specify the unique fields `emp_id` and `c_id` for the `employee` and `customer` tables, respectively. + +[discrete#connectors-postgresql-sync-rules-advanced-examples-2] +*Filtering data with `WHERE` clause* + +[source,js] +---- +[ + { + "tables": ["employee"], + "query": "SELECT * FROM employee WHERE emp_id > 5" + } +] +---- +// NOTCONSOLE + +[discrete#connectors-postgresql-sync-rules-advanced-examples-3] +*`JOIN` operations* + +[source,js] +---- +[ + { + "tables": ["employee", "customer"], + "query": "SELECT * FROM employee INNER JOIN customer ON employee.emp_id = customer.c_id" + } +] +---- +// NOTCONSOLE + +[WARNING] +==== +When using advanced rules, a query can bypass the configuration field `tables`. +This will happen if the query specifies a table that doesn't appear in the configuration. +This can also happen if the configuration specifies `*` to fetch all tables while the advanced sync rule requests for only a subset of tables. +==== + +[discrete#connectors-postgresql-known-issues] +==== Known issues + +There are no known issues for this connector. +Refer to <> for a list of known issues for all connectors. + +[discrete#connectors-postgresql-troubleshooting] +==== Troubleshooting + +See <>. + +[discrete#connectors-postgresql-security] +==== Security + +See <>. + +// Closing the collapsible section +=============== + +[discrete#es-connectors-postgresql-connector-client-reference] +=== *Self-managed connector* + +.View *self-managed connector* reference + +[%collapsible] +=============== + +[discrete#es-connectors-postgresql-client-availability-prerequisites] +==== Availability and prerequisites + +This connector is available as a self-managed *self-managed connector*. +To use this connector, satisfy all <>. + +[discrete#es-connectors-postgresql-create-connector-client] +==== Create a {service-name} connector +include::_connectors-create-client.asciidoc[] + +[discrete#es-connectors-postgresql-client-usage] +==== Usage + +To use this connector as a *self-managed connector*, see <>. +[TIP] +==== +Users must set `track_commit_timestamp` to `on`. +To do this, run `ALTER SYSTEM SET track_commit_timestamp = on;` in PostgreSQL server. +==== + +For additional operations, see. + +[NOTE] +==== +For an end-to-end example of the self-managed connector workflow, see <>. +==== + +[discrete#es-connectors-postgresql-client-compatibility] +==== Compatibility + +PostgreSQL versions 11 to 15 are compatible with Elastic connector frameworks. + +[discrete#es-connectors-postgresql-client-configuration] +==== Configuration + +[TIP] +==== +When using the <>, initially these fields will use the default configuration set in the https://github.com/elastic/connectors-python/blob/{branch}/connectors/sources/postgresql.py[connector source code^]. + +These configurable fields will be rendered with their respective *labels* in the Kibana UI. +Once connected, users will be able to update these values in Kibana. +==== + +Set the following configuration fields: + +`host`:: +The server host address where the PostgreSQL instance is hosted. +Examples: ++ +* `192.158.1.38` +* `demo.instance.demo-region.demo.service.com` + +`port`:: +The port where the PostgreSQL instance is hosted. +Examples: ++ +* `5432` +* `9090` + +`username`:: +The username of the PostgreSQL account. + +`password`:: +The password of the PostgreSQL account. + +`database`:: +Name of the PostgreSQL database. +Examples: ++ +* `employee_database` +* `customer_database` + +`schema`:: +The schema of the PostgreSQL database. + +`tables`:: +A list of tables separated by commas. +The PostgreSQL connector will fetch data from all tables present in the configured database, if the value is `*` . +Default value is `*`. +Examples: ++ +* `table_1, table_2` +* `*` ++ +[WARNING] +==== +This field can be bypassed when using advanced sync rules. +==== + +`ssl_enabled`:: +Whether SSL verification will be enabled. +Default value is `True`. + +`ssl_ca`:: +Content of SSL certificate (if SSL is enabled). +If SSL is disabled, the `ssl_ca` value will be ignored. ++ +.*Expand* to see an example certificate +[%collapsible] +==== +``` +-----BEGIN CERTIFICATE----- +MIID+jCCAuKgAwIBAgIGAJJMzlxLMA0GCSqGSIb3DQEBCwUAMHoxCzAJBgNVBAYT +AlVTMQwwCgYDVQQKEwNJQk0xFjAUBgNVBAsTDURlZmF1bHROb2RlMDExFjAUBgNV +BAsTDURlZmF1bHRDZWxsMDExGTAXBgNVBAsTEFJvb3QgQ2VydGlmaWNhdGUxEjAQ +BgNVBAMTCWxvY2FsaG9zdDAeFw0yMTEyMTQyMjA3MTZaFw0yMjEyMTQyMjA3MTZa +MF8xCzAJBgNVBAYTAlVTMQwwCgYDVQQKEwNJQk0xFjAUBgNVBAsTDURlZmF1bHRO +b2RlMDExFjAUBgNVBAsTDURlZmF1bHRDZWxsMDExEjAQBgNVBAMTCWxvY2FsaG9z +dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMv5HCsJZIpI5zCy+jXV +z6lmzNc9UcVSEEHn86h6zT6pxuY90TYeAhlZ9hZ+SCKn4OQ4GoDRZhLPTkYDt+wW +CV3NTIy9uCGUSJ6xjCKoxClJmgSQdg5m4HzwfY4ofoEZ5iZQ0Zmt62jGRWc0zuxj +hegnM+eO2reBJYu6Ypa9RPJdYJsmn1RNnC74IDY8Y95qn+WZj//UALCpYfX41hko +i7TWD9GKQO8SBmAxhjCDifOxVBokoxYrNdzESl0LXvnzEadeZTd9BfUtTaBHhx6t +njqqCPrbTY+3jAbZFd4RiERPnhLVKMytw5ot506BhPrUtpr2lusbN5svNXjuLeea +MMUCAwEAAaOBoDCBnTATBgNVHSMEDDAKgAhOatpLwvJFqjAdBgNVHSUEFjAUBggr +BgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4E+UHJvZmlsZVVVSUQ6QXBwU3J2 +MDEtQkFTRS05MDkzMzJjMC1iNmFiLTQ2OTMtYWI5NC01Mjc1ZDI1MmFmNDiCCWxv +Y2FsaG9zdDARBgNVHQ4ECgQITzqhA5sO8O4wDQYJKoZIhvcNAQELBQADggEBAKR0 +gY/BM69S6BDyWp5dxcpmZ9FS783FBbdUXjVtTkQno+oYURDrhCdsfTLYtqUlP4J4 +CHoskP+MwJjRIoKhPVQMv14Q4VC2J9coYXnePhFjE+6MaZbTjq9WaekGrpKkMaQA +iQt5b67jo7y63CZKIo9yBvs7sxODQzDn3wZwyux2vPegXSaTHR/rop/s/mPk3YTS +hQprs/IVtPoWU4/TsDN3gIlrAYGbcs29CAt5q9MfzkMmKsuDkTZD0ry42VjxjAmk +xw23l/k8RoD1wRWaDVbgpjwSzt+kl+vJE/ip2w3h69eEZ9wbo6scRO5lCO2JM4Pr +7RhLQyWn2u00L7/9Omw= +-----END CERTIFICATE----- +``` +==== + +[discrete#es-connectors-postgresql-client-docker] +==== Deployment using Docker + +include::_connectors-docker-instructions.asciidoc[] + +[discrete#es-connectors-postgresql-client-documents-syncs] +==== Documents and syncs + +* Tables must be owned by a PostgreSQL user. +* Tables with no primary key defined are skipped. +* To fetch the last updated time in PostgreSQL, `track_commit_timestamp` must be set to `on`. +Otherwise, all data will be indexed in every sync. + +[NOTE] +==== +* Files bigger than 10 MB won't be extracted. +* Permissions are not synced. +**All documents** indexed to an Elastic deployment will be visible to **all users with access** to that Elastic Deployment. +==== + +[discrete#es-connectors-postgresql-client-sync-rules] +==== Sync rules + +//sync-rules-basic,Basic sync rules are identical for all connectors and are available by default. + +[discrete#es-connectors-postgresql-client-sync-rules-advanced] +===== Advanced sync rules + +[NOTE] +==== +A //connectors-sync-types-full, full sync is required for advanced sync rules to take effect. +==== + +Advanced sync rules are defined through a source-specific DSL JSON snippet. + +[discrete#es-connectors-postgresql-client-sync-rules-advanced-example-data] +====== Example data + +Here is some example data that will be used in the following examples. + +[discrete#es-connectors-postgresql-client-sync-rules-advanced-example-data-1] +*`employee` table* + +[cols="3*", options="header"] +|=== +| emp_id | name | age +| 3 | John | 28 +| 10 | Jane | 35 +| 14 | Alex | 22 +|=== + +[discrete#es-connectors-postgresql-client-sync-rules-advanced-example-2] +*`customer` table* + +[cols="3*", options="header"] +|=== +| c_id | name | age +| 2 | Elm | 24 +| 6 | Pine | 30 +| 9 | Oak | 34 +|=== + +[discrete#es-connectors-postgresql-client-sync-rules-advanced-examples] +====== Advanced sync rules examples + +[discrete#es-connectors-postgresql-client-sync-rules-advanced-examples-1] +*Multiple table queries* + +[source,js] +---- +[ + { + "tables": [ + "employee" + ], + "query": "SELECT * FROM employee" + }, + { + "tables": [ + "customer" + ], + "query": "SELECT * FROM customer" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-postgresql-client-sync-rules-advanced-examples-1-id-columns] +*Multiple table queries with `id_columns`* + +In 8.15.0, we added a new optional `id_columns` field in our advanced sync rules for the PostgreSQL connector. +Use the `id_columns` field to ingest tables which do not have a primary key. Include the names of unique fields so that the connector can use them to generate unique IDs for documents. + +[source,js] +---- +[ + { + "tables": [ + "employee" + ], + "query": "SELECT * FROM employee", + "id_columns": ["emp_id"] + }, + { + "tables": [ + "customer" + ], + "query": "SELECT * FROM customer", + "id_columns": ["c_id"] + } +] +---- +// NOTCONSOLE + +This example uses the `id_columns` field to specify the unique fields `emp_id` and `c_id` for the `employee` and `customer` tables, respectively. + +[discrete#es-connectors-postgresql-client-sync-rules-advanced-examples-2] +*Filtering data with `WHERE` clause* + +[source,js] +---- +[ + { + "tables": ["employee"], + "query": "SELECT * FROM employee WHERE emp_id > 5" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-postgresql-client-sync-rules-advanced-examples-3] +*`JOIN` operations* + +[source,js] +---- +[ + { + "tables": ["employee", "customer"], + "query": "SELECT * FROM employee INNER JOIN customer ON employee.emp_id = customer.c_id" + } +] +---- +// NOTCONSOLE + +[WARNING] +==== +When using advanced rules, a query can bypass the configuration field `tables`. +This will happen if the query specifies a table that doesn't appear in the configuration. +This can also happen if the configuration specifies `*` to fetch all tables while the advanced sync rule requests for only a subset of tables. +==== + +[discrete#es-connectors-postgresql-client-client-operations-testing] +==== End-to-end testing + +The connector framework enables operators to run functional tests against a real data source. +Refer to <> for more details. + +To perform E2E testing for the PostgreSQL connector, run the following command: + +[source,shell] +---- +$ make ftest NAME=postgresql +---- + +For faster tests, add the `DATA_SIZE=small` flag: + +[source,shell] +---- +make ftest NAME=postgresql DATA_SIZE=small +---- + +[discrete#es-connectors-postgresql-client-known-issues] +==== Known issues + +There are no known issues for this connector. +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-postgresql-client-troubleshooting] +==== Troubleshooting + +See <>. + +[discrete#es-connectors-postgresql-client-security] +==== Security + +See <>. + +// Closing the collapsible section +=============== \ No newline at end of file diff --git a/docs/reference/connector/docs/connectors-redis.asciidoc b/docs/reference/connector/docs/connectors-redis.asciidoc new file mode 100644 index 0000000000000..7aad7b0b41497 --- /dev/null +++ b/docs/reference/connector/docs/connectors-redis.asciidoc @@ -0,0 +1,293 @@ +[#es-connectors-redis] +=== Redis connector reference +++++ +Redis +++++ + +// Attributes (AKA variables) used in this file +:service-name: Redis +:service-name-stub: redis + +The Redis connector is built with the Elastic connectors Python framework and is available as a self-managed <>. +View the {connectors-python}/connectors/sources/{service-name-stub}.py[*source code* for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + +[discrete#es-connectors-redis-connector-availability-and-prerequisites] +==== Availability and prerequisites + +This connector was introduced in Elastic *8.13.0*, available as a *self-managed* self-managed connector. + +To use this connector, satisfy all <>. +Importantly, you must deploy the connectors service on your own infrastructure. +You have two deployment options: + +* <>. Use this option if you're comfortable working with Python and want to iterate quickly locally. +* <>. Use this option if you want to deploy the connectors to a server, or use a container orchestration platform. + +[NOTE] +==== +This connector is in *technical preview* and is subject to change. The design and code is less mature than official GA features and is being provided as-is with no warranties. Technical preview features are not subject to the support SLA of official GA features. +==== + +[discrete#es-connectors-redis-connector-usage] +==== Usage + +To set up this connector in the UI, select the *Redis* tile when creating a new connector under *Search -> Connectors*. + +For additional operations, see <>. + +[discrete#es-connectors-redis-connector-docker] +==== Deploy with Docker + +include::_connectors-docker-instructions.asciidoc[] + +[discrete#es-connectors-redis-connector-configuration] +==== Configuration + +`host` (required):: +The IP of your Redis server/cloud. Example: + +* `127.0.0.1` +* `redis-12345.us-east-1.ec2.cloud.redislabs.com` + +`port` (required):: +Port where the Redis server/cloud instance is hosted. Example: + +* `6379` + +`username` (optional):: +Username for your Redis server/cloud. Example: + +* `default` + +`password` (optional):: +Password for your Redis server/cloud instance. Example: + +* `changeme` + +`database` (required):: +List of database index for your Redis server/cloud. * will fetch data from all databases. Example: + +* `0,1,2` +* `*` ++ +[NOTE] +==== +This field is ignored when using advanced sync rules. +==== + +`ssl_enabled`:: +Toggle to use SSL/TLS. Disabled by default. + +`mutual_tls_enabled`:: +Toggle to use secure mutual SSL/TLS. Ensure that your Redis deployment supports mutual SSL/TLS connections. Disabled by default. Depends on `ssl_enabled`. + +`tls_certfile`:: +Specifies the certificate from the Certificate Authority. The value of the certificate is used to validate the certificate presented by the Redis instance. Depends on `mutual_tls_enabled`. + +`tls_keyfile`:: +Specifies the client private key. The value of the key is used to validate the connection in the Redis instance. +Depends on `mutual_tls_enabled`. + +[discrete#es-connectors-redis-connector-documents-and-syncs] +==== Documents and syncs + +The connector syncs the following objects and entities: + +* KEYS and VALUES of every database index + + +[NOTE] +==== +* Permissions are not synced. *All documents* indexed to an Elastic deployment will be visible to *all users with access* to the relevant Elasticsearch index. +==== + +[discrete#es-connectors-redis-connector-sync-rules] +==== Sync rules +<> are identical for all connectors and are available by default. + + +[discrete#es-connectors-redis-connector-advanced-sync-rules] +==== Advanced Sync Rules + +<> are defined through a source-specific DSL JSON snippet. + +Use advanced sync rules to filter data at the Redis source, without needing to index all data into Elasticsearch. + +They take the following parameters: + +1. `database`: Specify the Redis database index as an integer value. +2. `key_pattern`: 2. `key_pattern`: Pattern for finding keys in Redis. +3. `type`: Key type for the Redis. ++ +Supported values: + +* `HASH` +* `LIST` +* `SET` +* `STREAM` +* `STRING` +* `ZSET` + +[NOTE] +==== +Provide at least one of the following: `key_pattern` or `type`, or both. +==== + +[discrete#es-connectors-redis-connector-advanced-sync-rules-examples] +===== Advanced sync rules examples + +[discrete#es-connectors-redis-connector-advanced-sync-rules-example-1] +====== Example 1 + +*Fetch database records where keys start with `alpha`*: + +[source,js] +---- +[ + { + "database": 0, + "key_pattern": "alpha*" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-redis-connector-advanced-sync-rules-example-2] +====== Example 2 + +*Fetch database records with exact match by specifying the full key name:* + +[source,js] +---- +[ + { + "database": 0, + "key_pattern": "alpha" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-redis-connector-advanced-sync-rules-example-3] +====== Example 3 + +*Fetch database records where keys start with `test1`, `test2` or `test3`:* + +[source,js] +---- +[ + { + "database": 0, + "key_pattern": "test[123]" + } + +---- +// NOTCONSOLE + +[discrete#es-connectors-redis-connector-advanced-sync-rules-example-4] +====== Example 4 + +*Exclude database records where keys start with `test1`, `test2` or `test3`:* + +[source,js] +---- +[ + { + "database": 0, + "key_pattern": "test[^123]" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-redis-connector-advanced-sync-rules-example-5] +====== Example 5 + +*Fetch all database records:* + +[source,js] +---- +[ + { + "database": 0, + "key_pattern": "*" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-redis-connector-advanced-sync-rules-example-6] +====== Example 6 + +*Fetch all database records where type is `SET`:* + +[source,js] +---- +[ + { + "database": 0, + "key_pattern": "*", + "type": "SET" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-redis-connector-advanced-sync-rules-example-7] +====== Example 7 + +*Fetch database records where type is `SET`*: + +[source,js] +---- +[ + { + "database": 0, + "type": "SET" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-redis-connector-connector-client-operations] +==== Connector Client operations + +[discrete#es-connectors-redis-connector-end-to-end-testing] +===== End-to-end Testing + +The connector framework enables operators to run functional tests against a real data source, using Docker Compose. +You don't need a running Elasticsearch instance or Redis source to run this test. + +Refer to <> for more details. + +To perform E2E testing for the Redis connector, run the following command: + +```shell +$ make ftest NAME=redis +``` +For faster tests, add the `DATA_SIZE=small` flag: + +[source,shell] +---- +make ftest NAME=redis DATA_SIZE=small +---- + +By default, `DATA_SIZE=MEDIUM`. + +[discrete#es-connectors-redis-connector-known-issues] +==== Known issues + +* The last modified time is unavailable when retrieving keys/values from the Redis database. +As a result, *all objects* are indexed each time an advanced sync rule query is executed. + +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-redis-connector-troubleshooting] +==== Troubleshooting + +See <>. + +[discrete#es-connectors-redis-connector-security] +==== Security + +See <>. \ No newline at end of file diff --git a/docs/reference/connector/docs/connectors-refs.asciidoc b/docs/reference/connector/docs/connectors-refs.asciidoc new file mode 100644 index 0000000000000..51580410d5bc5 --- /dev/null +++ b/docs/reference/connector/docs/connectors-refs.asciidoc @@ -0,0 +1,64 @@ +[#es-connectors-refs] +== Connectors references + +The following pages contain the reference documentation for each individual connector: + +include::_connectors-list-clients.asciidoc[] + +include::connectors-azure-blob.asciidoc[] + +include::connectors-box.asciidoc[] + +include::connectors-confluence.asciidoc[] + +include::connectors-dropbox.asciidoc[] + +include::connectors-github.asciidoc[] + +include::connectors-gmail.asciidoc[] + +include::connectors-google-cloud.asciidoc[] + +include::connectors-google-drive.asciidoc[] + +include::connectors-graphql.asciidoc[] + +include::connectors-jira.asciidoc[] + +include::connectors-ms-sql.asciidoc[] + +include::connectors-mongodb.asciidoc[] + +include::connectors-mysql.asciidoc[] + +include::connectors-network-drive.asciidoc[] + +include::connectors-notion.asciidoc[] + +include::connectors-onedrive.asciidoc[] + +include::connectors-opentext-documentum.asciidoc[] + +include::connectors-oracle.asciidoc[] + +include::connectors-outlook.asciidoc[] + +include::connectors-postgresql.asciidoc[] + +include::connectors-redis.asciidoc[] + +include::connectors-s3.asciidoc[] + +include::connectors-salesforce.asciidoc[] + +include::connectors-servicenow.asciidoc[] + +include::connectors-sharepoint-online.asciidoc[] + +include::connectors-sharepoint.asciidoc[] + +include::connectors-slack.asciidoc[] + +include::connectors-teams.asciidoc[] + +include::connectors-zoom.asciidoc[] \ No newline at end of file diff --git a/docs/reference/connector/docs/connectors-release-notes.asciidoc b/docs/reference/connector/docs/connectors-release-notes.asciidoc new file mode 100644 index 0000000000000..ff3d859e1a888 --- /dev/null +++ b/docs/reference/connector/docs/connectors-release-notes.asciidoc @@ -0,0 +1,84 @@ +[#es-connectors-release-notes] +== Connector release notes +++++ +Release notes +++++ + +[NOTE] +==== +Prior to version *8.16.0*, the connector release notes were published as part of the {enterprise-search-ref}/changelog.html[Enterprise Search documentation]. +==== + +[discrete] +[[es-connectors-release-notes-8-17-0]] +=== 8.17.0 + +No notable changes in this release. + +[discrete] +[[es-connectors-release-notes-8-16-1]] +=== 8.16.1 + +[discrete] +[[es-connectors-release-notes-8-16-1-bug-fixes]] +==== Bug fixes + +* Fixed a bug in the Outlook Connector where having deactivated users could cause the sync to fail. +See https://github.com/elastic/connectors/pull/2967[*PR 2967*]. +* Fixed a bug where the Confluence connector was not downloading some blog post documents due to unexpected response format. +See https://github.com/elastic/connectors/pull/2984[*PR 2984*]. + +[discrete] +[[es-connectors-release-notes-8-16-0]] +=== 8.16.0 + +[discrete] +[[es-connectors-release-notes-deprecation-notice]] +==== Deprecation notices + +* *Direct index access for connectors and sync jobs* ++ +IMPORTANT: Directly accessing connector and sync job state through `.elastic-connectors*` indices is deprecated, and will be disallowed entirely in a future release. + +* Instead, the Elasticsearch Connector APIs should be used. Connectors framework code now uses the <> by default. +See https://github.com/elastic/connectors/pull/2884[*PR 2902*]. + +* *Docker `enterprise-search` namespace deprecation* ++ +IMPORTANT: The `enterprise-search` Docker namespace is deprecated and will be discontinued in a future release. ++ +Starting in `8.16.0`, Docker images are being transitioned to the new `integrations` namespace, which will become the sole location for future releases. This affects the https://github.com/elastic/connectors[Elastic Connectors] and https://github.com/elastic/data-extraction-service[Elastic Data Extraction Service]. ++ +During this transition period, images are published to both namespaces: ++ +** *Example*: ++ +Deprecated namespace:: +`docker.elastic.co/enterprise-search/elastic-connectors:v8.16.0` ++ +New namespace:: +`docker.elastic.co/integrations/elastic-connectors:v8.16.0` ++ +Users should migrate to the new `integrations` namespace as soon as possible to ensure continued access to future releases. + +[discrete] +[[es-connectors-release-notes-8-16-0-enhancements]] +==== Enhancements + +* Docker images now use Chainguard's Wolfi base image (`docker.elastic.co/wolfi/jdk:openjdk-11-dev`), replacing the previous `ubuntu:focal` base. + +* The Sharepoint Online connector now works with the `Sites.Selected` permission instead of the broader permission `Sites.Read.All`. +See https://github.com/elastic/connectors/pull/2762[*PR 2762*]. + +* Starting in 8.16.0, connectors will start using proper SEMVER, with `MAJOR.MINOR.PATCH`, which aligns with Elasticsearch/Kibana versions. This drops the previous `.BUILD` suffix, which we used to release connectors between Elastic stack releases. Going forward, these inter-stack-release releases will be suffixed instead with `+`, aligning with Elastic Agent and conforming to SEMVER. +See https://github.com/elastic/connectors/pull/2749[*PR 2749*]. + +* Connector logs now use UTC timestamps, instead of machine-local timestamps. This only impacts logging output. +See https://github.com/elastic/connectors/pull/2695[*PR 2695*]. + +[discrete] +[[es-connectors-release-notes-8-16-0-bug-fixes]] +==== Bug fixes + +* The Dropbox connector now fetches the files from team shared folders. +See https://github.com/elastic/connectors/pull/2718[*PR 2718*]. diff --git a/docs/reference/connector/docs/connectors-run-from-docker.asciidoc b/docs/reference/connector/docs/connectors-run-from-docker.asciidoc new file mode 100644 index 0000000000000..1688b945f05bc --- /dev/null +++ b/docs/reference/connector/docs/connectors-run-from-docker.asciidoc @@ -0,0 +1,95 @@ +[#es-connectors-run-from-docker] +=== Running from a Docker container + +[TIP] +==== +Use our <> to quickly get started with a full Elastic Stack deployment using Connectors. +==== + +Instead of running the Connectors Service from source, you can use the official Docker image to run the service in a container. + +As a prerequisite, you need to have an Elasticsearch and Kibana instance running. +From inside your Kibana UI, You will need to <> in the same manner as if you are running the service from source. + +When you are ready to run Connectors: + +*Step 1: Download sample configuration file* + +Download the sample configuration file. +You can either download it manually or run the following command: + +[source,sh] +---- +curl https://raw.githubusercontent.com/elastic/connectors/main/config.yml.example --output /connectors-config/config.yml +---- +// NOTCONSOLE + +Don't forget to change the `--output` argument value to the path where you want to save the `config.yml` file on your local system. +But keep note of where you wrote this file, as it is required in the `docker run` step below. + +*Step 2: Update the configuration file for your self-managed connector* + +Update the configuration file with the following settings to match your environment: + +* `elasticsearch.host` +* `elasticsearch.api_key` +* `connectors` + +If you're running the connector service against a Dockerized version of Elasticsearch and Kibana, your config file will look like this: + +[source,yaml,subs="attributes"] +---- +# When connecting to your cloud deployment you should edit the host value +elasticsearch.host: http://host.docker.internal:9200 +elasticsearch.api_key: + +connectors: + - + connector_id: + service_type: {service-name} # sharepoint_online (example) + api_key: # Optional. If not provided, the connector will use the elasticsearch.api_key instead + +---- + +*Step 3: Run the Docker image* + +Run the Docker image with the Connector Service using the following command: + +[source,sh,subs="attributes"] +---- +docker run \ +-v "/connectors-config:/config" \ # NOTE: you must change this path to match where the config.yml is located +--rm \ +--tty -i \ +--network host \ +docker.elastic.co/enterprise-search/elastic-connectors:{version}.0 \ +/app/bin/elastic-ingest \ +-c /config/config.yml +---- + +[TIP] +==== +For unreleased versions, append the `-SNAPSHOT` suffix to the version number. +For example, `docker.elastic.co/enterprise-search/elastic-connectors:8.14.0.0-SNAPSHOT`. +==== + +Find all available Docker images in the https://www.docker.elastic.co/r/enterprise-search/elastic-connectors[official registry]. + +[discrete#es-build-connector-finalizes-kibana] +==== Enter data source details in Kibana + +Once the connector service is running, it's time to head back to the Kibana UI to finalize the connector configuration. +In this step, you need to add the specific connection details about your data source instance, like URL, authorization credentials, etc. +As mentioned above, these details will vary based on the third-party data source you’re connecting to. + +For example, the PostgreSQL connector requires the following details: + +* *Host* +* *Port* +* *Username* +* *Password* +* *Database* +* *Comma-separated list of tables* + +You're now ready to run a sync. +Select the *Full sync* button in the Kibana UI to start ingesting documents into Elasticsearch. diff --git a/docs/reference/connector/docs/connectors-run-from-source.asciidoc b/docs/reference/connector/docs/connectors-run-from-source.asciidoc new file mode 100644 index 0000000000000..a5d1dc31074f2 --- /dev/null +++ b/docs/reference/connector/docs/connectors-run-from-source.asciidoc @@ -0,0 +1,103 @@ +[#es-connectors-run-from-source] +=== Running from the source code + +The basic way to run connectors is to clone the repository and run the code locally. +This is a good option if you are comfortable with Python and want to iterate quickly. + +[discrete#es-connectors-run-from-source-setup-kibana] +==== Initial setup in Kibana + +Follow the *Connector* workflow in the Kibana UI to select the *Connector* ingestion method. + +Next, complete these steps: + +1. Choose which third-party service you’d like to use by selecting a *data source*. +2. Create and name a new *Elasticsearch index*. +3. Generate a new *API key* and save it somewhere safe. +4. Name your connector and provide an optional description +5. *Convert* managed connector to a self-managed connector (_Only applicable if connector is also available natively_). This action is irreversible. +6. Copy the configuration block from the example shown on the screen. You’ll use this in a later step: ++ +[source,yaml] +---- +# ... +connectors: + - connector_id: + api_key: # Scoped API key for this connector (optional). If not specified, the top-level `elasticsearch.api_key` value is used. + service_type: gmail # example +---- + +[discrete#es-connectors-run-from-source-source-clone] +====== Clone the repository and edit `config.yml` + +Once you've created an index, and entered the access details for your data source, you're ready to deploy the connector service. + +First, you need to clone the `elastic/connectors` repository. + +Follow these steps: + +* Clone or fork the `connectors` repository locally with the following command: `git clone https://github.com/elastic/connectors`. +* Run `make config` to generate your initial `config.yml` file +* Open the `config.yml` configuration file in your editor of choice. +* Replace the values for `host` (your Elasticsearch endpoint), `api_key`, `connector_id`, and `service_type`. ++ +.*Expand* to see an example `config.yml` file +[%collapsible] +==== +Replace the values for `api_key`, `connector_id`, and `service_type` with the values you copied earlier. +[source,yaml] +---- +elasticsearch: + api_key: # Used to write data to .elastic-connectors and .elastic-connectors-sync-jobs + # Any connectors without a specific `api_key` value will default to using this key +connectors: + - connector_id: 1234 + api_key: # Used to write data to the `search-*` index associated with connector 1234 + # You may have multiple connectors in your config file! + - connector_id: 5678 + api_key: # Used to write data to the `search-*` index associated with connector 5678 + - connector_id: abcd # No explicit api key specified, so this connector will use +---- + +[discrete#es-connectors-run-from-source-api-keys] +[NOTE] +===== +**API keys for connectors** + +You can configure multiple connectors in your `config.yml` file. + +The Kibana UI enables you to create API keys that are scoped to a specific index/connector. +If you don't create an API key for a specific connector, the top-level `elasticsearch.api_key` or `elasticsearch.username:elasticsearch.password` value is used. + +If these top-level Elasticsearch credentials are not sufficiently privileged to write to individual connector indices, you'll need to create these additional, scoped API keys. + +Use the example above as a guide. +===== +==== + +[discrete#es-connectors-run-from-source-run] +====== Run the connector service + +[NOTE] +==== +You need Python version `3.10` or `3.11` to run the connectors service from source. +==== + +Once you've configured the connector code, you can run the connector service. + +In your terminal or IDE: + +. `cd` into the root of your `connectors` clone/fork. +. Run the following commands to compile and run the connector service: ++ +[source,shell] +---- +make install +make run +---- + +The connector service should now be running. +The UI will let you know that the connector has successfully connected to your Elasticsearch instance. + +As a reminder, here we're working locally. +In a production setup, you'll deploy the connector service to your own infrastructure. diff --git a/docs/reference/connector/docs/connectors-s3.asciidoc b/docs/reference/connector/docs/connectors-s3.asciidoc new file mode 100644 index 0000000000000..90c070f7b8044 --- /dev/null +++ b/docs/reference/connector/docs/connectors-s3.asciidoc @@ -0,0 +1,437 @@ +[#es-connectors-s3] +=== Elastic S3 connector reference +++++ +S3 +++++ +// Attributes used in this file: +:service-name: Amazon S3 +:service-name-stub: s3 + +The _Elastic S3 connector_ is a <> for https://aws.amazon.com/s3/[Amazon S3^] data sources. + + +// //////// //// //// //// //// //// //// //////// +// //////// NATIVE CONNECTOR REFERENCE (MANAGED SERVICE) /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-s3-native-connector-reference] +==== *Elastic managed connector reference* + +.View *Elastic managed connector* reference +[%collapsible] +=============== + +[discrete#es-connectors-s3-prerequisites] +===== Availability and prerequisites + +This connector is available natively in Elastic Cloud as of version *8.12.0*. +To use this connector, satisfy all <>. + +[discrete#es-connectors-s3-create-native-connector] +===== Create a {service-name} connector +include::_connectors-create-native.asciidoc[] + +[discrete#es-connectors-s3-usage] +===== Usage + +To use this managed connector, see <>. + +For additional operations, see <>. + +S3 users will also need to <> + +[discrete#es-connectors-s3-usage-create-iam] +====== Create an IAM identity + +Users need to create an IAM identity to use this connector as a *self-managed connector*. +Refer to https://docs.aws.amazon.com/IAM/latest/UserGuide/getting-set-up.html[the AWS documentation^]. + +The https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html[policy^] associated with the IAM identity must have the following *AWS permissions*: + +* `ListAllMyBuckets` +* `ListBucket` +* `GetBucketLocation` +* `GetObject` + +[discrete#es-connectors-s3-compatibility] +===== Compatibility + +Currently the connector does not support S3-compatible vendors. + +[discrete#es-connectors-s3-configuration] +===== Configuration + +The following configuration fields are required to *set up* the connector: + +AWS Buckets:: +List of S3 bucket names. +`*` will fetch data from all buckets. +Examples: ++ +* `testbucket, prodbucket` +* `testbucket` +* `*` + +[NOTE] +==== +This field is ignored when using advanced sync rules. +==== + +AWS Access Key ID:: +Access Key ID for the AWS identity that will be used for bucket access. + +AWS Secret Key:: +Secret Access Key for the AWS identity that will be used for bucket access. + +[discrete#es-connectors-s3-documents-syncs] +===== Documents and syncs + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted. (Self-managed connectors can use the <> to handle larger binary files.) +* Permissions are not synced. +**All documents** indexed to an Elastic deployment will be visible to **all users with access** to that Elastic Deployment. +==== + +[discrete#es-connectors-s3-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +[discrete#es-connectors-s3-sync-rules-advanced] +====== Advanced sync rules + +[NOTE] +==== +A <> is required for advanced sync rules to take effect. +==== + +Advanced sync rules are defined through a source-specific DSL JSON snippet. + +Use advanced sync rules to filter data to be fetched from Amazon S3 buckets. +They take the following parameters: + +1. `bucket`: S3 bucket the rule applies to. +2. `extension` (optional): Lists which file types to sync. Defaults to syncing all types. +3. `prefix` (optional): String of prefix characters. +The connector will fetch file and folder data that matches the string. +Defaults to `""` (syncs all bucket objects). + +[discrete#es-connectors-s3-sync-rules-advanced-examples] +*Advanced sync rules examples* + +*Fetching files and folders recursively by prefix* + +*Example*: Fetch files/folders in `folder1/docs`. + +[source,js] +---- +[ + { + "bucket": "bucket1", + "prefix": "folder1/docs" + } + +] +---- +// NOTCONSOLE + +*Example*: Fetch files/folder starting with `folder1`. + +[source,js] +---- +[ + { + "bucket": "bucket2", + "prefix": "folder1" + } +] +---- +// NOTCONSOLE + +*Fetching files and folders by specifying extensions* + +*Example*: Fetch all objects which start with `abc` and then filter using file extensions. + +[source,js] +---- +[ + { + "bucket": "bucket2", + "prefix": "abc", + "extension": [".txt", ".png"] + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-s3-content-extraction] +===== Content extraction + +See <>. + +[discrete#es-connectors-s3-known-issues] +===== Known issues + +There are no known issues for this connector. + +See <> for any issues affecting all connectors. + +[discrete#es-connectors-s3-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-s3-security] +===== Security + +See <>. + +[discrete#es-connectors-s3-source] +===== Framework and source + +This connector is built with the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/s3.py[source code for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + + +// Closing the collapsible section +=============== + + +// //////// //// //// //// //// //// //// //////// +// //////// CONNECTOR CLIENT REFERENCE (SELF-MANAGED) /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-s3-connector-client-reference] +==== *Self-managed connector reference* + +.View *self-managed connector* reference +[%collapsible] +=============== + +[discrete#es-connectors-s3-client-prerequisites] +===== Availability and prerequisites + +This connector is available as a self-managed *self-managed connector*. +This self-managed connector is compatible with Elastic versions *8.6.0+*. +To use this connector, satisfy all <>. + +[discrete#es-connectors-s3-create-connector-client] +===== Create a {service-name} connector +include::_connectors-create-client.asciidoc[] + +[discrete#es-connectors-s3-client-usage] +===== Usage + +To use this connector as a *self-managed connector*, see <>. + +For additional operations, see <>. + +S3 users will also need to <> + +[discrete#es-connectors-s3-client-usage-create-iam] +====== Create an IAM identity + +Users need to create an IAM identity to use this connector as a *self-managed connector*. +Refer to https://docs.aws.amazon.com/IAM/latest/UserGuide/getting-set-up.html[the AWS documentation^]. + +The https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html[policy^] associated with the IAM identity must have the following *AWS permissions*: + +* `ListAllMyBuckets` +* `ListBucket` +* `GetBucketLocation` +* `GetObject` + +[discrete#es-connectors-s3-client-compatibility] +===== Compatibility + +Currently the connector does not support S3-compatible vendors. + +[discrete#es-connectors-s3-client-configuration] +===== Configuration + +[TIP] +==== +When using the <> workflow, these fields will use the default configuration set in the https://github.com/elastic/connectors/blob/a5976d20cd8277ae46511f7176662afc889e56ec/connectors/sources/s3.py#L231-L258[connector source code^]. +These configurable fields will be rendered with their respective *labels* in the Kibana UI. +Once connected, you'll be able to update these values in Kibana. +==== + +The following configuration fields are required to *set up* the connector: + +`buckets`:: +List of S3 bucket names. +`*` will fetch data from all buckets. +Examples: ++ +* `testbucket, prodbucket` +* `testbucket` +* `*` + +[NOTE] +==== +This field is ignored when using advanced sync rules. +==== + +`aws_access_key_id`:: +Access Key ID for the AWS identity that will be used for bucket access. + +`aws_secret_access_key`:: +Secret Access Key for the AWS identity that will be used for bucket access. + +`read_timeout`:: +The `read_timeout` for Amazon S3. +Default value is `90`. + +`connect_timeout`:: +Connection timeout for crawling S3. +Default value is `90`. + +`max_attempts`:: +Maximum retry attempts. +Default value is `5`. + +`page_size`:: +Page size for iterating bucket objects in Amazon S3. +Default value is `100`. + +[discrete#es-connectors-s3-client-docker] +===== Deployment using Docker + +include::_connectors-docker-instructions.asciidoc[] + +[discrete#es-connectors-s3-client-documents-syncs] +===== Documents and syncs + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted by default. You can use the <> to handle larger binary files. +* Permissions are not synced. +**All documents** indexed to an Elastic deployment will be visible to **all users with access** to that Elastic Deployment. +==== + +[discrete#es-connectors-s3-client-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +[discrete#es-connectors-s3-client-sync-rules-advanced] +====== Advanced sync rules + +[NOTE] +==== +A <> is required for advanced sync rules to take effect. +==== + +Advanced sync rules are defined through a source-specific DSL JSON snippet. + +Use advanced sync rules to filter data to be fetched from Amazon S3 buckets. +They take the following parameters: + +1. `bucket`: S3 bucket the rule applies to. +2. `extension` (optional): Lists which file types to sync. Defaults to syncing all types. +3. `prefix` (optional): String of prefix characters. +The connector will fetch file and folder data that matches the string. +Defaults to `""` (syncs all bucket objects). + +[discrete#es-connectors-s3-client-sync-rules-advanced-examples] +*Advanced sync rules examples* + +*Fetching files and folders recursively by prefix* + +*Example*: Fetch files/folders in `folder1/docs`. + +[source,js] +---- +[ + { + "bucket": "bucket1", + "prefix": "folder1/docs" + } + +] +---- +// NOTCONSOLE + +*Example*: Fetch files/folder starting with `folder1`. + +[source,js] +---- +[ + { + "bucket": "bucket2", + "prefix": "folder1" + } +] +---- +// NOTCONSOLE + +*Fetching files and folders by specifying extensions* + +*Example*: Fetch all objects which start with `abc` and then filter using file extensions. + +[source,js] +---- +[ + { + "bucket": "bucket2", + "prefix": "abc", + "extension": [".txt", ".png"] + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-s3-client-content-extraction] +===== Content extraction + +See <>. + +[discrete#es-connectors-s3-client-testing] +===== End-to-end testing + +The connector framework enables operators to run functional tests against a real data source. +Refer to <> for more details. + +To execute a functional test for the Amazon S3 *self-managed connector*, run the following command: + +[source,shell] +---- +make ftest NAME=s3 +---- + +By default, this will use a medium-sized dataset. +To make the test faster add the `DATA_SIZE=small` argument: + +[source,shell] +---- +make ftest NAME=s3 DATA_SIZE=small +---- + +[discrete#es-connectors-s3-client-known-issues] +===== Known issues + +There are no known issues for this connector. + +See <> for any issues affecting all connectors. + +[discrete#es-connectors-s3-client-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-s3-client-security] +===== Security + +See <>. + +[discrete#es-connectors-s3-client-source] +===== Framework and source + +This connector is built with the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/s3.py[source code for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + +// Closing the collapsible section +=============== diff --git a/docs/reference/connector/docs/connectors-salesforce.asciidoc b/docs/reference/connector/docs/connectors-salesforce.asciidoc new file mode 100644 index 0000000000000..f5c5512ad5cc4 --- /dev/null +++ b/docs/reference/connector/docs/connectors-salesforce.asciidoc @@ -0,0 +1,813 @@ +[#es-connectors-salesforce] +=== Elastic Salesforce connector reference +++++ +Salesforce +++++ +// Attributes used in this file +:service-name: Salesforce +:service-name-stub: salesforce + +The _Elastic Salesforce connector_ is a <> for https://www.salesforce.com/[Salesforce^] data sources. + + +// //////// //// //// //// //// //// //// //////// +// //////// NATIVE CONNECTOR REFERENCE (MANAGED SERVICE) /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-salesforce-native-connector-reference] +==== *Elastic managed connector reference* + +.View *Elastic managed connector* reference +[%collapsible] +=============== + +[discrete#es-connectors-salesforce-availability-prerequisites] +===== Availability and prerequisites + +This connector is available as a *managed connector* in Elastic Cloud since *8.12.0*. +To use this connector, satisfy all <>. + +[discrete#es-connectors-salesforce-compatability] +===== Compatibility + +This connector is compatible with the following: + +* Salesforce +* Salesforce Sandbox + +[discrete#es-connectors-salesforce-create-native-connector] +===== Create a {service-name} connector +include::_connectors-create-native.asciidoc[] + +[discrete#es-connectors-salesforce-usage] +===== Usage + +To use this connector as a **self-managed connector**, use the *Connector* workflow in the Kibana UI. + +For additional operations, see <>. + +[NOTE] +==== +You need to create an Salesforce connected app with OAuth2.0 enabled to authenticate with Salesforce. +==== + +[discrete#es-connectors-salesforce-connected-app] +====== Create a Salesforce connected app + +The Salesforce connector authenticates with Salesforce through a *connected app*. +Follow the official Salesforce documentation for https://help.salesforce.com/s/articleView?id=sf.connected_app_client_credentials_setup.htm[Configuring a Connected App for the OAuth 2.0 Client Credentials Flow^]. + +When creating the connected app, in the section titled *API (Enable OAuth Settings)* ensure the following settings are _enabled_: + +* *Enable OAuth Settings* +* *Enable for Device Flow* +** *Callback URL* should be the Salesforce dummy callback URL, `https://test.salesforce.com/services/oauth2/success` +* *Require Secret for Web Server Flow* +* *Require Secret for Refresh Token Flow* +* *Enable Client Credentials Flow* + +All other options should be disabled. +Finally, in the section *Selected OAuth Scopes*, include the following OAuth scopes: + +* *Manage user data via APIs (api)* +* *Perform requests at any time (refresh_token, offline_access)* + +[discrete#es-connectors-salesforce-admin-prerequisites] +===== Salesforce admin requirements + +By default, the Salesforce connector requires global administrator permissions to access Salesforce data. +Expand the section below to learn how to create a custom Salesforce user with minimal permissions. + +.*Create a custom Salesforce user with minimal permissions* +[%collapsible] +========================== +By creating a custom profile with sufficient permissions from the Setup menu, you can remove the system administrator role requirement for fetching data from Salesforce. + +To create a new profile: + +1. From the Salesforce Setup menu, go to *Administration => Users => Profiles*. +2. Create a new profile. +3. Choose `Read Only` or `Standard User` from the *Existing Profile* dropdown. Name the profile and save it. ++ +[TIP] +==== +By default, `Read Only` or `Standard User` users have read permission to access all standard objects. +==== ++ +4. Edit the newly created profile. Under *Object Permissions*, assign at least `Read` access to the standard objects and custom objects you want to ingest into Elasticsearch. +5. Make sure the newly created profile has at least `Read` access for the following standard objects: + +* Account +* Campaign +* Case +* Contact +* EmailMessage +* Lead +* Opportunity +* User ++ +[TIP] +==== +If using <> you'll need to assign `Read` access for that specific object in the profile. +==== ++ +6. Go to *Users => Profiles* and assign the newly created profile to the user. +7. Go to *Connected apps*, select your app and then select *Edit policies*. +Assign the client credentials flow to the user with the custom profile in Salesforce. ++ +Now, the connector can be configured for this user profile to fetch all object records, without needing the system administration role. +========================== + +[discrete#es-connectors-salesforce-configuration] +===== Configuration + +The following settings are required to set up this connector: + +Domain (required):: +The domain for your Salesforce account. +This is the subdomain that appears in your Salesforce URL. +For example, if your Salesforce URL is `foo.my.salesforce.com`, then your domain would be `foo`. +If you are using Salesforce Sandbox, your URL will contain an extra subdomain and will look similar to `foo.sandbox.my.salesforce.com`. +In this case, your domain would be `foo.sandbox`. + +Client ID (required):: +The Client ID generated by your connected app. +The Salesforce documentation will sometimes also call this a *Consumer Key* + +Client Secret (required):: +The Client Secret generated by your connected app. +The Salesforce documentation will sometimes also call this a *Consumer Secret*. + +Enable document level security:: +Toggle to enable document level security (DLS). +Optional, disabled by default. +Refer to the <> for more information, including how to set various Salesforce permission types. ++ +When enabled: + +* Full syncs will fetch access control lists for each document and store them in the `_allow_access_control` field. +* Access control syncs will fetch users' access control lists and store them in a separate index. + +[discrete#es-connectors-salesforce-configuration-credentials] +====== Finding the Client ID and Client Secret + +The Client ID and Client Secret are not automatically shown to you after you create a connected app. +You can find them by taking the following steps: + +* Navigate to *Setup* +* Go to *Platform Tools > Apps > App Manager* +* Click on the triangle next to your app and select *View* +* After the page loads, click on *Manage Consumer Details* + +Your Client ID and Client Secret should now be visible at the top of the page. + +[discrete#es-connectors-salesforce-dls] +===== Document level security (DLS) + +<> enables you to restrict access to documents based on a user'­s permissions. +This feature is available by default for the Salesforce connector and supports both *standard and custom objects*. + +Salesforce allows users to set permissions in the following ways: + +* *Profiles* +* *Permission sets* +* *Permission set Groups* + +For guidance, refer to these https://howtovideos.hubs.vidyard.com/watch/B1bQnMFg2VyZq7V6zXQjPg#:~:text=This%20is%20a%20must%20watch,records%20in%20your%20Salesforce%20organization[video tutorials] about setting Salesforce permissions. + +To ingest any standard or custom objects, users must ensure that at least `Read` permission is granted to that object. +This can be granted using any of the following methods for setting permissions. + +[discrete#es-connectors-salesforce-dls-profiles] +====== Set Permissions using Profiles + +Refer to the https://help.salesforce.com/s/articleView?id=sf.admin_userprofiles.htm&type=5[Salesforce documentation] for setting permissions via Profiles. + +[discrete#es-connectors-salesforce-dls-permission-sets] +====== Set Permissions using Permissions Set + +Refer to the https://help.salesforce.com/s/articleView?id=sf.perm_sets_overview.htm&language=en_US&type=5[Salesforce documentation] for setting permissions via Permissions Sets. + +[discrete#es-connectors-salesforce-dls-permission-set-groups] +====== Set Permissions using Permissions Set group + +Refer to the https://help.salesforce.com/s/articleView?id=sf.perm_set_groups.htm&type=5[Salesforce documentation] for setting permissions via Permissions Set Groups. + +[discrete#es-connectors-salesforce-dls-assign-permissions] +====== Assign Profiles, Permission Set and Permission Set Groups to the User + +Once the permissions are set, assign the Profiles, Permission Set or Permission Set Groups to the user. +Follow these steps in Salesforce: + +1. Navigate to `Administration` under the `Users` section. +2. Select `Users` and choose the user to set the permissions to. +3. Set the `Profile`, `Permission Set` or `Permission Set Groups` created in the earlier steps. + +[discrete#es-connectors-salesforce-sync-rules] +===== Sync rules + +_Basic_ sync rules are identical for all connectors and are available by default. +For more information read <>. + +[discrete#es-connectors-salesforce-sync-rules-advanced] +====== Advanced sync rules + +[NOTE] +==== +A <> is required for advanced sync rules to take effect. +==== + +The following section describes *advanced sync rules* for this connector. +Advanced sync rules enable filtering of data in Salesforce _before_ indexing into Elasticsearch. + +They take the following parameters: + +. `query` : Salesforce query to filter the documents. +. `language` : Salesforce query language. +Allowed values are *SOQL* and *SOSL*. + +[discrete#es-connectors-salesforce-sync-rules-advanced-fetch-query-language] +*Fetch documents based on the query and language specified* + +**Example**: Fetch documents using SOQL query + +[source,js] +---- +[ + { + "query": "SELECT Id, Name FROM Account", + "language": "SOQL" + } +] +---- +// NOTCONSOLE + +**Example**: Fetch documents using SOSL query. + +[source,js] +---- +[ + { + "query": "FIND {Salesforce} IN ALL FIELDS", + "language": "SOSL" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-salesforce-sync-rules-advanced-fetch-objects] +*Fetch standard and custom objects using SOQL and SOSL queries* + +**Example**: Fetch documents for standard objects via SOQL and SOSL query. + +[source,js] +---- +[ + { + "query": "SELECT Account_Id, Address, Contact_Number FROM Account", + "language": "SOQL" + }, + { + "query": "FIND {Alex Wilber} IN ALL FIELDS RETURNING Contact(LastModifiedDate, Name, Address)", + "language": "SOSL" + } +] +---- +// NOTCONSOLE + +**Example**: Fetch documents for custom objects via SOQL and SOSL query. + +[source,js] +---- +[ + { + "query": "SELECT Connector_Name, Version FROM Connector__c", + "language": "SOQL" + }, + { + "query": "FIND {Salesforce} IN ALL FIELDS RETURNING Connectors__c(Id, Connector_Name, Connector_Version)", + "language": "SOSL" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-salesforce-sync-rules-advanced-fetch-standard-custom-fields] +*Fetch documents with standard and custom fields* + +**Example**: Fetch documents with all standard and custom fields for Account object. + +[source,js] +---- +[ + { + "query": "SELECT FIELDS(ALL) FROM Account", + "language": "SOQL" + } +] +---- +// NOTCONSOLE + +**Example**: Fetch documents with all custom fields for Connector object. + +[source,js] +---- +[ + { + "query": "SELECT FIELDS(CUSTOM) FROM Connector__c", + "language": "SOQL" + } +] +---- +// NOTCONSOLE + +**Example**: Fetch documents with all standard fields for Account object. + +[source,js] +---- +[ + { + "query": "SELECT FIELDS(STANDARD) FROM Account", + "language": "SOQL" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-salesforce-documents-syncs] +===== Documents and syncs + +The connector syncs the following Salesforce objects: + +* *Accounts* +* *Campaigns* +* *Cases* +* *Contacts* +* *Content Documents* (files uploaded to Salesforce) +* *Leads* +* *Opportunities* + +The connector will not ingest any objects that it does not have permissions to query. + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted. (Self-managed connectors can use the <> to handle larger binary files.) +* Permissions are not synced by default. You must enable <>. Otherwise, **all documents** indexed to an Elastic deployment will be visible to **all users with access** to that Elastic Deployment. +==== + +[discrete#es-connectors-salesforce-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-salesforce-content-extraction] +===== Content Extraction + +The connector will retrieve Content Documents from your Salesforce source if they meet the following criteria: + +* Are attached to one or more objects that are synced +* Are of a file type that can be extracted + +This means that the connector will not ingest any Content Documents you have that are _not_ attached to a supported Salesforce object. +See <> for a list of supported object types. + +If a single Content Document is attached to multiple supported objects, only one Elastic document will be created for it. +This document will retain links to every object that it was connected to in the `related_ids` field. + +See <> for more specifics on content extraction. + +[discrete#es-connectors-salesforce-known-issues] +===== Known issues + +* *DLS feature is "type-level" not "document-level"* ++ +Salesforce DLS, added in 8.13.0, does not accomodate specific access controls to specific Salesforce Objects. +Instead, if a given user/group can have access to _any_ Objects of a given type (`Case`, `Lead`, `Opportunity`, etc), that user/group will appear in the `\_allow_access_control` list for _all_ of the Objects of that type. +See https://github.com/elastic/connectors/issues/3028 for more details. ++ + +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-salesforce-security] +===== Security + +See <>. + +[discrete#es-connectors-salesforce-source] +===== Framework and source + +This connector is built with the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/salesforce.py[source code for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + +// Closing the collapsible section +=============== + + +// //////// //// //// //// //// //// //// //////// +// //////// CONNECTOR CLIENT REFERENCE (SELF-MANAGED) /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-salesforce-connector-client-reference] +==== *Self-managed connector reference* + +.View *self-managed connector* reference +[%collapsible] +=============== + +[discrete#es-connectors-salesforce-client-availability-prerequisites] +===== Availability and prerequisites + +This connector is available as a self-managed *self-managed connector*. +This self-managed connector is compatible with Elastic versions *8.10.0+*. +To use this connector, satisfy all <>. + +[discrete#es-connectors-salesforce-client-compatability] +===== Compatibility + +This connector is compatible with the following: + +* Salesforce +* Salesforce Sandbox + +[discrete#es-connectors-salesforce-create-connector-client] +===== Create a {service-name} connector +include::_connectors-create-client.asciidoc[] + +[discrete#es-connectors-salesforce-client-usage] +===== Usage + +To use this connector as a **self-managed connector**, use the *Connector* workflow in the Kibana UI. + +For additional operations, see <>. + +[NOTE] +==== +You need to create an Salesforce connected app with OAuth2.0 enabled to authenticate with Salesforce. +==== + +[discrete#es-connectors-salesforce-client-connected-app] +====== Create a Salesforce connected app + +The Salesforce connector authenticates with Salesforce through a *connected app*. +Follow the official Salesforce documentation for https://help.salesforce.com/s/articleView?id=sf.connected_app_client_credentials_setup.htm[Configuring a Connected App for the OAuth 2.0 Client Credentials Flow^]. + +When creating the connected app, in the section titled *API (Enable OAuth Settings)* ensure the following settings are _enabled_: + +* *Enable OAuth Settings* +* *Enable for Device Flow* +** *Callback URL* should be the Salesforce dummy callback URL, `https://test.salesforce.com/services/oauth2/success` +* *Require Secret for Web Server Flow* +* *Require Secret for Refresh Token Flow* +* *Enable Client Credentials Flow* + +All other options should be disabled. +Finally, in the section *Selected OAuth Scopes*, include the following OAuth scopes: + +* *Manage user data via APIs (api)* +* *Perform requests at any time (refresh_token, offline_access)* + +[discrete#es-connectors-client-salesforce-admin-prerequisites] +===== Salesforce admin requirements + +By default, the Salesforce connector requires global administrator permissions to access Salesforce data. +Expand the section below to learn how to create a custom Salesforce user with minimal permissions. + +.*Create a custom Salesforce user with minimal permissions* +[%collapsible] +========================== +By creating a custom profile with sufficient permissions from the Setup menu, you can remove the system administrator role requirement for fetching data from Salesforce. + +To create a new profile: + +1. From the Salesforce Setup menu, go to *Administration => Users => Profiles*. +2. Create a new profile. +3. Choose `Read Only` or `Standard User` from the *Existing Profile* dropdown. Name the profile and save it. ++ +[TIP] +==== +By default, `Read Only` or `Standard User` users have read permission to access all standard objects. +==== ++ +4. Edit the newly created profile. Under *Object Permissions*, assign at least `Read` access to the standard objects and custom objects you want to ingest into Elasticsearch. +5. Make sure the newly created profile has at least `Read` access for the following standard objects: + +* Account +* Campaign +* Case +* Contact +* EmailMessage +* Lead +* Opportunity +* User ++ +[TIP] +==== +If using <> you'll need to assign `Read` access for that specific object in the profile. +==== ++ +6. Go to *Users => Profiles* and assign the newly created profile to the user. +7. Go to *Connected apps*, select your app and then select *Edit policies*. +Assign the client credentials flow to the user with the custom profile in Salesforce. ++ +Now, the connector can be configured for this user profile to fetch all object records, without needing the system administration role. +========================== + +[discrete#es-connectors-salesforce-client-docker] +===== Deployment using Docker + +Self-managed connectors are run on your own infrastructure. + +include::_connectors-docker-instructions.asciidoc[] + +[discrete#es-connectors-salesforce-client-configuration] +===== Configuration + +The following settings are required to set up this connector: + +`domain`(required):: +The domain for your Salesforce account. +This is the subdomain that appears in your Salesforce URL. +For example, if your Salesforce URL is `foo.my.salesforce.com`, then your domain would be `foo`. +If you are using Salesforce Sandbox, your URL will contain an extra subdomain and will look similar to `foo.sandbox.my.salesforce.com`. +In this case, your domain would be `foo.sandbox`. + +`client_id`(required):: +The Client ID generated by your connected app. +The Salesforce documentation will sometimes also call this a *Consumer Key* + +`client_secret`(required):: +The Client Secret generated by your connected app. +The Salesforce documentation will sometimes also call this a *Consumer Secret*. + +`use_document_level_security`:: +Toggle to enable document level security (DLS). +Optional, disabled by default. +Refer to the <> for more information, including how to set various Salesforce permission types. ++ +When enabled: + +* Full syncs will fetch access control lists for each document and store them in the `_allow_access_control` field. +* Access control syncs will fetch users' access control lists and store them in a separate index. + +[discrete#es-connectors-salesforce-client-configuration-credentials] +====== Finding the Client ID and Client Secret + +The Client ID and Client Secret are not automatically shown to you after you create a connected app. +You can find them by taking the following steps: + +* Navigate to *Setup* +* Go to *Platform Tools > Apps > App Manager* +* Click on the triangle next to your app and select *View* +* After the page loads, click on *Manage Consumer Details* + +Your Client ID and Client Secret should now be visible at the top of the page. + +[discrete#es-connectors-salesforce-client-dls] +===== Document level security (DLS) + +<> enables you to restrict access to documents based on a user'­s permissions. +This feature is available by default for the Salesforce connector and supports both *standard and custom objects*. + +Salesforce allows users to set permissions in the following ways: + +* *Profiles* +* *Permission sets* +* *Permission set Groups* + +For guidance, refer to these https://howtovideos.hubs.vidyard.com/watch/B1bQnMFg2VyZq7V6zXQjPg#:~:text=This%20is%20a%20must%20watch,records%20in%20your%20Salesforce%20organization[video tutorials] about setting Salesforce permissions. + +To ingest any standard or custom objects, users must ensure that at least `Read` permission is granted to that object. +This can be granted using any of the following methods for setting permissions. + +[discrete#es-connectors-salesforce-client-dls-profiles] +====== Set Permissions using Profiles + +Refer to the https://help.salesforce.com/s/articleView?id=sf.admin_userprofiles.htm&type=5[Salesforce documentation] for setting permissions via Profiles. + +[discrete#es-connectors-salesforce-client-dls-permission-sets] +====== Set Permissions using Permissions Set + +Refer to the https://help.salesforce.com/s/articleView?id=sf.perm_sets_overview.htm&language=en_US&type=5[Salesforce documentation] for setting permissions via Permissions Sets. + +[discrete#es-connectors-salesforce-client-dls-permission-set-groups] +====== Set Permissions using Permissions Set group + +Refer to the https://help.salesforce.com/s/articleView?id=sf.perm_set_groups.htm&type=5[Salesforce documentation] for setting permissions via Permissions Set Groups. + +[discrete#es-connectors-salesforce-client-dls-assign-permissions] +====== Assign Profiles, Permission Set and Permission Set Groups to the User + +Once the permissions are set, assign the Profiles, Permission Set or Permission Set Groups to the user. +Follow these steps in Salesforce: + +1. Navigate to `Administration` under the `Users` section. +2. Select `Users` and choose the user to set the permissions to. +3. Set the `Profile`, `Permission Set` or `Permission Set Groups` created in the earlier steps. + +[discrete#es-connectors-salesforce-client-sync-rules] +===== Sync rules + +_Basic_ sync rules are identical for all connectors and are available by default. + +For more information read <>. + +[discrete#es-connectors-salesforce-client-sync-rules-advanced] +====== Advanced sync rules + +[NOTE] +==== +A <> is required for advanced sync rules to take effect. +==== + +The following section describes *advanced sync rules* for this connector. +Advanced sync rules enable filtering of data in Salesforce _before_ indexing into Elasticsearch. + +They take the following parameters: + +. `query` : Salesforce query to filter the documents. +. `language` : Salesforce query language. +Allowed values are *SOQL* and *SOSL*. + +[discrete#es-connectors-salesforce-client-sync-rules-advanced-fetch-query-language] +*Fetch documents based on the query and language specified* + +**Example**: Fetch documents using SOQL query + +[source,js] +---- +[ + { + "query": "SELECT Id, Name FROM Account", + "language": "SOQL" + } +] +---- +// NOTCONSOLE + +**Example**: Fetch documents using SOSL query. + +[source,js] +---- +[ + { + "query": "FIND {Salesforce} IN ALL FIELDS", + "language": "SOSL" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-salesforce-client-sync-rules-advanced-fetch-objects] +*Fetch standard and custom objects using SOQL and SOSL queries* + +**Example**: Fetch documents for standard objects via SOQL and SOSL query. + +[source,js] +---- +[ + { + "query": "SELECT Account_Id, Address, Contact_Number FROM Account", + "language": "SOQL" + }, + { + "query": "FIND {Alex Wilber} IN ALL FIELDS RETURNING Contact(LastModifiedDate, Name, Address)", + "language": "SOSL" + } +] +---- +// NOTCONSOLE + +**Example**: Fetch documents for custom objects via SOQL and SOSL query. + +[source,js] +---- +[ + { + "query": "SELECT Connector_Name, Version FROM Connector__c", + "language": "SOQL" + }, + { + "query": "FIND {Salesforce} IN ALL FIELDS RETURNING Connectors__c(Id, Connector_Name, Connector_Version)", + "language": "SOSL" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-salesforce-client-sync-rules-advanced-fetch-standard-custom-fields] +*Fetch documents with standard and custom fields* + +**Example**: Fetch documents with all standard and custom fields for Account object. + +[source,js] +---- +[ + { + "query": "SELECT FIELDS(ALL) FROM Account", + "language": "SOQL" + } +] +---- +// NOTCONSOLE + +**Example**: Fetch documents with all custom fields for Connector object. + +[source,js] +---- +[ + { + "query": "SELECT FIELDS(CUSTOM) FROM Connector__c", + "language": "SOQL" + } +] +---- +// NOTCONSOLE + +**Example**: Fetch documents with all standard fields for Account object. + +[source,js] +---- +[ + { + "query": "SELECT FIELDS(STANDARD) FROM Account", + "language": "SOQL" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-salesforce-client-documents-syncs] +===== Documents and syncs + +The connector syncs the following Salesforce objects: + +* *Accounts* +* *Campaigns* +* *Cases* +* *Contacts* +* *Content Documents* (files uploaded to Salesforce) +* *Leads* +* *Opportunities* + + +The connector will not ingest any objects that it does not have permissions to query. + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted by default. Use the <> to handle larger binary files. +* Permissions are not synced by default. You must enable <>. Otherwise, **all documents** indexed to an Elastic deployment will be visible to **all users with access** to that Elastic Deployment. +==== + +[discrete#es-connectors-salesforce-client-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-salesforce-client-content-extraction] +===== Content Extraction + +The connector will retrieve Content Documents from your Salesforce source if they meet the following criteria: + +* Are attached to one or more objects that are synced +* Are of a file type that can be extracted + +This means that the connector will not ingest any Content Documents you have that are _not_ attached to a supported Salesforce object. +See <> for a list of supported object types. + +If a single Content Document is attached to multiple supported objects, only one Elastic document will be created for it. +This document will retain links to every object that it was connected to in the `related_ids` field. + +See <> for more specifics on content extraction. + +[discrete#es-connectors-salesforce-client-known-issues] +===== Known issues + +* *DLS feature is "type-level" not "document-level"* ++ +Salesforce DLS, added in 8.13.0, does not accomodate specific access controls to specific Salesforce Objects. +Instead, if a given user/group can have access to _any_ Objects of a given type (`Case`, `Lead`, `Opportunity`, etc), that user/group will appear in the `\_allow_access_control` list for _all_ of the Objects of that type. +See https://github.com/elastic/connectors/issues/3028 for more details. ++ + +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-salesforce-client-security] +===== Security + +See <>. + +[discrete#es-connectors-salesforce-client-source] +===== Framework and source + +This connector is built with the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/salesforce.py[source code for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + + +// Closing the collapsible section +=============== diff --git a/docs/reference/connector/docs/connectors-scalability.asciidoc b/docs/reference/connector/docs/connectors-scalability.asciidoc new file mode 100644 index 0000000000000..6ebcb58e11487 --- /dev/null +++ b/docs/reference/connector/docs/connectors-scalability.asciidoc @@ -0,0 +1,49 @@ +[#es-connectors-scalability] +=== Connector Scalability +++++ +Scalability +++++ + +[discrete#es-connectors-scalability-redundancy] +==== Redundancy + +Users can create a backup (secondary) server with an identical connector service setup (settings, code, etc..). +If the primary server running the connector service fails, users can start up the connector service on the secondary +server and restart the sync jobs. +Because connector definitions and job status information are all stored in Elasticsearch, there is no risk of data loss +or corruption when switching servers. + +However, note that any in-progress syncs will need to be restarted from scratch, and cannot be resumed where they were +interrupted from. + +[discrete#es-connectors-scalability-failover] +==== Failover + +There is currently no automatic failover or transfer of workload in case of failure. +If the server running the connector service fails, all outstanding connector sync jobs will go into a suspended state. +When the connector service returns (or if a replacement connector service is launched), it will identify any jobs that +need to be cleaned up and automatically restart (from scratch) the suspended jobs. + +[discrete#es-connectors-scalability-balancing] +==== Workload balancing + +There is currently no division/balancing of workload for a single sync job across multiple connector service deployments. +Once a sync job is claimed by a connector service, it will run the job to completion - unless the connector service +instance fails. +In that case, another connector service instance will pick up the suspended job and restart it (from scratch). + +In 8.8.0+, the Connector Service provides concurrency control when there are multiple connector services connected to +the same Elasticsearch cluster, with the following expectations: + +* Multiple sync jobs can be scheduled for a given search index but only 1 sync job can be executed for a search index at any single time. +* Each sync job can only be claimed by 1 connector service. +* Only 1 connector service can perform management tasks at a time, for example: populating service types and configurations, validating filters, etc. + +[discrete#es-connectors-scalability-horizontal] +==== Horizontal Scalability + +Horizontal scaling can work if there are multiple connector services running and are configured to allow concurrent +syncs via their `service.max_concurrent_syncs` settings. + +Hypothetically, multiple Connector Services would naturally load balance to some extent even though we do not currently +have explicit load balancing functionality. diff --git a/docs/reference/connector/docs/connectors-security.asciidoc b/docs/reference/connector/docs/connectors-security.asciidoc new file mode 100644 index 0000000000000..4cbeacf3f28f0 --- /dev/null +++ b/docs/reference/connector/docs/connectors-security.asciidoc @@ -0,0 +1,55 @@ +[#es-connectors-security] +=== Connectors security +++++ +Security +++++ + +This document describes security considerations for <> and <>. + +Elastic Cloud deployments have strong security defaults. +For example, data is encrypted by default, whether at rest or in transit. + +Self-managed deployments require more upfront work to ensure strong security. +Refer to {ref}/secure-cluster.html[Secure the Elastic Stack^] in the Elasticsearch documentation for more information. + +[discrete#es-native-connectors-security-connections] +==== Access to credentials + +Credentials for the data source — such as API keys or username/password pair— are stored in your deployment's `.elastic-connectors` Elasticsearch index. +Therefore, the credentials are visible to all Elastic users with the `read` {ref}/security-privileges.html[indices privilege^] for that index. +By default, the following Elastic users have this privilege: the `elastic` superuser and the `kibana_system` user. +Enterprise Search service account tokens can also read the `.elastic-connectors` index. + +[discrete#es-native-connectors-security-api-key] +==== Access to internally stored API keys + +API keys for Elastic managed connectors are stored in the internal system index `.connector-secrets`. +Access to this index is restricted to authorized API calls only. +The cluster privilege `write_connector_secrets` is required to store or update secrets through the API. +Only the Enterprise Search instance has permission to read from this index. + +[discrete#es-native-connectors-security-dls] +===== Document-level security + +Document-level security is available for a subset of connectors. +DLS is available by default for the following connectors: + +include::_connectors-list-dls.asciidoc[] + +Learn more about this feature in <>, including availability and prerequisites. + +[discrete#es-native-connectors-security-deployment] +==== Access to documents + +Data synced from your data source are stored as documents in the Elasticsearch index you created. +This data is visible to all Elastic users with the `read` {ref}/security-privileges.html[indices privilege^] for that index. +Be careful to ensure that access to this index is _at least_ as restrictive as access to the original data source. + +[discrete#es-native-connectors-security-encryption] +==== Encryption + +Elastic Cloud automatically encrypts data at rest. +Data in transit is automatically encrypted using `https`. + +Self-managed deployments must implement encryption at rest. +See {ref}/configuring-stack-security.html[Configure security for the Elastic Stack^] in the Elasticsearch documentation for more information. diff --git a/docs/reference/connector/docs/connectors-self-managed.asciidoc b/docs/reference/connector/docs/connectors-self-managed.asciidoc new file mode 100644 index 0000000000000..e119953019442 --- /dev/null +++ b/docs/reference/connector/docs/connectors-self-managed.asciidoc @@ -0,0 +1,123 @@ +[#es-build-connector] +== Self-managed connectors + +.Naming history +**** +Self-managed connectors were initially known as "connector clients". You might find this term in older documentation. +**** + +Self-managed <> are run on your own infrastructure. +This means they run outside of your Elastic deployment. + +You can run the <> from source or from a Docker container. + +We also have a quickstart option using *Docker Compose*, to spin up all the required services at once: Elasticsearch, Kibana, and the connectors service. +Refer to <> for more information. + +The following connectors are available as self-managed connectors: + +include::_connectors-list-clients.asciidoc[] + +[discrete#es-build-connector-prerequisites] +=== Availability and Elastic prerequisites + +[NOTE] +==== +Self-managed connectors currently don't support Windows. +Use this https://www.elastic.co/support/matrix#matrix_os[compatibility matrix^] to check which operating systems are supported by self-managed connectors. +Find this information under *self-managed connectors* on that page. +==== + +.*Expand* for Elastic prerequisites information +[%collapsible] +==== +Your Elastic deployment must include the following Elastic services: + +* *Elasticsearch* +* *Kibana* + +(A new Elastic Cloud deployment includes these services by default.) + +To run self-managed connectors, your self-deployed connector service version must match your Elasticsearch version. +For example, if you're running Elasticsearch 8.10.1, your connector service should be version 8.10.1.x. +Elastic does not support deployments running mismatched versions (except during upgrades). + +[NOTE] +====== +As of 8.10.0 _new_ self-managed connectors no longer require the Enterprise Search service. +However, if you are upgrading connectors from versions earlier than 8.9, you'll need to run Enterprise Search once to migrate your connectors to the new format. +In future releases, you may still need to run Enterprise Search for the purpose of migrations or upgrades. +====== + +You must have access to Kibana and have `write` {ref}/security-privileges.html[indices privileges^] for the `.elastic-connectors` index. + +To use connector clients in a self-managed environment, you must deploy the <>. + +*Support and licensing requirements* + +Depending on how you use self-managed connectors, support and licensing requirements will vary. + +Refer to the following subscriptions pages for details. +Find your connector of interest in the *Elastic Search* section under *Client Integrations*: + +* https://www.elastic.co/subscriptions/[Elastic self-managed subscriptions page] +* https://www.elastic.co/subscriptions/cloud[Elastic Cloud subscriptions page] + +Note the following information regarding support for self-managed connectors: + +* A converted but _unmodified_ managed connector is supported by Elastic. +* A converted but _customized_ managed connector is _not_ supported by Elastic. + +==== + +[discrete#es-build-connector-data-source-prerequisites] +.Data source prerequisites +**** +The first decision you need to make before deploying a connector is which third party service (data source) you want to sync to Elasticsearch. +Note that each data source will have specific prerequisites you'll need to meet to authorize the connector to access its data. +For example, certain data sources may require you to create an OAuth application, or create a service account. + +You'll need to check the individual connector documentation for these details. +**** + +[discrete#es-connectors-deploy-connector-service] +=== Deploy the connector service + +The connector service is a Python application that you must run on your own infrastructure when using self-managed connectors. +The source code is hosted in the https://github.com/elastic/connectors[elastic/connectors^] repository. + +You can run the connector service from source or use Docker: + +* <>. Use this option if you're comfortable working with Python and want to iterate quickly locally. +* <>. Use this option if you want to deploy the connectors to a server, or use a container orchestration platform. +** Refer to our <> for a quick way to spin up all the required services at once. + +[discrete#es-build-connector-example] +=== Tutorials + +* Follow our <> to learn how run the self-managed connector service and a set up a self-managed connector, *using the UI*. +* Follow our <> to learn how to set up a self-managed connector *using the* {ref}/connector-apis.html[*connector APIs*]. + +These examples use the PostgreSQL connector but the basic process is the same for all self-managed connectors. + +[discrete#es-build-connector-testing] +=== Connector testing + +The connector framework enables you to run end-to-end (E2E) tests on your self-managed connectors, against a real data source. + +To avoid tampering with a real Elasticsearch instance, E2E tests run an isolated Elasticsearch instance in Docker. +Configuration values are set in your `docker-compose.yml` file. +Docker Compose manages the setup of the development environment, including both the mock Elastic instance and mock data source. + +E2E tests use *default* configuration values for the connector. +Find instructions about testing in each connector's documentation. + +[discrete#es-build-connector-framework] +=== Connector framework + +The Elastic connector framework enables you to: + +* Customize existing self-managed connectors. +* Build your own self-managed connectors. + +Refer to <> for more information. diff --git a/docs/reference/connector/docs/connectors-servicenow.asciidoc b/docs/reference/connector/docs/connectors-servicenow.asciidoc new file mode 100644 index 0000000000000..3dc98ed9a44c9 --- /dev/null +++ b/docs/reference/connector/docs/connectors-servicenow.asciidoc @@ -0,0 +1,494 @@ +[#es-connectors-servicenow] +=== Elastic ServiceNow connector reference +++++ +ServiceNow +++++ +// Attributes used in this file +:service-name: ServiceNow +:service-name-stub: servicenow + +The _Elastic ServiceNow connector_ is a <> for https://www.servicenow.com[ServiceNow^]. + +This connector is written in Python using the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/{service-name-stub}.py[*source code* for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + +.Choose your connector reference +******************************* +Are you using a managed connector on Elastic Cloud or a self-managed connector? Expand the documentation based on your deployment method. +******************************* + +// //////// //// //// //// //// //// //// //////// +// //////// NATIVE CONNECTOR REFERENCE /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-servicenow-native-connector-reference] +==== *Elastic managed connector reference* + +.View *Elastic managed connector* reference + +[%collapsible] +=============== + +[discrete#es-connectors-servicenow-availability-prerequisites] +===== Availability and prerequisites + +The ServiceNow connector is available natively in Elastic Cloud since 8.10.0. + +To use this connector natively in Elastic Cloud, satisfy all <>. + +[discrete#es-connectors-servicenow-create-native-connector] +===== Create a {service-name} connector +include::_connectors-create-native.asciidoc[] + +[discrete#es-connectors-servicenow-usage] +===== Usage + +To use this connector natively in Elastic Cloud, see <>. + +For additional operations, see <> + +[discrete#es-connectors-servicenow-compatibility] +===== Compatibility + +The ServiceNow connector is compatible with the following versions of ServiceNow: + +* ServiceNow "Tokyo" +* ServiceNow "San Diego" +* ServiceNow "Rome" +* ServiceNow "Utah" +* ServiceNow "Vancouver" +* ServiceNow "Washington" +* ServiceNow "Xanadu" + +[discrete#es-connectors-servicenow-configuration] +===== Configuration + +The following configuration fields are required to set up the connector: + +ServiceNow URL:: +The host URL of the ServiceNow instance. + +Username:: +The username of the account used for ServiceNow. + +Password:: +The password of the account used for ServiceNow. + +Comma-separated list of services:: +Comma-separated list of services to fetch data from ServiceNow. If the value is `*`, the connector will fetch data from the list of basic services provided by ServiceNow: +- link:https://docs.servicenow.com/bundle/utah-platform-administration/page/administer/roles/concept/user.html[User] +- link:https://docs.servicenow.com/bundle/tokyo-it-service-management/page/product/incident-management/concept/c_IncidentManagement.html[Incident] +- link:https://docs.servicenow.com/bundle/tokyo-servicenow-platform/page/use/service-catalog-requests/task/t_AddNewRequestItems.html[Requested Item] +- link:https://docs.servicenow.com/bundle/tokyo-customer-service-management/page/product/customer-service-management/task/t_SearchTheKnowledgeBase.html[Knowledge] +- link:https://docs.servicenow.com/bundle/tokyo-it-service-management/page/product/change-management/task/t_CreateAChange.html[Change request] ++ +[NOTE] +==== +If you have configured a custom service, the `*` value will not fetch data from the basic services above by default. In this case you'll need to mention these service names explicitly. +==== +Default value is `*`. Examples: ++ + - `User, Incident, Requested Item, Knowledge, Change request` + - `*` + +Enable document level security:: +Restrict access to documents based on a user's permissions. +Refer to <> for more details. + +[discrete#es-connectors-servicenow-documents-syncs] +===== Documents and syncs + +All services and records the user has access to will be indexed according to the configurations provided. +The connector syncs the following ServiceNow object types: + +* Records +* Attachments + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted. (Self-managed connectors can use the <> to handle larger binary files.) +* Permissions are not synced by default. Refer to <> for more details. +==== + +[discrete#es-connectors-servicenow-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-servicenow-dls] +===== Document level security + +<> ensures identities and permissions set in ServiceNow are maintained in Elasticsearch. +This enables you to restrict and personalize read-access users and groups have to documents in this index. +Access control syncs ensure this metadata is kept up to date in your Elasticsearch documents. + +The ServiceNow connector supports roles for access control lists (ACLs) to enable document level security in {es}. +For default services, connectors use the following roles to find users who have access to documents. + +|=== +| Service | Roles + +| User | `admin` + +| Incident | `admin`, `sn_incident_read`, `ml_report_user`, `ml_admin`, `itil` + +| Requested Item | `admin`, `sn_request_read`, `asset`, `atf_test_designer`, `atf_test_admin` + +| Knowledge | `admin`, `knowledge`, `knowledge_manager`, `knowledge_admin` + +| Change request | `admin`, `sn_change_read`, `itil` +|=== + +For services other than these defaults, the connector iterates over access controls with `read` operations and finds the respective roles for those services. + +[NOTE] +==== +The ServiceNow connector does not support scripted and conditional permissions. +==== + +[discrete#es-connectors-servicenow-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +[discrete#es-connectors-servicenow-sync-rules-advanced] +====== Advanced sync rules + +[NOTE] +==== +A <> is required for advanced sync rules to take effect. +==== + +Advanced sync rules are defined through a source-specific DSL JSON snippet. + +The following sections provide examples of advanced sync rules for this connector. + +[discrete#es-connectors-servicenow-sync-rules-number-incident-service] +*Indexing document based on incident number for Incident service* + +[source,js] +---- +[ + { + "service": "Incident", + "query": "numberSTARTSWITHINC001" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-servicenow-sync-rules-active-false-user-service] +*Indexing document based on user activity state for User service* + +[source,js] +---- +[ + { + "service": "User", + "query": "active=False" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-servicenow-sync-rules-author-administrator-knowledge-service] +*Indexing document based on author name for Knowledge service* + +[source,js] +---- +[ + { + "service": "Knowledge", + "query": "author.nameSTARTSWITHSystem Administrator" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-servicenow-known-issues] +===== Known issues + +There are no known issues for this connector. +Refer to <> for a list of known issues that impact all connectors. + +[discrete#es-connectors-servicenow-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-servicenow-security] +===== Security + +See <>. + +[discrete#es-connectors-servicenow-content-extraction] +===== Content extraction + +See <>. + + +// Closing the collapsible section +=============== + + +// //////// //// //// //// //// //// //// //////// +// //////// CONNECTOR CLIENT REFERENCE /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-servicenow-connector-client-reference] +==== *Self-managed connector* + +.View *self-managed connector* reference + +[%collapsible] +=============== + +[discrete#es-connectors-servicenow-client-availability-prerequisites] +===== Availability and prerequisites + +The ServiceNow connector was introduced in Elastic version 8.9.0. +This connector is available as a self-managed *self-managed connector*. +To use this connector as a self-managed connector, satisfy all <>. + +[discrete#es-connectors-servicenow-create-connector-client] +===== Create a {service-name} connector +include::_connectors-create-client.asciidoc[] + +[discrete#es-connectors-servicenow-client-usage] +===== Usage +To use this connector as a *self-managed connector*, use the *Customized connector* workflow. + +For additional operations, see <>. + +[discrete#es-connectors-servicenow-client-compatibility] +===== Compatibility + +The ServiceNow connector is compatible with the following versions of ServiceNow: + +* ServiceNow "Tokyo" +* ServiceNow "San Diego" +* ServiceNow "Rome" +* ServiceNow "Utah" +* ServiceNow "Vancouver" +* ServiceNow "Washington" +* ServiceNow "Xanadu" + +[discrete#es-connectors-servicenow-client-configuration] +===== Configuration + +[TIP] +==== +When using the <> workflow, initially these fields will use the default configuration set in the {connectors-python}/connectors/sources/servicenow.py[connector source code^]. +These are set in the `get_default_configuration` function definition. + +These configurable fields will be rendered with their respective *labels* in the Kibana UI. +Once connected, you'll be able to update these values in Kibana. +==== + +The following configuration fields are required to set up the connector: + +`url`:: +The host url of the ServiceNow instance. + +`username`:: +The username of the account for ServiceNow. + +`password`:: +The password of the account used for ServiceNow. + +`services`:: +Comma-separated list of services to fetch data from ServiceNow. If the value is `*`, the connector will fetch data from the list of basic services provided by ServiceNow: +- link:https://docs.servicenow.com/bundle/utah-platform-administration/page/administer/roles/concept/user.html[User] +- link:https://docs.servicenow.com/bundle/tokyo-it-service-management/page/product/incident-management/concept/c_IncidentManagement.html[Incident] +- link:https://docs.servicenow.com/bundle/tokyo-servicenow-platform/page/use/service-catalog-requests/task/t_AddNewRequestItems.html[Requested Item] +- link:https://docs.servicenow.com/bundle/tokyo-customer-service-management/page/product/customer-service-management/task/t_SearchTheKnowledgeBase.html[Knowledge] +- link:https://docs.servicenow.com/bundle/tokyo-it-service-management/page/product/change-management/task/t_CreateAChange.html[Change request] ++ +[NOTE] +==== +If you have configured a custom service, the `*` value will not fetch data from the basic services above by default. In this case you'll need to mention these service names explicitly. +==== +Default value is `*`. Examples: ++ + - `User, Incident, Requested Item, Knowledge, Change request` + - `*` + +`retry_count`:: +The number of retry attempts after a failed request to ServiceNow. Default value is `3`. + +`concurrent_downloads`:: +The number of concurrent downloads for fetching the attachment content. This speeds up the content extraction of attachments. Defaults to `10`. + +`use_text_extraction_service`:: +Requires a separate deployment of the <>. +Requires that ingest pipeline settings disable text extraction. +Default value is `False`. + +`use_document_level_security`:: +Restrict access to documents based on a user's permissions. +Refer to <> for more details. + +[discrete#es-connectors-servicenow-client-documents-syncs] +===== Documents and syncs + +All services and records the user has access to will be indexed according to the configurations provided. +The connector syncs the following ServiceNow object types: + +* Records +* Attachments + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted. Use the <> to handle larger binary files. +* Permissions are not synced by default. You must enable <>. Otherwise, **all documents** indexed to an Elastic deployment will be visible to **all users with access** to that Elastic Deployment. +==== + +[discrete#es-connectors-servicenow-client-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-servicenow-client-dls] +===== Document level security + +<> ensures identities and permissions set in ServiceNow are maintained in Elasticsearch. +This enables you to restrict and personalize read-access users and groups have to documents in this index. +Access control syncs ensure this metadata is kept up to date in your Elasticsearch documents. + +The ServiceNow connector supports roles for access control lists (ACLs) to enable document level security in {es}. +For default services, connectors use the following roles to find users who have access to documents. + +|=== +| Service | Roles + +| User | `admin` + +| Incident | `admin`, `sn_incident_read`, `ml_report_user`, `ml_admin`, `itil` + +| Requested Item | `admin`, `sn_request_read`, `asset`, `atf_test_designer`, `atf_test_admin` + +| Knowledge | `admin`, `knowledge`, `knowledge_manager`, `knowledge_admin` + +| Change request | `admin`, `sn_change_read`, `itil` +|=== + +For services other than these defaults, the connector iterates over access controls with `read` operations and finds the respective roles for those services. + +[NOTE] +==== +The ServiceNow connector does not support scripted and conditional permissions. +==== + +[discrete#es-connectors-servicenow-client-docker] +===== Deployment using Docker + +include::_connectors-docker-instructions.asciidoc[] + +[discrete#es-connectors-servicenow-client-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +[discrete#es-connectors-servicenow-client-sync-rules-advanced] +====== Advanced sync rules + +[NOTE] +==== +A <> is required for advanced sync rules to take effect. +==== + +Advanced sync rules are defined through a source-specific DSL JSON snippet. + +The following sections provide examples of advanced sync rules for this connector. + +[discrete#es-connectors-servicenow-client-sync-rules-number-incident-service] +*Indexing document based on incident number for Incident service* + +[source,js] +---- +[ + { + "service": "Incident", + "query": "numberSTARTSWITHINC001" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-servicenow-client-sync-rules-active-false-user-service] +*Indexing document based on user activity state for User service* + +[source,js] +---- +[ + { + "service": "User", + "query": "active=False" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-servicenow-client-sync-rules-author-administrator-knowledge-service] +*Indexing document based on author name for Knowledge service* + +[source,js] +---- +[ + { + "service": "Knowledge", + "query": "author.nameSTARTSWITHSystem Administrator" + } +] +---- +// NOTCONSOLE + +[discrete#es-connectors-servicenow-client-connector-client-operations-testing] +===== End-to-end Testing + +The connector framework enables operators to run functional tests against a real data source. +Refer to <> for more details. + +To perform E2E testing for the ServiceNow connector, run the following command: + +[source,shell] +---- +$ make ftest NAME=servicenow +---- + +Generate performance reports using the following flag: `PERF8=yes`. +Toggle test data set size between SMALL, MEDIUM and LARGE with the argument `DATA_SIZE=`. +By default, it is set to `MEDIUM`. + +Users do not need to have a running Elasticsearch instance or a ServiceNow source to run this test. +Docker Compose manages the complete setup of the development environment. + +[discrete#es-connectors-servicenow-client-known-issues] +===== Known issues + +There are no known issues for this connector. +Refer to <> for a list of known issues that impact all connectors. + +[discrete#es-connectors-servicenow-client-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-servicenow-client-security] +===== Security + +See <>. + +[discrete#es-connectors-servicenow-client-content-extraction] +===== Content extraction + +See <>. + + +// Closing the collapsible section +=============== diff --git a/docs/reference/connector/docs/connectors-sharepoint-online.asciidoc b/docs/reference/connector/docs/connectors-sharepoint-online.asciidoc new file mode 100644 index 0000000000000..2680e3ff840a6 --- /dev/null +++ b/docs/reference/connector/docs/connectors-sharepoint-online.asciidoc @@ -0,0 +1,1097 @@ +[#es-connectors-sharepoint-online] +=== Elastic SharePoint Online connector reference +++++ +SharePoint Online +++++ +// Attributes used in this file +:service-name: SharePoint Online +:service-name-stub: sharepoint_online + +[TIP] +==== +Looking for the SharePoint *Server* connector? See <>. +==== + +The _Elastic SharePoint Online connector_ is a <> for https://www.microsoft.com/en-ww/microsoft-365/sharepoint/[Microsoft SharePoint Online^]. + +This connector is written in Python using the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/{service-name-stub}.py[*source code* for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + +.Choose your connector reference +******************************* +Are you using a managed connector on Elastic Cloud or a self-managed connector? Expand the documentation based on your deployment method. +******************************* + +// //////// //// //// //// //// //// //// //////// +// //////// NATIVE CONNECTOR REFERENCE /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-sharepoint-online-native-connector-reference] +==== *Elastic managed connector reference* + +.View *Elastic managed connector* reference + +[%collapsible] +=============== + +[discrete#es-connectors-sharepoint-online-availability-prerequisites] +===== Availability and prerequisites + +This connector is available as a *managed connector* in Elastic versions *8.9.0 and later*. +To use this connector natively in Elastic Cloud, satisfy all <>. + +[NOTE] +==== +This connector requires a subscription. +View the requirements for this feature under the *Elastic Search* section of the https://www.elastic.co/subscriptions[Elastic Stack subscriptions^] page. +==== + +[discrete#es-connectors-sharepoint-online-usage] +===== Usage + +To use this connector as a *managed connector*, see <>. + +For additional operations, see <>. + +[discrete#es-connectors-sharepoint-online-sharepoint-prerequisites] +===== SharePoint prerequisites + +[discrete#es-connectors-sharepoint-online-oauth-app-create] +====== Create SharePoint OAuth app + +Before you can configure the connector, you must create an **OAuth App** in the SharePoint Online platform. +Your connector will authenticate to SharePoint as the registered OAuth application/client. +You'll collect values (`client ID`, `tenant ID`, and `client secret`) during this process that you'll need for the <> in Kibana. + +To get started, first log in to SharePoint Online and access your administrative dashboard. +Ensure you are logged in as the Azure Portal **service account**. + +Follow these steps: + +* Sign in to https://portal.azure.com/ and click on **Azure Active Directory**. +* Locate **App Registrations** and Click **New Registration**. +* Give your app a name - like "Search". +* Leave the *Redirect URIs* blank for now. +* *Register* the application. +* Find and keep the **Application (client) ID** and **Directory (tenant) ID** handy. +* Locate the **Secret** by navigating to **Client credentials: Certificates & Secrets**. +* Select **New client secret** +* Pick a name for your client secret. +Select an expiration date. (At this expiration date, you will need to generate a new secret and update your connector configuration.) +** Save the client secret **Secret ID** before leaving this screen. +** Save the client secret **Value** before leaving this screen. +* Set up the permissions the OAuth App will request from the Azure Portal service account. +** Navigate to **API Permissions** and click **Add Permission**. +** Add **application permissions** until the list looks like the following: ++ +``` +Graph API +- Sites.Selected +- Files.Read.All +- Group.Read.All +- User.Read.All + +Sharepoint +- Sites.Selected +``` +NOTE: If the `Comma-separated list of sites` configuration is set to `*` or if a user enables the toggle button `Enumerate all sites`, the connector requires `Sites.Read.All` permission. + +* **Grant admin consent**, using the `Grant Admin Consent` link from the permissions screen. +* Save the tenant name (i.e. Domain name) of Azure platform. + +[WARNING] +==== +The connector requires application permissions. It does not support delegated permissions (scopes). +==== + +[NOTE] +==== +The connector uses the https://learn.microsoft.com/en-us/sharepoint/dev/apis/sharepoint-rest-graph[Graph API^] (stable https://learn.microsoft.com/en-us/graph/api/overview?view=graph-rest-1.0#other-api-versions[v1.0 API^]) where possible to fetch data from Sharepoint Online. +When entities are not available via the Graph API the connector falls back to using the Sharepoint https://learn.microsoft.com/en-us/sharepoint/dev/sp-add-ins/get-to-know-the-sharepoint-rest-service[REST API^]. +==== + +[discrete#es-connectors-sharepoint-online-oauth-app-permissions] +====== SharePoint permissions + +Refer to the following documentation for setting https://learn.microsoft.com/en-us/sharepoint/dev/solution-guidance/security-apponly-azureacs[SharePoint permissions^]. + +* To set `DisableCustomAppAuthentication` to false, connect to SharePoint using PowerShell and run `set-spotenant -DisableCustomAppAuthentication $false` +* To assign full permissions to the tenant in SharePoint Online, go to the tenant URL in your browser. +The URL follows this pattern: `https:///_layouts/15/appinv.aspx`. +This loads the SharePoint admin center page. +** In the *App ID* box, enter the application ID that you recorded earlier, and then click *Lookup*. +The application name will appear in the Title box. +** In the *App Domain* box, type .onmicrosoft.com +** In the *App's Permission Request XML* box, type the following XML string: ++ +[source, xml] +---- + + + + +---- + +[discrete#es-connectors-sharepoint-online-sites-selected-permissions] +====== Granting `Sites.Selected` permissions + +To configure `Sites.Selected` permissions, follow these steps in the Azure Active Directory portal. These permissions enable precise access control to specific SharePoint sites. + +. Sign in to the https://portal.azure.com/[Azure Active Directory portal^]. +. Navigate to **App registrations** and locate the application created for the connector. +. Under **API permissions**, click **Add permission**. +. Select **Microsoft Graph** > **Application permissions**, then add `Sites.Selected`. +. Click **Grant admin consent** to approve the permission. + +[TIP] +==== +Refer to the official https://learn.microsoft.com/en-us/graph/permissions-reference[Microsoft documentation] for managing permissions in Azure AD. +==== + +To assign access to specific SharePoint sites using `Sites.Selected`: + +. Use Microsoft Graph Explorer or PowerShell to grant access. +. To fetch the site ID, run the following Graph API query: ++ +[source, http] +---- +GET https://graph.microsoft.com/v1.0/sites?select=webUrl,Title,Id&$search="*" +---- ++ +This will return the `id` of the site. + +. Use the `id` to assign read or write access: ++ +[source, http] +---- +POST https://graph.microsoft.com/v1.0/sites//permissions +{ + "roles": ["read"], // or "write" + "grantedToIdentities": [ + { + "application": { + "id": "", + "displayName": "" + } + } + ] +} +---- + +[NOTE] +==== +When using the `Comma-separated list of sites` configuration field, ensure the sites specified match those granted `Sites.Selected` permission in SharePoint. +If the `Comma-separated list of sites` field is set to `*` or the `Enumerate all sites` toggle is enabled, the connector will attempt to access all sites. This requires broader permissions, which are not supported with `Sites.Selected`. +==== + +.Graph API permissions +**** +Microsoft recommends using Graph API for all operations with Sharepoint Online. Graph API is well-documented and more efficient at fetching data, which helps avoid throttling. +Refer to https://learn.microsoft.com/en-us/sharepoint/dev/general-development/how-to-avoid-getting-throttled-or-blocked-in-sharepoint-online[Microsoft's throttling policies^] for more information. + +Here's a summary of why we use these Graph API permissions: + +* *Sites.Selected* is used to fetch the sites and their metadata +* *Files.Read.All* is used to fetch Site Drives and files in these drives +* *Groups.Read.All* is used to fetch groups for document-level permissions +* *User.Read.All* is used to fetch user information for document-level permissions + +Due to the way the Graph API is designed, these permissions are "all or nothing" - it's currently impossible to limit access to these resources. +**** + +[discrete#es-connectors-sharepoint-online-compatability] +===== Compatibility + +This connector is compatible with SharePoint Online. + +[discrete#es-connectors-sharepoint-online-configuration] +===== Configuration + +Use the following configuration fields to set up the connector: + +Tenant ID:: +The tenant id for the Azure account hosting the Sharepoint Online instance. + +Tenant Name:: +The tenant name for the Azure account hosting the Sharepoint Online instance. + +Client ID:: +The client id to authenticate with SharePoint Online. + +Secret value:: +The secret value to authenticate with SharePoint Online. + +Comma-separated list of sites:: +List of site collection names or paths to fetch from SharePoint. +When enumerating all sites, these values should be the _names_ of the sites. +Use `*` to include all available sites. +Examples: +* `collection1` +* `collection1,sub-collection` +* `*` ++ +When **not** enumerating all sites, these values should be the _paths_ (URL after `/sites/`) of the sites. +Examples: +* `collection1` +* `collection1,collection1/sub-collection` + +Enumerate all sites?:: +If enabled, the full list of all sites will be fetched from the API, in bulk, and will be filtered down to match the configured list of site names. +If disabled, each path in the configured list of site paths will be fetched individually from the API. +When disabled, `*` is not a valid configuration for `Comma-separated list of sites`. +Enabling this configuration is most useful when syncing large numbers (more than total/200) of sites. +This is because, at high volumes, it is more efficient to fetch sites in bulk. +When syncing fewer sites, disabling this configuration can result in improved performance. +This is because, at low volumes, it is more efficient to only fetch the sites that you need. + +Fetch sub-sites of configured sites?:: +Whether sub-sites of the configured site(s) should be automatically fetched. +This option is only available when not enumerating all sites (see above). + +Enable document level security:: +Toggle to enable <>. +When enabled, full and incremental syncs will fetch access control lists for each document and store them in the `_allow_access_control` field. +Access control syncs will fetch users' access control lists and store them in a separate index. ++ +Once enabled, the following granular permissions toggles will be available: + +** *Fetch drive item permissions*: Enable this option to fetch *drive item* specific permissions. +** *Fetch unique page permissions*: Enable this option to fetch unique *page* permissions. If this setting is disabled a page will inherit permissions from its parent site. +** *Fetch unique list permissions*: Enable this option to fetch unique *list* permissions. If this setting is disabled a list will inherit permissions from its parent site. +** *Fetch unique list item permissions*: Enable this option to fetch unique *list item* permissions. If this setting is disabled a list item will inherit permissions from its parent site. ++ +[NOTE] +==== +If left empty the default value `true` will be used for these granular permissions toggles. +Note that these settings may increase sync times. +==== + +[discrete#es-connectors-sharepoint-online-documents-syncs] +===== Documents and syncs + +The connector syncs the following SharePoint object types: + +* *Sites* (and subsites) +* *Lists* +* *List items* and *attachment content* +* *Document libraries* and *attachment content* (including web pages) + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted. (Self-managed connectors can use the <> to handle larger binary files.) +* Permissions are not synced by default. Enable <> to sync permissions. +==== + +[TIP] +==== +*Making Sharepoint Site Pages Web Part content searchable* + +If you're using Web Parts on Sharepoint Site Pages and want to make this content searchable, you'll need to consult the https://learn.microsoft.com/en-us/sharepoint/dev/spfx/web-parts/guidance/integrate-web-part-properties-with-sharepoint#specify-web-part-property-value-type/[official documentation^]. + +We recommend setting `isHtmlString` to *True* for all Web Parts that need to be searchable. +==== + +[discrete#es-connectors-sharepoint-online-documents-syncs-limitations] +====== Limitations + +* The connector does not currently sync content from Teams-connected sites. + +[discrete#es-connectors-sharepoint-online-sync-rules] +===== Sync rules + +_Basic_ sync rules are identical for all connectors and are available by default. +For more information read <>. + +[discrete#es-connectors-sharepoint-online-sync-rules-advanced] +====== Advanced sync rules + +[NOTE] +==== +A <> is required for advanced sync rules to take effect. +==== + +The following section describes *advanced sync rules* for this connector. +Advanced sync rules are defined through a source-specific DSL JSON snippet. + +<> for the Sharepoint Online connector enable you to avoid extracting and syncing older data that might no longer be relevant for search. + +Example: + +[source,js] +---- +{ + "skipExtractingDriveItemsOlderThan": 60 +} +---- +// NOTCONSOLE + +This rule will not extract content of any drive items (files in document libraries) that haven't been modified for 60 days or more. + +[discrete#es-connectors-sharepoint-online-sync-rules-limitations] +*Limitations of sync rules with incremental syncs* + +Changing sync rules after Sharepoint Online content has already been indexed can bring unexpected results, when using <>. + +Incremental syncs ensure _updates_ from 3rd-party system, but do not modify existing documents in the index. + +*To avoid these issues, run a full sync after changing sync rules (basic or advanced).* + +Let's take a look at several examples where incremental syncs might lead to inconsistent data on your index. + +[discrete#es-connectors-sharepoint-online-sync-rules-limitations-restrictive-added] +*Example: Restrictive basic sync rule added after a full sync* + +Imagine your Sharepoint Online drive contains the following drive items: + +[source,txt] +---- +/Documents/Report.doc +/Documents/Spreadsheet.xls +/Presentations/Q4-2020-Report.pdf +/Presentations/Q4-2020-Report-Data.xls +/Personal/Documents/Sales.xls +---- + +After a sync, all these drive items will be stored on your Elasticsearch index. +Let's add a basic sync rule, filtering files by their path: + +[source,txt] +---- +Exclude WHERE path CONTAINS "Documents" +---- + +These filtering rules will exclude all files with "Documents" in their path, leaving only files in `/Presentations` directory: + +[source,txt] +---- +/Presentations/Q4-2020-Report.pdf +/Presentations/Q4-2020-Report-Data.xls +---- + +If no files were changed, incremental sync will not receive information about changes from Sharepoint Online and won't be able to delete any files, leaving the index in the same state it was before the sync. + +After a *full sync*, the index will be updated and files that are excluded by sync rules will be removed. + +[discrete#es-connectors-sharepoint-online-sync-rules-limitations-restrictive-removed] +*Example: Restrictive basic sync rules removed after a full sync* + +Imagine that Sharepoint Online drive has the following drive items: + +[source,txt] +---- +/Documents/Report.doc +/Documents/Spreadsheet.xls +/Presentations/Q4-2020-Report.pdf +/Presentations/Q4-2020-Report-Data.xls +/Personal/Documents/Sales.xls +---- + +Before doing a sync, we add a restrictive basic filtering rule: + +[source,txt] +---- +Exclude WHERE path CONTAINS "Documents" +---- + +After a full sync, the index will contain only files in the `/Presentations` directory: + +[source,txt] +---- +/Presentations/Q4-2020-Report.pdf +/Presentations/Q4-2020-Report-Data.xls +---- + +Afterwards, we can remove the filtering rule and run an incremental sync. If no changes happened to the files, incremental sync will not mirror these changes in the Elasticsearch index, because Sharepoint Online will not report any changes to the items. +Only a *full sync* will include the items previously ignored by the sync rule. + +[discrete#es-connectors-sharepoint-online-sync-rules-limitations-restrictive-changed] +*Example: Advanced sync rules edge case* + +Advanced sync rules can be applied to limit which documents will have content extracted. +For example, it's possible to set a rule so that documents older than 180 days won't have content extracted. + +However, there is an edge case. +Imagine a document that is 179 days old and its content is extracted and indexed into Elasticsearch. +After 2 days, this document will be 181 days old. +Since this document was already ingested it will not be modified. +Therefore, the content will not be removed from the index, following an incremental sync. + +In this situation, if you want older documents to be removed, you will need to clean the index up manually. +For example, you can manually run an Elasticsearch query that removes drive item content older than 180 days: + +[source, console] +---- +POST INDEX_NAME/_update_by_query?conflicts=proceed +{ + "query": { + "bool": { + "filter": [ + { + "match": { + "object_type": "drive_item" + } + }, + { + "exists": { + "field": "file" + } + }, + { + "range": { + "lastModifiedDateTime": { + "lte": "now-180d" + } + } + } + ] + } + }, + "script": { + "source": "ctx._source.body = ''", + "lang": "painless" + } +} +---- +// TEST[skip:TODO] + +[discrete#es-connectors-sharepoint-online-dls] +===== Document-level security + +Document-level security (DLS) enables you to restrict access to documents based on a user's permissions. +This feature is available by default for this connector. + +Refer to <> on this page for how to enable DLS for this connector. + +[TIP] +==== +Refer to <> to learn how to ingest data from SharePoint Online with DLS enabled, when building a search application. +==== + +[discrete#es-connectors-sharepoint-online-content-extraction] +===== Content extraction + +[discrete#es-connectors-sharepoint-online-content-extraction-pipeline] +====== Default content extraction + +The default content extraction service is powered by the Enterprise Search default ingest pipeline. +(See {ref}/ingest-pipeline-search.html[Ingest pipelines for Search indices].) + +See <>. + +[discrete#es-connectors-sharepoint-online-content-extraction-local] +====== Local content extraction (for large files) + +The SharePoint Online self-managed connector supports large file content extraction (> *100MB*). +This requires: + +* A self-managed deployment of the Elastic Text Extraction Service. +* Text extraction to be _disabled_ in the default ingest pipeline settings. + +Refer to <> for more information. + +[discrete#es-connectors-sharepoint-online-known-issues] +===== Known issues + +* *Documents failing to sync due to SharePoint file and folder limits* ++ +SharePoint has limits on the number of files and folders that can be synced. +You might encounter an error like the following written to the body of documents that failed to sync: +`The file size exceeds the allowed limit. CorrelationId: fdb36977-7cb8-4739-992f-49878ada6686, UTC DateTime: 4/21/2022 11:24:22 PM` ++ +Refer to https://support.microsoft.com/en-us/office/download-files-and-folders-from-onedrive-or-sharepoint-5c7397b7-19c7-4893-84fe-d02e8fa5df05#:~:text=Downloads%20are%20subject%20to%20the,zip%20file%20and%2020GB%20overall[SharePoint documentation^] for more information about these limits. ++ +** *Syncing a large number of files* ++ +The connector will fail to download files from folders that contain more than 5000 files. +The List View Threshold (default 5000) is a limit that prevents operations with a high performance impact on the SharePoint Online environment. ++ +*Workaround:* Reduce batch size to avoid this issue. ++ +** *Syncing large files* ++ +SharePoint has file size limits, but these are configurable. ++ +*Workaround:* Increase the file size limit. +Refer to https://learn.microsoft.com/en-us/sharepoint/manage-site-collection-storage-limits#set-automatic-or-manual-site-storage-limits[SharePoint documentation^] for more information. ++ +** *Deleted documents counter is not updated during incremental syncs* ++ +If the configuration `Enumerate All Sites?` is enabled, incremental syncs may not behave as expected. +Drive Item documents that were deleted between incremental syncs may not be detected as deleted. ++ +*Workaround*: Disable `Enumerate All Sites?`, and configure full site paths for all desired sites. + +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-sharepoint-online-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-sharepoint-onlinesecurity] +===== Security + +See <>. + +// Closing the collapsible section +=============== + +// //////// //// //// //// //// //// //// //////// +// //////// CONNECTOR CLIENT REFERENCE /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-sharepoint-online-connector-client-reference] +==== *Self-managed connector* + +.View *self-managed connector* reference + +[%collapsible] +=============== + +[discrete#es-connectors-sharepoint-online-client-availability-prerequisites] +===== Availability and prerequisites + +This connector is available as a self-managed *self-managed connector*. +To use this connector as a self-managed connector, satisfy all <>. + +[NOTE] +==== +This connector requires a subscription. +View the requirements for this feature under the *Elastic Search* section of the https://www.elastic.co/subscriptions[Elastic Stack subscriptions^] page. +==== + +[discrete#es-connectors-sharepoint-online-client-usage] +===== Usage + +To use this connector as a *self-managed connector*, see <> +For additional operations, see <>. + +[discrete#es-connectors-sharepoint-online-client-sharepoint-prerequisites] +===== SharePoint prerequisites + +[discrete#es-connectors-sharepoint-online-client-oauth-app-create] +====== Create SharePoint OAuth app + +Before you can configure the connector, you must create an **OAuth App** in the SharePoint Online platform. +Your connector will authenticate to SharePoint as the registered OAuth application/client. +You'll collect values (`client ID`, `tenant ID`, and `client secret`) during this process that you'll need for the <> in Kibana. + +To get started, first log in to SharePoint Online and access your administrative dashboard. +Ensure you are logged in as the Azure Portal **service account**. + +Follow these steps: + +* Sign in to https://portal.azure.com/ and click on **Azure Active Directory**. +* Locate **App Registrations** and Click **New Registration**. +* Give your app a name - like "Search". +* Leave the *Redirect URIs* blank for now. +* *Register* the application. +* Find and keep the **Application (client) ID** and **Directory (tenant) ID** handy. +* Locate the **Secret** by navigating to **Client credentials: Certificates & Secrets**. +* Select **New client secret** +* Pick a name for your client secret. +Select an expiration date. (At this expiration date, you will need to generate a new secret and update your connector configuration.) +** Save the client secret **Secret ID** before leaving this screen. +** Save the client secret **Value** before leaving this screen. +* Set up the permissions the OAuth App will request from the Azure Portal service account. +** Navigate to **API Permissions** and click **Add Permission**. +** Add **application permissions** until the list looks like the following: ++ +``` +Graph API +- Sites.Selected +- Files.Read.All +- Group.Read.All +- User.Read.All + +Sharepoint +- Sites.Selected +``` +NOTE: If the `Comma-separated list of sites` configuration is set to `*` or if a user enables the toggle button `Enumerate all sites`, the connector requires `Sites.Read.All` permission. + +* **Grant admin consent**, using the `Grant Admin Consent` link from the permissions screen. +* Save the tenant name (i.e. Domain name) of Azure platform. + +[WARNING] +==== +The connector requires application permissions. It does not support delegated permissions (scopes). +==== + +[NOTE] +==== +The connector uses the https://learn.microsoft.com/en-us/sharepoint/dev/apis/sharepoint-rest-graph[Graph API^] (stable https://learn.microsoft.com/en-us/graph/api/overview?view=graph-rest-1.0#other-api-versions[v1.0 API^]) where possible to fetch data from Sharepoint Online. +When entities are not available via the Graph API the connector falls back to using the Sharepoint https://learn.microsoft.com/en-us/sharepoint/dev/sp-add-ins/get-to-know-the-sharepoint-rest-service[REST API^]. +==== + +[discrete#es-connectors-sharepoint-online-client-oauth-app-permissions] +====== SharePoint permissions + +Refer to the following documentation for setting https://learn.microsoft.com/en-us/sharepoint/dev/solution-guidance/security-apponly-azureacs[SharePoint permissions^]. + +* To set `DisableCustomAppAuthentication` to false, connect to SharePoint using PowerShell and run `set-spotenant -DisableCustomAppAuthentication $false` +* To assign full permissions to the tenant in SharePoint Online, go to the tenant URL in your browser. +The URL follows this pattern: `https:///_layouts/15/appinv.aspx`. +This loads the SharePoint admin center page. +** In the *App ID* box, enter the application ID that you recorded earlier, and then click *Lookup*. +The application name will appear in the Title box. +** In the *App Domain* box, type .onmicrosoft.com +** In the *App's Permission Request XML* box, type the following XML string: ++ +[source, xml] +---- + + + + +---- + +[discrete#es-connectors-sharepoint-online-sites-selected-permissions-self-managed] +====== Granting `Sites.Selected` permissions + +To configure `Sites.Selected` permissions, follow these steps in the Azure Active Directory portal. These permissions enable precise access control to specific SharePoint sites. + +. Sign in to the https://portal.azure.com/[Azure Active Directory portal^]. +. Navigate to **App registrations** and locate the application created for the connector. +. Under **API permissions**, click **Add permission**. +. Select **Microsoft Graph** > **Application permissions**, then add `Sites.Selected`. +. Click **Grant admin consent** to approve the permission. + +[TIP] +==== +Refer to the official https://learn.microsoft.com/en-us/graph/permissions-reference[Microsoft documentation] for managing permissions in Azure AD. +==== + + +To assign access to specific SharePoint sites using `Sites.Selected`: + +. Use Microsoft Graph Explorer or PowerShell to grant access. +. To fetch the site ID, run the following Graph API query: ++ +[source, http] +---- +GET https://graph.microsoft.com/v1.0/sites?select=webUrl,Title,Id&$search="*" +---- ++ +This will return the `id` of the site. + +. Use the `id` to assign read or write access: ++ +[source, http] +---- +POST https://graph.microsoft.com/v1.0/sites//permissions +{ + "roles": ["read"], // or "write" + "grantedToIdentities": [ + { + "application": { + "id": "", + "displayName": "" + } + } + ] +} +---- + +[NOTE] +==== +When using the `Comma-separated list of sites` configuration field, ensure the sites specified match those granted `Sites.Selected` permission in SharePoint. +If the `Comma-separated list of sites` field is set to `*` or the `Enumerate all sites` toggle is enabled, the connector will attempt to access all sites. This requires broader permissions, which are not supported with `Sites.Selected`. +==== + +.Graph API permissions +**** +Microsoft recommends using Graph API for all operations with Sharepoint Online. Graph API is well-documented and more efficient at fetching data, which helps avoid throttling. +Refer to https://learn.microsoft.com/en-us/sharepoint/dev/general-development/how-to-avoid-getting-throttled-or-blocked-in-sharepoint-online[Microsoft's throttling policies^] for more information. + +Here's a summary of why we use these Graph API permissions: + +* *Sites.Selected* is used to fetch the sites and their metadata +* *Files.Read.All* is used to fetch Site Drives and files in these drives +* *Groups.Read.All* is used to fetch groups for document-level permissions +* *User.Read.All* is used to fetch user information for document-level permissions + +Due to the way the Graph API is designed, these permissions are "all or nothing" - it's currently impossible to limit access to these resources. +**** + +[discrete#es-connectors-sharepoint-online-client-compatability] +===== Compatibility + +This connector is compatible with SharePoint Online. + +[discrete#es-connectors-sharepoint-online-client-configuration] +===== Configuration + +[TIP] +==== +When using the <> workflow, initially these fields will use the default configuration set in the {connectors-python}/connectors/sources/sharepoint_online.py[connector source code^]. +These are set in the `get_default_configuration` function definition. + +These configurable fields will be rendered with their respective *labels* in the Kibana UI. +Once connected, you'll be able to update these values in Kibana. +==== + +Use the following configuration fields to set up the connector: + +`tenant_id`:: +The tenant id for the Azure account hosting the Sharepoint Online instance. + +`tenant_name`:: +The tenant name for the Azure account hosting the Sharepoint Online instance. + +`client_id`:: +The client id to authenticate with SharePoint Online. + +`secret_value`:: +The secret value to authenticate with SharePoint Online. + +`site_collections`:: +List of site collection names or paths to fetch from SharePoint. +When enumerating all sites, these values should be the _names_ of the sites. +Use `*` to include all available sites. +Examples: +* `collection1` +* `collection1,sub-collection` +* `*` ++ +When **not** enumerating all sites, these values should be the _paths_ (URL after `/sites/`) of the sites. +Examples: +* `collection1` +* `collection1,collection1/sub-collection` + +`enumerate_all_sites`:: +If enabled, the full list of all sites will be fetched from the API, in bulk, and will be filtered down to match the configured list of site names. +If disabled, each path in the configured list of site paths will be fetched individually from the API. +Enabling this configuration is most useful when syncing large numbers (more than total/200) of sites. +This is because, at high volumes, it is more efficient to fetch sites in bulk. +When syncing fewer sites, disabling this configuration can result in improved performance. +This is because, at low volumes, it is more efficient to only fetch the sites that you need. ++ +[NOTE] +==== +When disabled, `*` is not a valid configuration for `Comma-separated list of sites`. +==== + +`fetch_subsites`:: +Whether sub-sites of the configured site(s) should be automatically fetched. +This option is only available when not enumerating all sites (see above). + +`use_text_extraction_service`:: +Toggle to enable local text extraction service for documents. +Requires a separate deployment of the <>. +Requires that ingest pipeline settings disable text extraction. +Default value is `False`. + +`use_document_level_security`:: +Toggle to enable <>. +When enabled, full and incremental syncs will fetch access control lists for each document and store them in the `_allow_access_control` field. +Access control syncs will fetch users' access control lists and store them in a separate index. ++ +Once enabled, the following granular permissions toggles will be available: + +** *Fetch drive item permissions*: Enable this option to fetch *drive item* specific permissions. +** *Fetch unique page permissions*: Enable this option to fetch unique *page* permissions. If this setting is disabled a page will inherit permissions from its parent site. +** *Fetch unique list permissions*: Enable this option to fetch unique *list* permissions. If this setting is disabled a list will inherit permissions from its parent site. +** *Fetch unique list item permissions*: Enable this option to fetch unique *list item* permissions. If this setting is disabled a list item will inherit permissions from its parent site. ++ +[NOTE] +==== +If left empty the default value `true` will be used for these granular permissions toggles. +Note that these settings may increase sync times. +==== + +[discrete#es-connectors-sharepoint-online-client-docker] +===== Deployment using Docker + +include::_connectors-docker-instructions.asciidoc[] + +[discrete#es-connectors-sharepoint-online-client-documents-syncs] +===== Documents and syncs + +The connector syncs the following SharePoint object types: + +* *Sites* (and subsites) +* *Lists* +* *List items* and *attachment content* +* *Document libraries* and *attachment content* (including web pages) + +[TIP] +==== +*Making Sharepoint Site Pages Web Part content searchable* + +If you're using Web Parts on Sharepoint Site Pages and want to make this content searchable, you'll need to consult the https://learn.microsoft.com/en-us/sharepoint/dev/spfx/web-parts/guidance/integrate-web-part-properties-with-sharepoint#specify-web-part-property-value-type/[official documentation^]. + +We recommend setting `isHtmlString` to *True* for all Web Parts that need to be searchable. +==== + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted by default. Use the <> to handle larger binary files. +* Permissions are not synced by default. Enable <> to sync permissions. +==== + +[discrete#es-connectors-sharepoint-online-client-documents-syncs-limitations] +====== Limitations + +* The connector does not currently sync content from Teams-connected sites. + +[discrete#es-connectors-sharepoint-online-client-sync-rules] +===== Sync rules + +_Basic_ sync rules are identical for all connectors and are available by default. +For more information read <>. + +[discrete#es-connectors-sharepoint-online-client-sync-rules-advanced] +====== Advanced sync rules + +[NOTE] +==== +A <> is required for advanced sync rules to take effect. +==== + +The following section describes *advanced sync rules* for this connector. +Advanced sync rules are defined through a source-specific DSL JSON snippet. + +<> for the Sharepoint Online connector enable you to avoid extracting and syncing older data that might no longer be relevant for search. + +Example: + +[source,js] +---- +{ + "skipExtractingDriveItemsOlderThan": 60 +} + +---- +// NOTCONSOLE + +This rule will not extract content of any drive items (files in document libraries) that haven't been modified for 60 days or more. + +[discrete#es-connectors-sharepoint-online-client-sync-rules-limitations] +*Limitations of sync rules with incremental syncs* + +Changing sync rules after Sharepoint Online content has already been indexed can bring unexpected results, when using <>. + +Incremental syncs ensure _updates_ from 3rd-party system, but do not modify existing documents in the index. + +*To avoid these issues, run a full sync after changing sync rules (basic or advanced).* + +Let's take a look at several examples where incremental syncs might lead to inconsistent data on your index. + +[discrete#es-connectors-sharepoint-online-client-sync-rules-limitations-restrictive-added] +*Example: Restrictive basic sync rule added after a full sync* + +Imagine your Sharepoint Online drive contains the following drive items: + +[source,txt] +---- +/Documents/Report.doc +/Documents/Spreadsheet.xls +/Presentations/Q4-2020-Report.pdf +/Presentations/Q4-2020-Report-Data.xls +/Personal/Documents/Sales.xls +---- + +After a sync, all these drive items will be stored on your Elasticsearch index. +Let's add a basic sync rule, filtering files by their path: + +[source,txt] +---- +Exclude WHERE path CONTAINS "Documents" +---- + +These filtering rules will exclude all files with "Documents" in their path, leaving only files in `/Presentations` directory: + +[source,txt] +---- +/Presentations/Q4-2020-Report.pdf +/Presentations/Q4-2020-Report-Data.xls +---- + +If no files were changed, incremental sync will not receive information about changes from Sharepoint Online and won't be able to delete any files, leaving the index in the same state it was before the sync. + +After a *full sync*, the index will be updated and files that are excluded by sync rules will be removed. + +[discrete#es-connectors-sharepoint-online-client-sync-rules-limitations-restrictive-removed] +*Example: Restrictive basic sync rules removed after a full sync* + +Imagine that Sharepoint Online drive has the following drive items: + +[source,txt] +---- +/Documents/Report.doc +/Documents/Spreadsheet.xls +/Presentations/Q4-2020-Report.pdf +/Presentations/Q4-2020-Report-Data.xls +/Personal/Documents/Sales.xls +---- + +Before doing a sync, we add a restrictive basic filtering rule: + +[source,txt] +---- +Exclude WHERE path CONTAINS "Documents" +---- + +After a full sync, the index will contain only files in the `/Presentations` directory: + +[source,txt] +---- +/Presentations/Q4-2020-Report.pdf +/Presentations/Q4-2020-Report-Data.xls +---- + +Afterwards, we can remove the filtering rule and run an incremental sync. If no changes happened to the files, incremental sync will not mirror these changes in the Elasticsearch index, because Sharepoint Online will not report any changes to the items. +Only a *full sync* will include the items previously ignored by the sync rule. + +[discrete#es-connectors-sharepoint-online-client-sync-rules-limitations-restrictive-changed] +*Example: Advanced sync rules edge case* + +Advanced sync rules can be applied to limit which documents will have content extracted. +For example, it's possible to set a rule so that documents older than 180 days won't have content extracted. + +However, there is an edge case. +Imagine a document that is 179 days old and its content is extracted and indexed into Elasticsearch. +After 2 days, this document will be 181 days old. +Since this document was already ingested it will not be modified. +Therefore, the content will not be removed from the index, following an incremental sync. + +In this situation, if you want older documents to be removed, you will need to clean the index up manually. +For example, you can manually run an Elasticsearch query that removes drive item content older than 180 days: + +[source, console] +---- +POST INDEX_NAME/_update_by_query?conflicts=proceed +{ + "query": { + "bool": { + "filter": [ + { + "match": { + "object_type": "drive_item" + } + }, + { + "exists": { + "field": "file" + } + }, + { + "range": { + "lastModifiedDateTime": { + "lte": "now-180d" + } + } + } + ] + } + }, + "script": { + "source": "ctx._source.body = ''", + "lang": "painless" + } +} +---- +// TEST[skip:TODO] + +[discrete#es-connectors-sharepoint-online-client-dls] +===== Document-level security + +Document-level security (DLS) enables you to restrict access to documents based on a user's permissions. +This feature is available by default for this connector. + +Refer to <> on this page for how to enable DLS for this connector. + +[TIP] +==== +Refer to <> to learn how to ingest data from SharePoint Online with DLS enabled, when building a search application. +==== + +[discrete#es-connectors-sharepoint-online-client-content-extraction] +===== Content extraction + +[discrete#es-connectors-sharepoint-online-client-content-extraction-pipeline] +====== Default content extraction + +The default content extraction service is powered by the Enterprise Search default ingest pipeline. +(See {ref}/ingest-pipeline-search.html[Ingest pipelines for Search indices].) + +See <>. + +[discrete#es-connectors-sharepoint-online-client-content-extraction-local] +====== Local content extraction (for large files) + +The SharePoint Online self-managed connector supports large file content extraction (> *100MB*). +This requires: + +* A self-managed deployment of the Elastic Text Extraction Service. +* Text extraction to be _disabled_ in the default ingest pipeline settings. + +Refer to <> for more information. + +[discrete#es-connectors-sharepoint-online-client-testing] +===== End-to-end testing + +The connector framework enables operators to run functional tests against a real data source. +Refer to <> for more details. + +To perform E2E testing for the SharePoint Online connector, run the following command: + +[source,shell] +---- +$ make ftest NAME=sharepoint_online +---- + +For faster tests, add the `DATA_SIZE=small` flag: + +[source,shell] +---- +make ftest NAME=sharepoint_online DATA_SIZE=small +---- + +[discrete#es-connectors-sharepoint-online-client-known-issues] +===== Known issues + +* *Documents failing to sync due to SharePoint file and folder limits* ++ +SharePoint has limits on the number of files and folders that can be synced. +You might encounter an error like the following written to the body of documents that failed to sync: +`The file size exceeds the allowed limit. CorrelationId: fdb36977-7cb8-4739-992f-49878ada6686, UTC DateTime: 4/21/2022 11:24:22 PM` ++ +Refer to https://support.microsoft.com/en-us/office/download-files-and-folders-from-onedrive-or-sharepoint-5c7397b7-19c7-4893-84fe-d02e8fa5df05#:~:text=Downloads%20are%20subject%20to%20the,zip%20file%20and%2020GB%20overall[SharePoint documentation^] for more information about these limits. ++ +** *Syncing a large number of files* ++ +The connector will fail to download files from folders that contain more than 5000 files. +The List View Threshold (default 5000) is a limit that prevents operations with a high performance impact on the SharePoint Online environment. ++ +*Workaround:* Reduce batch size to avoid this issue. ++ +** *Syncing large files* ++ +SharePoint has file size limits, but these are configurable. ++ +*Workaround:* Increase the file size limit. +Refer to https://learn.microsoft.com/en-us/sharepoint/manage-site-collection-storage-limits#set-automatic-or-manual-site-storage-limits[SharePoint documentation^] for more information. ++ +** *Deleted documents counter is not updated during incremental syncs* ++ +If the configuration `Enumerate All Sites?` is enabled, incremental syncs may not behave as expected. +Drive Item documents that were deleted between incremental syncs may not be detected as deleted. ++ +*Workaround*: Disable `Enumerate All Sites?`, and configure full site paths for all desired sites. + +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-sharepoint-online-client-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-sharepoint-online-client-security] +===== Security + +See <>. + +// Closing the collapsible section +=============== diff --git a/docs/reference/connector/docs/connectors-sharepoint.asciidoc b/docs/reference/connector/docs/connectors-sharepoint.asciidoc new file mode 100644 index 0000000000000..d7a2307a9db80 --- /dev/null +++ b/docs/reference/connector/docs/connectors-sharepoint.asciidoc @@ -0,0 +1,418 @@ +[#es-connectors-sharepoint] +=== Elastic SharePoint Server connector reference +++++ +SharePoint Server +++++ +// Attributes used in this file +:service-name: SharePoint Server +:service-name-stub: sharepoint_server + +The _Elastic SharePoint Server connector_ is a <> for https://www.microsoft.com/en-ww/microsoft-365/sharepoint/[Microsoft SharePoint Server^]. + +This connector is written in Python using the open code {connectors-python}[Elastic connector framework^]. +View the {connectors-python}/connectors/sources/sharepoint_server.py[source code for this connector^]. + +[TIP] +==== +Looking for the SharePoint *Online* connector? See the <>. +==== + + +// //////// //// //// //// //// //// //// //////// +// //////// NATIVE CONNECTOR REFERENCE (MANAGED SERVICE) /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-sharepoint-native-connector-reference] +==== *Elastic managed connector reference* + +.View *Elastic managed connector* reference +[%collapsible] +=============== + +[discrete#es-connectors-sharepoint-availability-prerequisites] +===== Availability and prerequisites + +This connector is available as a managed service since Elastic *8.15.0*. +To use this connector, satisfy all <>. + +[NOTE] +==== +This connector is in *beta* and is subject to change. +Beta features are subject to change and are not covered by the support SLA of generally available (GA) features. +Elastic plans to promote this feature to GA in a future release. +==== + +[discrete#es-connectors-sharepoint-create-connector-client] +===== Create a {service-name} connector +include::_connectors-create-native.asciidoc[] + +[discrete#es-connectors-sharepoint-usage] +===== Usage + +See <>. + +For additional operations, see <>. + +[discrete#es-connectors-sharepoint-compatability] +===== Compatibility + +The following SharePoint Server versions are compatible: + +* SharePoint 2013 +* SharePoint 2016 +* SharePoint 2019 + +[discrete#es-connectors-sharepoint-configuration] +===== Configuration + +The following configuration fields are required to set up the connector: + +`authentication`:: +Authentication mode, either *Basic* or *NTLM*. + +`username`:: +The username of the account for the SharePoint Server instance. + +`password`:: +The password of the account. + +`host_url`:: +The server host url where the SharePoint Server instance is hosted. Examples: +* `https://192.158.1.38:8080` +* `https://.sharepoint.com` + +`site_collections`:: +Comma-separated list of site collections to fetch from SharePoint Server. Examples: +* `collection1` +* `collection1, collection2` + +`ssl_enabled`:: +Whether SSL verification will be enabled. +Default value is `False`. + +`ssl_ca`:: +Content of SSL certificate needed for SharePoint Server. +Keep this field empty, if `ssl_enabled` is set to `False`. ++ +Example certificate: ++ +[source, txt] +---- +-----BEGIN CERTIFICATE----- +MIID+jCCAuKgAwIBAgIGAJJMzlxLMA0GCSqGSIb3DQEBCwUAMHoxCzAJBgNVBAYT +... +7RhLQyWn2u00L7/9Omw= +-----END CERTIFICATE----- +---- + +`retry_count`:: +The number of retry attempts after a failed request to the SharePoint Server instance. Default value is `3`. + +`use_document_level_security`:: +Toggle to enable <>. +When enabled, full syncs will fetch access control lists for each document and store them in the `_allow_access_control` field. Access control syncs fetch users' access control lists and store them in a separate index. ++ +Once enabled, the following granular permissions toggles will be available: ++ +** *Fetch unique list permissions*: Enable this option to fetch unique *list* permissions. If this setting is disabled a list will inherit permissions from its parent site. +** *Fetch unique list item permissions*: Enable this option to fetch unique *list item* permissions. If this setting is disabled a list item will inherit permissions from its parent site. ++ +[NOTE] +==== +If left empty the default value `true` will be used for these granular permissions toggles. +Note that these settings may increase sync times. +==== + +[discrete#es-connectors-sharepoint-documents-syncs] +===== Documents and syncs + +The connector syncs the following SharePoint object types: + +* Sites and Subsites +* Lists +* List Items and its attachment content +* Document Libraries and its attachment content(include Web Pages) + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted by default. Use the <> to handle larger binary files. +* Permissions are not synced by default. Enable <> to sync permissions. +==== + +[discrete#es-connectors-sharepoint-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-sharepoint-document-level-security] +===== Document level security + +Document level security (DLS) enables you to restrict access to documents based on a user’s permissions. +Refer to <> on this page for how to enable DLS for this connector. + +[NOTE] +==== +Refer to <> to learn how to ingest data from a connector with DLS enabled, when building a search application. +The example uses SharePoint _Online_ as the data source, but the same steps apply to every connector. +==== + +[discrete#es-connectors-sharepoint-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +Advanced sync rules are not available for this connector in the present version. +Currently filtering is controlled via ingest pipelines. + +[discrete#es-connectors-sharepoint-content-extraction] +===== Content Extraction + +See <>. + +[discrete#es-connectors-sharepoint-known-issues] +===== Known issues + +There are currently no known issues for this connector. +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-sharepoint-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-sharepoint-security] +===== Security + +See <>. + +[discrete#es-connectors-sharepoint-source] +===== Framework and source + +This connector is written in Python using the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/sharepoint_server.py[source code for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + +// Closing the collapsible section +=============== + + +// //////// //// //// //// //// //// //// //////// +// //////// CONNECTOR CLIENT REFERENCE (SELF-MANAGED) /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-sharepoint-connector-client-reference] +==== *Self-managed connector reference* + +.View *self-managed connector* reference +[%collapsible] +=============== + +[discrete#es-connectors-sharepoint-client-availability-prerequisites] +===== Availability and prerequisites + +This connector is available as a self-managed *self-managed connector*. +This self-managed connector is compatible with Elastic versions *8.9.0+*. +To use this connector, satisfy all <>. + +[NOTE] +==== +This connector is in *beta* and is subject to change. +Beta features are subject to change and are not covered by the support SLA of generally available (GA) features. +Elastic plans to promote this feature to GA in a future release. +==== + +[discrete#es-connectors-sharepoint-client-create-connector-client] +===== Create a {service-name} connector +include::_connectors-create-client.asciidoc[] + +[discrete#es-connectors-sharepoint-client-usage] +===== Usage + +To use this connector as a *self-managed connector*, see <>. + +For additional operations, see <>. + +[discrete#es-connectors-sharepoint-client-compatability] +===== Compatibility + +The following SharePoint Server versions are compatible with the Elastic connector framework: + +* SharePoint 2013 +* SharePoint 2016 +* SharePoint 2019 + +[discrete#es-connectors-sharepoint-client-configuration] +===== Configuration + +[TIP] +==== +When using the <> workflow, initially these fields will use the default configuration set in the {connectors-python}/connectors/sources/sharepoint_server.py[connector source code^]. +These are set in the `get_default_configuration` function definition. + +These configurable fields will be rendered with their respective *labels* in the Kibana UI. +Once connected, you'll be able to update these values in Kibana. +==== + +The following configuration fields are required to set up the connector: + +`authentication`:: +Authentication mode, either *Basic* or *NTLM*. + +`username`:: +The username of the account for the SharePoint Server instance. + +`password`:: +The password of the account. + +`host_url`:: +The server host url where the SharePoint Server instance is hosted. Examples: +* `https://192.158.1.38:8080` +* `https://.sharepoint.com` + +`site_collections`:: +Comma-separated list of site collections to fetch from SharePoint Server. Examples: +* `collection1` +* `collection1, collection2` + +`ssl_enabled`:: +Whether SSL verification will be enabled. +Default value is `False`. + +`ssl_ca`:: +Content of SSL certificate needed for the SharePoint Server instance. +Keep this field empty, if `ssl_enabled` is set to `False`. ++ +Example certificate: ++ +[source, txt] +---- +-----BEGIN CERTIFICATE----- +MIID+jCCAuKgAwIBAgIGAJJMzlxLMA0GCSqGSIb3DQEBCwUAMHoxCzAJBgNVBAYT +... +7RhLQyWn2u00L7/9Omw= +-----END CERTIFICATE----- +---- + +`retry_count`:: +The number of retry attempts after failed request to the SharePoint Server instance. Default value is `3`. + +`use_document_level_security`:: +Toggle to enable <>. +When enabled, full syncs will fetch access control lists for each document and store them in the `_allow_access_control` field. Access control syncs fetch users' access control lists and store them in a separate index. ++ +Once enabled, the following granular permissions toggles will be available: ++ +** *Fetch unique list permissions*: Enable this option to fetch unique *list* permissions. If this setting is disabled a list will inherit permissions from its parent site. +** *Fetch unique list item permissions*: Enable this option to fetch unique *list item* permissions. If this setting is disabled a list item will inherit permissions from its parent site. ++ +[NOTE] +==== +If left empty the default value `true` will be used for these granular permissions toggles. +Note that these settings may increase sync times. +==== + +[discrete#es-connectors-sharepoint-client-docker] +===== Deployment using Docker + +include::_connectors-docker-instructions.asciidoc[] + +[discrete#es-connectors-sharepoint-client-documents-syncs] + +===== Documents and syncs + +The connector syncs the following SharePoint object types: + +* Sites and Subsites +* Lists +* List Items and its attachment content +* Document Libraries and its attachment content(include Web Pages) + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted by default. Use the <> to handle larger binary files. +* Permissions are not synced. **All documents** indexed to an Elastic deployment will be visible to **all users with access** to that Elasticsearch Index. +==== + +[discrete#es-connectors-sharepoint-client-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>, but this feature is currently disabled by default. +Refer to the linked documentation for enabling incremental syncs. + +[discrete#es-connectors-sharepoint-client-document-level-security] +===== Document level security + +Document level security (DLS) enables you to restrict access to documents based on a user’s permissions. +Refer to <> on this page for how to enable DLS for this connector. + +[NOTE] +==== +Refer to <> to learn how to ingest data from a connector with DLS enabled, when building a search application. +The example uses SharePoint Online as the data source, but the same steps apply to every connector. +==== + +[discrete#es-connectors-sharepoint-client-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +Advanced sync rules are not available for this connector in the present version. +Currently filtering is controlled via ingest pipelines. + +[discrete#es-connectors-sharepoint-client-content-extraction] +===== Content Extraction + +See <>. + +[discrete#es-connectors-sharepoint-client-connector-client-operations] +===== Self-managed connector operations + +[discrete#es-connectors-sharepoint-client-testing] +===== End-to-end testing + +The connector framework enables operators to run functional tests against a real data source. +Refer to <> for more details. + +To perform E2E testing for the sharepoint connector, run the following command: + +[source,shell] +---- +$ make ftest NAME=sharepoint_server +---- + +For faster tests, add the `DATA_SIZE=small` flag: + +[source,shell] +---- +make ftest NAME=sharepoint_server DATA_SIZE=small +---- + +[discrete#es-connectors-sharepoint-client-known-issues] +===== Known issues + +There are currently no known issues for this connector. +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-sharepoint-client-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-sharepoint-client-security] +===== Security + +See <>. + +[discrete#es-connectors-sharepoint-client-source] +===== Framework and source + +This connector is written in Python using the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/sharepoint_server.py[source code for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + +// Closing the collapsible section +=============== diff --git a/docs/reference/connector/docs/connectors-slack.asciidoc b/docs/reference/connector/docs/connectors-slack.asciidoc new file mode 100644 index 0000000000000..059394f28d4ec --- /dev/null +++ b/docs/reference/connector/docs/connectors-slack.asciidoc @@ -0,0 +1,344 @@ +[#es-connectors-slack] +=== Elastic Slack connector reference +++++ +Slack +++++ +// Attributes used in this file +:service-name: Slack +:service-name-stub: slack + +The Slack connector is written in Python using the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/{service-name-stub}.py[*source code* for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + +// //////// //// //// //// //// //// //// //////// +// //////// NATIVE CONNECTOR REFERENCE (MANAGED SERVICE) /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-slack-native-connector-reference] +==== *Elastic managed connector reference* + +.View *Elastic managed connector* reference +[%collapsible] +=============== + +[discrete#es-connectors-slack-availability] +===== Availability and prerequisites + +This managed connector was introduced in Elastic *8.14.0* as a managed service on Elastic Cloud. + +To use this connector natively in Elastic Cloud, satisfy all <>. + +[NOTE] +==== +This connector is in **technical preview** and is subject to change. +The design and code is less mature than official GA features and is being provided as-is with no warranties. +Technical preview features are not subject to the support SLA of official GA features. +==== + +[discrete#es-connectors-slack-create-connector-native] +===== Create a {service-name} connector +include::_connectors-create-native.asciidoc[] + +[discrete#es-connectors-slack-usage] +===== Usage + +To use this connector in the UI, select the *Notion* tile when creating a new connector under *Search -> Connectors*. + +If you're already familiar with how connectors work, you can also use the {ref}/connector-apis.html[Connector APIs]. + +For additional operations, see <>. + +[NOTE] +==== +You need to create a Slack application to authenticate with Slack. +==== + +[discrete#es-connectors-slack-app] +====== Create a Slack application + +When created you'll receive a credential that the connector uses for authentication. +A new Bot user will also be created. + +[TIP] +==== +The connector will only sync messages from the channels of which the Bot user is a member. +==== + +To create the app, follow these steps: + +1. Go to https://api.slack.com/apps and click "Create New App". +2. Choose "From Scratch". +3. Name the app, and select the workspace you want to sync from. +Depending on the workspace's settings, you may get a warning about requiring admin approval. +That will be handled later. +4. Navigate to "OAuth & Permissions" in the sidebar. +5. Scroll down to the "Scopes" section and add these scopes: +* `channels:history` +* `channels:read` +* `users:read`. ++ +Optionally, you can also add `channels:join` if you want the App Bot to automatically be able to add itself to public channels. +6. Scroll up to "OAuth Tokens for Your Workspace" and install the application. Your workspace may require you to get administrator approval. If so, request approval now and return to the next step once it has been approved. +7. Copy and save the new "Bot User OAuth Token". +This credential will be used when configuring the connector. + +[discrete#es-connectors-slack-configuration] +===== Configuration + +The following settings are required to set up this connector: + +`token`(required) :: +The Bot User OAuth Token generated by creating and installing your Slack App. + +`fetch_last_n_days`(required) :: +The number of days of history to fetch from Slack. +This must be a positive number to fetch a subset of data, going back that many days. +If set to `0`, it will fetch all data since the beginning of the workspace. +The default is 180 days. + +`auto_join_channels`(required) :: +Whether or not the connector should have the App's Bot User automatically invite itself into all public channels. +The connector will only sync messages from the channels of which the Bot user is a member. +By default, the bot will not invite itself to any channels, and must be manually invited to each channel that you wish to sync. +If this setting is enabled, your App must have the `channels.join` scope. + +`sync_users`(required) :: + +Whether or not the connector should index a document for each Slack user. +By default, the connector will create documents only for Channels and Messages. +However, regardless of the value of this setting, the Slack App does need the `users.read` scope and will make requests to enumerate all of the workspace's users. +This allows the messages to be enriched with human-readable usernames, and not rely on unreadable User UIDs. +Therefore, disabling this setting does not result in a speed improvement, but merely results in less overall storage in Elasticsearch. + +[discrete#es-connectors-slack-sync-rules] +===== Sync rules + +_Basic_ sync rules are identical for all connectors and are available by default. + +Advanced sync rules are not available for this connector in the present version. + +For more information read <>. + +[discrete#es-connectors-slack-content-extraction] +===== Content Extraction + +This connector does not currently support processing Slack attachments or other binary files. + +//See <>. + +[discrete#es-connectors-slack-documents-syncs] +===== Documents and syncs + +The connector syncs the following objects and entities: + +* *Channels* +* *Messages* +* *Users* (configurable) + +[NOTE] +==== +* Only public channels and messages from public channels are synced. +* No permissions are synced. **All documents** indexed to an Elastic deployment will be visible to **all users with access** to that Elastic Deployment. +==== + + +[discrete#es-connectors-slack-known-issues] +===== Known issues + +There are currently no known issues for this connector. +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-slack-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-slack-security] +===== Security + +See <>. + + +// Closing the collapsible section +=============== + + +// //////// //// //// //// //// //// //// //////// +// //////// CONNECTOR CLIENT REFERENCE (SELF-MANAGED) /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-slack-connector-client-reference] +==== *Self-managed connector reference* + +.View *self-managed connector* reference +[%collapsible] +=============== + +[discrete#es-connectors-slack-client-availability] +===== Availability and prerequisites + +This connector is available as a self-managed *self-managed connector* from the *Elastic connector framework*. + +This self-managed connector is compatible with Elastic versions *8.10.0+*. + +To use this connector, satisfy all <>. + +[NOTE] +==== +This connector is in **technical preview** and is subject to change. +The design and code is less mature than official GA features and is being provided as-is with no warranties. +Technical preview features are not subject to the support SLA of official GA features. +==== + +[discrete#es-connectors-slack-client-create-connector-client] +===== Create a {service-name} connector +include::_connectors-create-client.asciidoc[] + +[discrete#es-connectors-slack-client-usage] +===== Usage + +To use this connector as a **self-managed connector**, use the *Connector* workflow in the Kibana UI. + +For additional operations, see <>. + +[NOTE] +==== +You need to create a Slack application to authenticate with Slack. +==== + +[discrete#es-connectors-slack-client-app] +====== Create a Slack application + +When created you'll receive a credential that the connector uses for authentication. +A new Bot user will also be created. + +[TIP] +==== +The connector will only sync messages from the channels of which the Bot user is a member. +==== + +To create the app, follow these steps: + +1. Go to https://api.slack.com/apps and click "Create New App". +2. Choose "From Scratch". +3. Name the app, and select the workspace you want to sync from. +Depending on the workspace's settings, you may get a warning about requiring admin approval. +That will be handled later. +4. Navigate to "OAuth & Permissions" in the sidebar. +5. Scroll down to the "Scopes" section and add these scopes: +* `channels:history` +* `channels:read` +* `users:read`. ++ +Optionally, you can also add `channels:join` if you want the App Bot to automatically be able to add itself to public channels. +6. Scroll up to "OAuth Tokens for Your Workspace" and install the application. Your workspace may require you to get administrator approval. If so, request approval now and return to the next step once it has been approved. +7. Copy and save the new "Bot User OAuth Token". +This credential will be used when configuring the connector. + +[discrete#es-connectors-slack-client-docker] +===== Deploy with Docker + +include::_connectors-docker-instructions.asciidoc[] + +[discrete#es-connectors-slack-client-configuration] +===== Configuration + +The following settings are required to set up this connector: + +`token`(required) :: +The Bot User OAuth Token generated by creating and installing your Slack App. + +`fetch_last_n_days`(required) :: +The number of days of history to fetch from Slack. +This must be a positive number to fetch a subset of data, going back that many days. +If set to `0`, it will fetch all data since the beginning of the workspace. +The default is 180 days. + +`auto_join_channels`(required) :: +Whether or not the connector should have the App's Bot User automatically invite itself into all public channels. +The connector will only sync messages from the channels of which the Bot user is a member. +By default, the bot will not invite itself to any channels, and must be manually invited to each channel that you wish to sync. +If this setting is enabled, your App must have the `channels.join` scope. + +`sync_users`(required) :: + +Whether or not the connector should index a document for each Slack user. +By default, the connector will create documents only for Channels and Messages. +However, regardless of the value of this setting, the Slack App does need the `users.read` scope and will make requests to enumerate all of the workspace's users. +This allows the messages to be enriched with human-readable usernames, and not rely on unreadable User UIDs. +Therefore, disabling this setting does not result in a speed improvement, but merely results in less overall storage in Elasticsearch. + +[discrete#es-connectors-slack-client-sync-rules] +===== Sync rules + +_Basic_ sync rules are identical for all connectors and are available by default. + +Advanced sync rules are not available for this connector in the present version. + +For more information read <>. + +[discrete#es-connectors-slack-client-content-extraction] +===== Content Extraction + +This connector does not currently support processing Slack attachments or other binary files. + +//See <>. + +[discrete#es-connectors-slack-client-documents-syncs] +===== Documents and syncs + +The connector syncs the following objects and entities: + +* *Channels* +* *Messages* +* *Users* (configurably) + +[NOTE] +==== +* Only public channels and messages from public channels are synced. +* No permissions are synced. **All documents** indexed to an Elastic deployment will be visible to **all users with access** to that Elastic Deployment. +==== + +[discrete#es-connectors-slack-client-connector-client-operations] +===== Self-managed connector operations + +[discrete#es-connectors-slack-client-testing] +===== End-to-end testing + +The connector framework enables operators to run functional tests against a real data source. +Refer to <> for more details. + +To perform E2E testing for the GitHub connector, run the following command: + +[source,shell] +---- +$ make ftest NAME=slack +---- + +For faster tests, add the `DATA_SIZE=small` flag: + +[source,shell] +---- +make ftest NAME=slack DATA_SIZE=small +---- + +[discrete#es-connectors-slack-client-known-issues] +===== Known issues + +There are currently no known issues for this connector. +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-slack-client-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-slack-client-security] +===== Security + +See <>. + + +// Closing the collapsible section +=============== diff --git a/docs/reference/connector/docs/connectors-teams.asciidoc b/docs/reference/connector/docs/connectors-teams.asciidoc new file mode 100644 index 0000000000000..cd7d05d884349 --- /dev/null +++ b/docs/reference/connector/docs/connectors-teams.asciidoc @@ -0,0 +1,359 @@ +[#es-connectors-teams] +=== Elastic Microsoft Teams connector reference +++++ +Teams +++++ +// Attributes used in this file +:service-name: Microsoft Teams +:service-name-stub: microsoft_teams + +The Microsoft Teams connector is written in Python using the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/microsoft_teams.py[source code for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + +// //////// //// //// //// //// //// //// //////// +// //////// NATIVE CONNECTOR REFERENCE (MANAGED SERVICE) /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-teams-native-connector-reference] +==== *Elastic managed connector reference* + +.View *Elastic managed connector* reference +[%collapsible] +=============== + +[discrete#es-connectors-microsoft-teams-native-availability-and-prerequisites] +===== Availability and prerequisites + +This managed connector was introduced in Elastic *8.14.0* as a managed service on Elastic Cloud. + +To use this connector natively in Elastic Cloud, satisfy all <>. + +[NOTE] +==== +This connector is in **technical preview** and is subject to change. +The design and code is less mature than official GA features and is being provided as-is with no warranties. +Technical preview features are not subject to the support SLA of official GA features. +==== + +[discrete#es-connectors-teams-create-connector-native] +===== Create a {service-name} connector +include::_connectors-create-native.asciidoc[] + +[discrete#es-connectors-microsoft-teams-native-usage] +===== Usage + +To use this connector in the UI, select the *Teams* tile when creating a new connector under *Search -> Connectors*. + +If you're already familiar with how connectors work, you can also use the {ref}/connector-apis.html[Connector APIs]. + +For additional operations, see <>. + +[discrete#es-connectors-microsoft-teams-native-connecting-to-microsoft-teams] +===== Connecting to Microsoft Teams + +To connect to Microsoft Teams you need to https://learn.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal[create an Azure Active Directory application and service principal] that can access resources. Follow these steps: + +1. Go to the https://portal.azure.com[Azure portal] and sign in with your Azure account. +2. Navigate to the *Azure Active Directory* service. +3. Select *App registrations* from the left-hand menu. +4. Click on the *New registration* button to register a new application. +5. Provide a *name* for your app, and _optionally_ select the supported account types (e.g., single tenant, multi-tenant). +6. Click on the *Register* button to create the app registration. +7. After the registration is complete, you will be redirected to the app's overview page. Take note of the *Application (client) ID* value, as you'll need it later. +8. Scroll down to the *API permissions* section and click on the "Add a permission" button. +9. In the "Request API permissions pane, select "Microsoft Graph" as the API. +10. Select the following *permissions*: ++ +* `TeamMember.Read.All` (Delegated) +* `Team.ReadBasic.All` (Delegated) +* `TeamsTab.Read.All` (Delegated) +* `Group.Read.All` (Delegated) +* `ChannelMessage.Read.All` (Delegated) +* `Chat.Read` (Delegated) & `Chat.Read.All` (Application) +* `Chat.ReadBasic` (Delegated) & `Chat.ReadBasic.All` (Application) +* `Files.Read.All` (Delegated and Application) +* `Calendars.Read` (Delegated and Application) ++ +11. Click on the *Add permissions* button to add the selected permissions to your app. +12. Click on the *Grant admin consent* button to grant the required permissions to the app. This step requires administrative privileges. *If you are not an admin, you need to request the admin to grant consent via their Azure Portal*. +13. Under the "Certificates & Secrets" tab, go to *Client Secrets*. +Generate a new client secret and keep a note of the string under the `Value` column. + +After completion, use the following configuration parameters to configure the connector. + +[discrete#es-connectors-microsoft-teams-native-configuration] +===== Configuration + +The following configuration fields are required: + +`client_id` (required):: +Unique identifier for your Azure Application, found on the app's overview page. Example: +* `ab123453-12a2-100a-1123-93fd09d67394` + +`secret_value` (required):: +String value that the application uses to prove its identity when requesting a token, available under the `Certificates & Secrets` tab of your Azure application menu. Example: +* `eyav1~12aBadIg6SL-STDfg102eBfCGkbKBq_Ddyu` + +`tenant_id` (required):: +Unique identifier for your Azure Active Directory instance, found on the app's overview page. Example: +* `123a1b23-12a3-45b6-7c8d-fc931cfb448d` + +`username` (required):: +Username for your Azure Application. Example: +* `dummy@3hmr2@onmicrosoft.com` + +`password` (required):: +Password for your Azure Application. Example: +* `changeme` + +[discrete#es-connectors-microsoft-teams-native-content-extraction] +====== Content Extraction + +Refer to <>. + +[discrete#es-connectors-microsoft-teams-native-documents-and-syncs] +===== Documents and syncs + +The connector syncs the following objects and entities: + +* *USER_CHATS_MESSAGE* +* *USER_CHAT_TABS* +* *USER_CHAT_ATTACHMENT* +* *USER_CHAT_MEETING_RECORDING* +* *USER_MEETING* +* *TEAMS* +* *TEAM_CHANNEL* +* *CHANNEL_TAB* +* *CHANNEL_MESSAGE* +* *CHANNEL_MEETING* +* *CHANNEL_ATTACHMENT* +* *CALENDAR_EVENTS* + +[NOTE] +==== +* Files bigger than 10 MB won't be extracted. +* Permissions are not synced. *All documents* indexed to an Elastic deployment will be visible to *all users with access* to that Elastic Deployment. +==== + +[discrete#es-connectors-microsoft-teams-native-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-microsoft-teams-native-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +[discrete#es-connectors-microsoft-teams-native-advanced-sync-rules] +===== Advanced Sync Rules + +Advanced sync rules are not available for this connector in the present version. + +[discrete#es-connectors-microsoft-teams-native-known-issues] +===== Known issues + +* Messages in one-on-one chats for _Chat with Self_ users are not fetched via Graph APIs. Therefore, these messages won't be indexed into Elasticsearch. + +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-microsoft-teams-native-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-microsoft-teams-native-security] +===== Security + +See <>. + +// Closing the collapsible section +=============== + + +// //////// //// //// //// //// //// //// //////// +// //////// CONNECTOR CLIENT REFERENCE (SELF-MANAGED) /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-teams-connector-client-reference] +==== *Self-managed connector reference* + +.View *self-managed connector* reference +[%collapsible] +=============== + +[discrete#es-connectors-microsoft-teams-availability-and-prerequisites] +===== Availability and prerequisites + +This connector is available as a self-managed *self-managed connector*. +To use this connector, satisfy all <>. + +[NOTE] +==== +This connector is in *technical preview* and is subject to change. +The design and code is less mature than official GA features and is being provided as-is with no warranties. +Technical preview features are not subject to the support SLA of official GA features. +==== + +[discrete#es-connectors-teams-client-create-connector-client] +===== Create a {service-name} connector +include::_connectors-create-client.asciidoc[] + +[discrete#es-connectors-microsoft-teams-usage] +===== Usage + +To use this connector as a *self-managed connector*, use the *Microsoft Teams* tile from the connectors list *Customized connector* workflow. + +For additional operations, see <>. + +[discrete#es-connectors-microsoft-teams-connecting-to-microsoft-teams] +===== Connecting to Microsoft Teams + +To connect to Microsoft Teams you need to https://learn.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal[create an Azure Active Directory application and service principal] that can access resources. Follow these steps: + +1. Go to the https://portal.azure.com[Azure portal] and sign in with your Azure account. +2. Navigate to the *Azure Active Directory* service. +3. Select *App registrations* from the left-hand menu. +4. Click on the *New registration* button to register a new application. +5. Provide a *name* for your app, and _optionally_ select the supported account types (e.g., single tenant, multi-tenant). +6. Click on the *Register* button to create the app registration. +7. After the registration is complete, you will be redirected to the app's overview page. Take note of the *Application (client) ID* value, as you'll need it later. +8. Scroll down to the *API permissions* section and click on the "Add a permission" button. +9. In the "Request API permissions pane, select "Microsoft Graph" as the API. +10. Select the following *permissions*: ++ +* `TeamMember.Read.All` (Delegated) +* `Team.ReadBasic.All` (Delegated) +* `TeamsTab.Read.All` (Delegated) +* `Group.Read.All` (Delegated) +* `ChannelMessage.Read.All` (Delegated) +* `Chat.Read` (Delegated) & `Chat.Read.All` (Application) +* `Chat.ReadBasic` (Delegated) & `Chat.ReadBasic.All` (Application) +* `Files.Read.All` (Delegated and Application) +* `Calendars.Read` (Delegated and Application) ++ +11. Click on the *Add permissions* button to add the selected permissions to your app. +12. Click on the *Grant admin consent* button to grant the required permissions to the app. This step requires administrative privileges. *If you are not an admin, you need to request the admin to grant consent via their Azure Portal*. +13. Under the "Certificates & Secrets" tab, go to *Client Secrets*. +Generate a new client secret and keep a note of the string under the `Value` column. + +After completion, use the following configuration parameters to configure the connector. + +[discrete#es-connectors-microsoft-teams-configuration] +===== Configuration + +The following configuration fields are required: + +`client_id` (required):: +Unique identifier for your Azure Application, found on the app's overview page. Example: +* `ab123453-12a2-100a-1123-93fd09d67394` + +`secret_value` (required):: +String value that the application uses to prove its identity when requesting a token, available under the `Certificates & Secrets` tab of your Azure application menu. Example: +* `eyav1~12aBadIg6SL-STDfg102eBfCGkbKBq_Ddyu` + +`tenant_id` (required):: +Unique identifier for your Azure Active Directory instance, found on the app's overview page. Example: +* `123a1b23-12a3-45b6-7c8d-fc931cfb448d` + +`username` (required):: +Username for your Azure Application. Example: +* `dummy@3hmr2@onmicrosoft.com` + +`password` (required):: +Password for your Azure Application. Example: +* `changeme` + +[discrete#es-connectors-microsoft-teams-client-docker] +====== Deployment using Docker + +include::_connectors-docker-instructions.asciidoc[] + +[discrete#es-connectors-microsoft-teams-content-extraction] +====== Content Extraction + +Refer to <>. + +[discrete#es-connectors-microsoft-teams-documents-and-syncs] +===== Documents and syncs + +The connector syncs the following objects and entities: + +* *USER_CHATS_MESSAGE* +* *USER_CHAT_TABS* +* *USER_CHAT_ATTACHMENT* +* *USER_CHAT_MEETING_RECORDING* +* *USER_MEETING* +* *TEAMS* +* *TEAM_CHANNEL* +* *CHANNEL_TAB* +* *CHANNEL_MESSAGE* +* *CHANNEL_MEETING* +* *CHANNEL_ATTACHMENT* +* *CALENDAR_EVENTS* + +[NOTE] +==== +* Files bigger than 10 MB won't be extracted. +* Permissions are not synced. *All documents* indexed to an Elastic deployment will be visible to *all users with access* to that Elastic Deployment. +==== + +[discrete#es-connectors-microsoft-teams-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-microsoft-teams-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +[discrete#es-connectors-microsoft-teams-advanced-sync-rules] +===== Advanced Sync Rules + +Advanced sync rules are not available for this connector in the present version. + +[discrete#es-connectors-microsoft-teams-end-to-end-testing] +===== End-to-end Testing + +The connector framework enables operators to run functional tests against a real data source. +Refer to <> for more details. + +To perform E2E testing for the Teams connector, run the following command: + +[source,shell] +---- +$ make ftest NAME=microsoft_teams +---- + +For faster tests, add the `DATA_SIZE=small` flag: + +[source,shell] +---- +make ftest NAME=microsoft_teams DATA_SIZE=small +---- + +[discrete#es-connectors-microsoft-teams-known-issues] +===== Known issues + +* Messages in one-on-one chats for _Chat with Self_ users are not fetched via Graph APIs. Therefore, these messages won't be indexed into Elasticsearch. + +Refer to <> for a list of known issues for all connectors. + +[discrete#es-connectors-microsoft-teams-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-microsoft-teams-security] +===== Security + +See <>. + +// Closing the collapsible section +=============== diff --git a/docs/reference/connector/docs/connectors-troubleshooting.asciidoc b/docs/reference/connector/docs/connectors-troubleshooting.asciidoc new file mode 100644 index 0000000000000..798e4e13a253d --- /dev/null +++ b/docs/reference/connector/docs/connectors-troubleshooting.asciidoc @@ -0,0 +1,14 @@ +[#es-connectors-troubleshooting] +=== Troubleshooting connectors +++++ +Troubleshooting +++++ + +Use the following actions to help diagnose and resolve issues with <> and <>: + +* <>. +* <>. +* <>. +* Edit your index configuration: <>, <>. + +You can also request help or support. diff --git a/docs/reference/connector/docs/connectors-usage.asciidoc b/docs/reference/connector/docs/connectors-usage.asciidoc new file mode 100644 index 0000000000000..e48c503971e42 --- /dev/null +++ b/docs/reference/connector/docs/connectors-usage.asciidoc @@ -0,0 +1,225 @@ +[#es-connectors-usage] +== Connectors UI in {kib} + +This document describes operations available to <> and <>, using the UI. + +In the Kibana UI, navigate to *Search > Content > Connectors* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. Here, you can view a summary of all your connectors and sync jobs, and to create new connectors. + +[TIP] +==== +In 8.12 we introduced a set of {ref}/connector-apis.html[Connector APIs] to create and manage Elastic connectors and sync jobs, along with a https://github.com/elastic/connectors/blob/main/docs/CLI.md[CLI tool]. +Use these tools if you'd like to work with connectors and sync jobs programmatically, without using the UI. +==== + +[discrete#es-connectors-usage-index-create] +=== Create and configure connectors + +You connector writes data to an {es} index. + +To create <> or self-managed <>, use the buttons under *Search > Content > Connectors*. +Once you've chosen the data source type you'd like to sync, you'll be prompted to create an {es} index. + +[discrete#es-connectors-usage-indices] +=== Manage connector indices + +View and manage all Elasticsearch indices managed by connectors. + +In the {kib} UI, navigate to *Search > Content > Connectors* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. Here, you can view a list of connector indices and their attributes, including connector type health and ingestion status. + +Within this interface, you can choose to view the details for each existing index or delete an index. +Or, you can <>. + +These operations require access to Kibana and additional index privileges. + +[discrete#es-connectors-usage-index-create-configure-existing-index] +=== Customize connector index mappings and settings + +{es} stores your data as documents in an index. Each index is made up of a set of fields and each field has a type (such as `keyword`, `boolean`, or `date`). + +*Mapping* is the process of defining how a document, and the fields it contains, are stored and indexed. +Connectors use {ref}/dynamic-field-mapping.html[dynamic mapping] to automatically create mappings based on the data fetched from the source. + +Index *settings* are configurations that can be adjusted on a per-index basis. They control things like the index's performance, the resources it uses, and how it should handle operations. + +When you create an index with a connector, the index is created with _default_ search-optimized field template mappings and index settings. Mappings for specific fields are then dynamically created based on the data fetched from the source. + +You can inspect your index mappings in the following ways: + +* *In the {kib} UI*: Navigate to *Search > Content > Indices > _YOUR-INDEX_ > Index Mappings* +* *By API*: Use the {ref}/indices-get-mapping.html[Get mapping API] + +You can manually *edit* the mappings and settings via the {es} APIs: + +* Use the {ref}/indices-put-mapping.html[Put mapping API] to update index mappings. +* Use the {ref}/indices-update-settings.html[Update index settings API] to update index settings. + +It's important to note that these updates are more complex when the index already contains data. + +Refer to the following sections for more information. + +[discrete#es-connectors-usage-index-create-configure-existing-index-no-data] +==== Customize mappings and settings before syncing data + +Updating mappings and settings is simpler when your index has no data. +If you create and attach a _new_ index while setting up a connector, you can customize the mappings and settings before syncing data, using the APIs mentioned earlier. + +[discrete#es-connectors-usage-index-create-configure-existing-index-have-data] +==== Customize mappings and settings after syncing data + +Once data has been added to {es} using dynamic mappings, you can't directly update existing field mappings. +If you've already synced data into an index and want to change the mappings, you'll need to {ref}/docs-reindex.html[reindex your data]. + +The workflow for these updates is as follows: + +. {ref}/indices-create-index.html[Create] a new index with the desired mappings and settings. +. {ref}/docs-reindex.html[Reindex] your data from the old index into this new index. +. Delete the old index. +. (Optional) Use an {ref}/aliases.html[alias], if you want to retain the old index name. +. Attach your connector to the new index or alias. + +[discrete#es-connectors-usage-syncs-recurring] +=== Manage recurring syncs + +After creating an index to be managed by a connector, you can configure automatic, recurring syncs. + +In the {kib} UI, navigate to *Search > Content > Connectors* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. + +Choose the index to configure, and then choose the *Scheduling* tab. + +Within this interface, you can enable or disable scheduled: + +. Full content syncs +. Incremental content syncs (if supported) +. Access control syncs (if supported) + +When enabled, you can additionally manage the sync schedule. + +This operation requires access to Kibana and the `write` {ref}/security-privileges.html[indices privilege^] for the `.elastic-connectors` index. + +Alternatively, you can <>. + +After you enable recurring syncs or sync once, the first sync will begin. +(There may be a short delay before the connector service begins the first sync.) +You may want to <> to see the status or errors, or <>. + +[discrete#es-connectors-usage-syncs-manual] +=== Sync once + +After creating the index to be managed by a connector, you can request a single sync at any time. + +In the {kib} UI, navigate to *Search > Content > Elasticsearch indices* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. + +Then choose the index to sync. + +Regardless of which tab is active, the *Sync* button is always visible in the top right. +Choose this button to reveal sync options: + +. Full content +. Incremental content (if supported) +. Access control (if supported) + +Choose one of the options to request a sync. +(There may be a short delay before the connector service begins the sync.) + +This operation requires access to Kibana and the `write` {ref}/security-privileges.html[indices privilege^] for the `.elastic-connectors` index. + +[discrete#es-connectors-usage-syncs-cancel] +=== Cancel sync + +After a sync has started, you can cancel the sync before it completes. + +In the {kib} UI, navigate to *Search > Content > Elasticsearch indices* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. + +Then choose the index with the running sync. + +Regardless of which tab is active, the *Sync* button is always visible in the top right. +Choose this button to reveal sync options, and choose *Cancel Syncs* to cancel active syncs. +This will cancel the running job, and marks all _pending_ and _suspended_ jobs as canceled as well. +(There may be a short delay before the connector service cancels the syncs.) + +This operation requires access to Kibana and the `write` {ref}/security-privileges.html[indices privilege^] for the `.elastic-connectors` and `.elastic-connectors-sync-jobs` index. + +[discrete#es-connectors-usage-index-view] +=== View status + +View the index details to see a variety of information that communicate the status of the index and connector. + +In the {kib} UI, navigate to *Search > Content > Elasticsearch indices* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. + +Then choose the index to view. + +The *Overview* tab presents a variety of information, including: + +* General information about the connector index, for example: name, description, ingestion type, connector type, and language analyzer. +* Any errors affecting the connector or sync process. +* The current ingestion status (see below for possible values). +* The current document count. + +Possible values of ingestion status: + +* Incomplete - A connector that is not configured yet. +* Configured - A connector that is configured. +* Connected - A connector that can successfully connect to a data source. +* Error - A connector that failed to connect to the data source. +* Connector failure - A connector that has not seen any update for more than 30 minutes. +* Sync failure - A connector that failed in the last sync job. + +This tab also displays the recent sync history, including sync status (see below for possible values). + +Possible values of sync status: + +* Sync pending - The initial job status, the job is pending to be picked up. +* Sync in progress - The job is running. +* Canceling sync - Cancelation of the job has been requested. +* Sync canceled - The job was canceled +* Sync suspended - The job was suspended due to service shutdown, and it can be resumed when the service restarts. +* Sync complete - The job completed successfully. +* Sync failure - The job failed. + +For each sync, choose the `view` button to display the job details, including: + +* The job ID +* Document stats, including: number of documents added/deleted, total number of documents, and volume of documented added +* Event logs +* Sync rules that were active when the sync was requested +* Pipelines that were active when the sync was requested + +This operation requires access to Kibana and the `read` {ref}/security-privileges.html[indices privilege^] for the `.elastic-connectors` index. + +[discrete#es-connectors-usage-documents] +=== View documents + +View the documents the connector has synced from the data. +Additionally view the index mappings to determine the current document schema. + +In the {kib} UI, navigate to *Search > Content > Elasticsearch indices* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. + +Then choose the index to view. + +Choose the *Documents* tab to view the synced documents. +Choose the *Index Mappings* tab to view the index mappings that were created by the connector. + +When setting up a new connector, ensure you are getting the documents and fields you were expecting from the data source. +If not, see <> for help. + +These operations require access to Kibana and the `read` and `manage` {ref}/security-privileges.html[indices privileges^] for the index containing the documents. + +See <> for security details. + +[discrete#es-connectors-usage-sync-rules] +=== Manage sync rules + +Use <> to limit which documents are fetched from the data source, or limit which fetched documents are stored in Elastic. + +In the {kib} UI, navigate to *Search > Content > Elasticsearch indices* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. + +Then choose the index to manage and choose the *Sync rules* tab. + +[discrete#es-connectors-usage-pipelines] +=== Manage ingest pipelines + +Use {ref}/ingest-pipeline-search.html[ingest pipelines] to transform fetched data before it is stored in Elastic. + +In the {kib} UI, navigate to *Search > Content > Elasticsearch indices* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. + +Then choose the index to manage and choose the *Pipelines* tab. diff --git a/docs/reference/connector/docs/connectors-use-cases.asciidoc b/docs/reference/connector/docs/connectors-use-cases.asciidoc new file mode 100644 index 0000000000000..0fd6e81a8e483 --- /dev/null +++ b/docs/reference/connector/docs/connectors-use-cases.asciidoc @@ -0,0 +1,11 @@ +[#es-connectors-use-cases] +== Connectors use cases +++++ +Use cases +++++ + +Learn how to use connectors for your use case. + +* <> + +include::connectors-architecture.asciidoc[] \ No newline at end of file diff --git a/docs/reference/connector/docs/connectors-zoom.asciidoc b/docs/reference/connector/docs/connectors-zoom.asciidoc new file mode 100644 index 0000000000000..d945a0aec3da1 --- /dev/null +++ b/docs/reference/connector/docs/connectors-zoom.asciidoc @@ -0,0 +1,364 @@ +[#es-connectors-zoom] +=== Elastic Zoom connector reference +++++ +Zoom +++++ +// Attributes used in this file +:service-name: Zoom +:service-name-stub: zoom + +The Zoom connector is written in Python using the {connectors-python}[Elastic connector framework^]. + +View the {connectors-python}/connectors/sources/{service-name-stub}.py[*source code* for this connector^] (branch _{connectors-branch}_, compatible with Elastic _{minor-version}_). + + +// //////// //// //// //// //// //// //// //////// +// //////// NATIVE CONNECTOR REFERENCE (MANAGED SERVICE) /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-zoom-native-connector-reference] +==== *Elastic managed connector reference* + +.View *Elastic managed connector* reference +[%collapsible] +=============== + +[discrete#es-connectors-zoom-connector-availability-and-prerequisites] +===== Availability and prerequisites + +This managed connector was introduced in Elastic *8.14.0* as a managed service on Elastic Cloud. + +To use this connector natively in Elastic Cloud, satisfy all <>. + +[NOTE] +==== +This connector is in **technical preview** and is subject to change. +The design and code is less mature than official GA features and is being provided as-is with no warranties. +Technical preview features are not subject to the support SLA of official GA features. +==== + +[discrete#es-connectors-zoom-create-native-connector] +===== Create a {service-name} connector +include::_connectors-create-native.asciidoc[] + +[discrete#es-connectors-zoom-connector-usage] +===== Usage + +To use this connector in the UI, select the *Zoom* tile when creating a new connector under *Search -> Connectors*. + +If you're already familiar with how connectors work, you can also use the {ref}/connector-apis.html[Connector APIs]. + +For additional operations, see <>. + +[discrete#es-connectors-zoom-connector-connecting-to-zoom] +===== Connecting to Zoom + +To connect to Zoom you need to https://developers.zoom.us/docs/internal-apps/s2s-oauth/[create an Server-to-Server OAuth application] that can access resources. Follow these steps: + +1. Go to the https://marketplace.zoom.us/[Zoom App Marketplace] and sign in with your Zoom account. +2. Navigate to the "Develop" service. +3. Select "Build App" from the dropdown menu. +4. Click on the "Server-to-Server OAuth" button to register a new application. +5. Provide a name for your app. +6. Click on the "Create" button to create the app registration. +7. After the registration is complete, you will be redirected to the app's overview page. Take note of the "App Credentials" value, as you'll need it later. +8. Navigate to the "Scopes" section and click on the "Add Scopes" button. +9. The following granular scopes need to be added to the app. ++ +[source,bash] +---- +user:read:list_users:admin +meeting:read:list_meetings:admin +meeting:read:list_past_participants:admin +cloud_recording:read:list_user_recordings:admin +team_chat:read:list_user_channels:admin +team_chat:read:list_user_messages:admin +---- +[NOTE] +==== +The connector requires a minimum scope of `user:read:list_users:admin` to ingest data into Elasticsearch. +==== ++ +10. Click on the "Done" button to add the selected scopes to your app. +11. Navigate to the "Activation" section and input the necessary information to activate the app. + +After completion, use the following configuration parameters to configure the connector. + +[discrete#es-connectors-zoom-connector-configuration] +===== Configuration + +The following configuration fields are required: + +`Zoom application Account ID`:: (required) +"Account ID" is a unique identifier associated with a specific Zoom account within the Zoom platform, found on the app's overview page. Example: + +* `KVx-aQssTOutOAGrDfgMaA` + +`Zoom application Client ID`:: (required) +"Client ID" refers to a unique identifier associated with an application that integrates with the Zoom platform, found on the app's overview page. Example: + +* `49Z69_rnRiaF4JYyfHusw` + +`Zoom application Client Secret`:: (required) +The "Client Secret" refers to a confidential piece of information generated when developers register an application on the Zoom Developer Portal for integration with the Zoom platform, found on the app's overview page. Example: + +* `eieiUJRsiH543P5NbYadavczjkqgdRTw` + +`Recording Age Limit (Months)`:: (required) +How far back in time to request recordings from Zoom. Recordings older than this will not be indexed. This configuration parameter allows you to define a time limit, measured in months, for which recordings will be indexed. + +`Fetch past meeting details`:: +Retrieve more information about previous meetings, including their details and participants. Default value is `False`. Enable this option to fetch past meeting details. This setting can increase sync time. + +[discrete#es-connectors-zoom-connector-content-extraction] +====== Content Extraction + +Refer to <>. + +[discrete#es-connectors-zoom-connector-documents-and-syncs] +===== Documents and syncs + +The connector syncs the following objects and entities: + +* *Users* +* *Live Meetings* +* *Upcoming Meetings* +* *Past Meetings* +* *Recordings* +* *Channels* +* *Chat Messages* +* *Chat Files* + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted. (Self-managed connectors can use the <> to handle larger binary files.) +* Permissions are not synced. *All documents* indexed to an Elastic deployment will be visible to *all users with access* to that Elastic Deployment. +==== + +[discrete#es-connectors-zoom-connector-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-zoom-connector-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +[discrete#es-connectors-zoom-connector-advanced-sync-rules] +===== Advanced Sync Rules + +Advanced sync rules are not available for this connector in the present version. + +[discrete#es-connectors-zoom-connector-known-issues] +===== Known issues + +* *Meetings*: Users can only index meetings that are less than a month old. +* *Chat Messages & Files*:Users can only index chats and files that are less than 6 months old. + +Refer to <> for a list of known issues for _all_ connectors. + +[discrete#es-connectors-zoom-connector-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-zoom-connector-security] +===== Security + +See <>. + + +// Closing the collapsible section +=============== + + +// //////// //// //// //// //// //// //// //////// +// //////// CONNECTOR CLIENT REFERENCE (SELF-MANAGED) /////// +// //////// //// //// //// //// //// //// //////// + +[discrete#es-connectors-zoom-connector-client-reference] +==== *Self-managed connector reference* + +.View *self-managed connector* reference +[%collapsible] +=============== + +[discrete#es-connectors-zoom-client-connector-availability-and-prerequisites] +===== Availability and prerequisites + +This connector is available as a self-managed *self-managed connector*. To use this connector, satisfy all <>. + +[NOTE] +==== +This connector is in *technical preview* and is subject to change. +The design and code is less mature than official GA features and is being provided as-is with no warranties. +Technical preview features are not subject to the support SLA of official GA features. +==== + +[discrete#es-connectors-zoom-client-create-connector-client] +===== Create a {service-name} connector +include::_connectors-create-client.asciidoc[] + +[discrete#es-connectors-zoom-client-connector-usage] +===== Usage + +To use this connector in the UI, select the *Teams* tile when creating a new connector under *Search -> Connectors*. + +If you're already familiar with how connectors work, you can also use the {ref}/connector-apis.html[Connector APIs]. + +For additional operations, see <>. + +[discrete#es-connectors-zoom-client-connector-connecting-to-zoom] +===== Connecting to Zoom + +To connect to Zoom you need to https://developers.zoom.us/docs/internal-apps/s2s-oauth/[create an Server-to-Server OAuth application] that can access resources. Follow these steps: + +1. Go to the https://marketplace.zoom.us/[Zoom App Marketplace] and sign in with your Zoom account. +2. Navigate to the "Develop" service. +3. Select "Build App" from the dropdown menu. +4. Click on the "Server-to-Server OAuth" button to register a new application. +5. Provide a name for your app. +6. Click on the "Create" button to create the app registration. +7. After the registration is complete, you will be redirected to the app's overview page. Take note of the "App Credentials" value, as you'll need it later. +8. Navigate to the "Scopes" section and click on the "Add Scopes" button. +9. The following granular scopes need to be added to the app. ++ +[source,bash] +---- +user:read:list_users:admin +meeting:read:list_meetings:admin +meeting:read:list_past_participants:admin +cloud_recording:read:list_user_recordings:admin +team_chat:read:list_user_channels:admin +team_chat:read:list_user_messages:admin +---- +[NOTE] +==== +The connector requires a minimum scope of `user:read:list_users:admin` to ingest data into Elasticsearch. +==== ++ +10. Click on the "Done" button to add the selected scopes to your app. +11. Navigate to the "Activation" section and input the necessary information to activate the app. + +After completion, use the following configuration parameters to configure the connector. + +[discrete#es-connectors-zoom-client-connector-configuration] +===== Configuration + +The following configuration fields are required: + +`Zoom application Account ID`:: (required) +"Account ID" is a unique identifier associated with a specific Zoom account within the Zoom platform, found on the app's overview page. Example: + +* `KVx-aQssTOutOAGrDfgMaA` + +`Zoom application Client ID`:: (required) +"Client ID" refers to a unique identifier associated with an application that integrates with the Zoom platform, found on the app's overview page. Example: + +* `49Z69_rnRiaF4JYyfHusw` + +`Zoom application Client Secret`:: (required) +The "Client Secret" refers to a confidential piece of information generated when developers register an application on the Zoom Developer Portal for integration with the Zoom platform, found on the app's overview page. Example: + +* `eieiUJRsiH543P5NbYadavczjkqgdRTw` + +`Recording Age Limit (Months)`:: (required) +How far back in time to request recordings from Zoom. Recordings older than this will not be indexed. This configuration parameter allows you to define a time limit, measured in months, for which recordings will be indexed. + +`Fetch past meeting details`:: +Retrieve more information about previous meetings, including their details and participants. Default value is `False`. Enable this option to fetch past meeting details. This setting can increase sync time. + +[discrete#es-connectors-zoom-client-client-docker] +====== Deployment using Docker + +include::_connectors-docker-instructions.asciidoc[] + +[discrete#es-connectors-zoom-client-connector-content-extraction] +====== Content Extraction + +Refer to <>. + +[discrete#es-connectors-zoom-client-connector-documents-and-syncs] +===== Documents and syncs + +The connector syncs the following objects and entities: + +* *Users* +* *Live Meetings* +* *Upcoming Meetings* +* *Past Meetings* +* *Recordings* +* *Channels* +* *Chat Messages* +* *Chat Files* + +[NOTE] +==== +* Content from files bigger than 10 MB won't be extracted by default. You can use the <> to handle larger binary files. +* Permissions are not synced. *All documents* indexed to an Elastic deployment will be visible to *all users with access* to that Elastic Deployment. +==== + +[discrete#es-connectors-zoom-client-connector-sync-types] +====== Sync types + +<> are supported by default for all connectors. + +This connector also supports <>. + +[discrete#es-connectors-zoom-client-connector-sync-rules] +===== Sync rules + +<> are identical for all connectors and are available by default. + +[discrete#es-connectors-zoom-client-connector-advanced-sync-rules] +===== Advanced Sync Rules + +Advanced sync rules are not available for this connector in the present version. + +[discrete#es-connectors-zoom-client-connector-connector-client-operations] +===== Connector Client operations + +[discrete#es-connectors-zoom-client-connector-end-to-end-testing] +====== End-to-end Testing + +The connector framework enables operators to run functional tests against a real data source. +Refer to <> for more details. + +To perform E2E testing for the Zoom connector, run the following command: + +[source,shell] +---- +$ make ftest NAME=zoom +---- + +For faster tests, add the `DATA_SIZE=small` flag: + +[source,shell] +---- +make ftest NAME=zoom DATA_SIZE=small +---- + +[discrete#es-connectors-zoom-client-connector-known-issues] +===== Known issues + +* *Meetings*: Users can only index meetings that are less than a month old. +* *Chat Messages & Files*:Users can only index chats and files that are less than 6 months old. + +Refer to <> for a list of known issues for _all_ connectors. + +[discrete#es-connectors-zoom-client-connector-troubleshooting] +===== Troubleshooting + +See <>. + +[discrete#es-connectors-zoom-client-connector-security] +===== Security + +See <>. + + +// Closing the collapsible section +=============== diff --git a/docs/reference/connector/docs/dls-e2e-guide.asciidoc b/docs/reference/connector/docs/dls-e2e-guide.asciidoc new file mode 100644 index 0000000000000..3670ed0730bc7 --- /dev/null +++ b/docs/reference/connector/docs/dls-e2e-guide.asciidoc @@ -0,0 +1,439 @@ +[#es-dls-e2e-guide] +=== Leverage document-level security from connectors in Search Applications +++++ +DLS in Search Applications +++++ + +This guide explains how to ensure document-level security (DLS) for documents ingested by <>, when building a search application. + +In this example we will: + +* Set up the SharePoint Online connector to ingest data from SharePoint Online +* Set up a *Search Application* using the Elasticsearch index created by the SharePoint Online connector +* Create Elasticsearch *API keys* with DLS and workflow restrictions to query your Search Application +* Build a search experience where authenticated users can search over the data ingested by connectors + +[discrete#es-dls-e2e-guide-connector-setup] +==== Set up connector to sync data with access control + +You can run SharePoint Online connector in Elastic Cloud (native) or on a self-managed deployment (self-managed connector). +Refer to <> to learn how to set up the SharePoint Online connector and enable DLS. + + +To run the self-managed connector, you'll need to run the *connectors service* in addition to your Elastic deployment. +Refer to <> for details on how to set up a self-managed connector and run the connectors service. + +[TIP] +==== +This guide assumes you already have an Elastic deployment, that satisfies the <> for running the connectors service. +If you don't have an Elastic deployment, sign up for a https://cloud.elastic.co/registration[free Elastic Cloud trial^]. +==== + +[NOTE] +==== +We use the SharePoint Online connector in this concrete example. +Refer to <> for a list of connectors that support DLS. +==== + +[discrete#es-dls-e2e-guide-sharepoint-data-overview] +==== Elasticsearch indices overview + +When the SharePoint Online connector is set up and you've started syncing content, the connector will create two separate Elasticsearch indices: + +* A *content* index that holds the searchable data in SharePoint Online. +We'll use this index to create our search application. +* An *access control* index that includes access control data for each user that has access to SharePoint Online. +It will be named `.search-acl-filter-`, where `` is the index name you chose. +For example, an index named `search-sharepoint` would have the ACL filter index `.search-acl-filter-search-sharepoint`. +We'll use this index to create Elasticsearch API keys that control access to the content index. + +[discrete#es-dls-e2e-guide-search-application-create] +==== Create a Search Application + +To build our search experience for our SharePoint Online data, we need to create a Search Application. + +Follow these steps to create a Search Application in the Kibana UI: + +. Navigate to *Search > Search Applications* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. +. Select *Create*. +. *Name* the Search Application. +. Select the *index* used by the SharePoint Online connector. +. Select *Create*. + +Alternatively, you can use the {ref}/put-search-application.html[Put Search Application] API. + +[discrete#es-dls-e2e-guide-elasticsearch-api-keys-setup] +==== Create Elasticsearch API keys + +Next we need to create Elasticsearch API keys to restrict queries to the search application. +These restrictions will ensure that users can only query documents they have access to. +To create this API key, we will leverage information in the access control index created by the connector. + +The access control index will contain documents similar to this example: + +[source,js] +---- +{ + "_index": ".search-acl-filter-search-sharepoint", + "_id": "john@example.co", + "_version": 1, + "_seq_no": 0, + "_primary_term": 1, + "found": true, + "_source": { + "identity": { + "email": "john@example.co", + "access_control": [ + "john@example.co", + "Engineering Members" + ] + }, + "query": { + "template": { + "params": { + "access_control": [ + "john@example.co", + "Engineering Members" + ] + }, + "source": """ + { + "bool": { + "should": [ + { + "bool": { + "must_not": { + "exists": { + "field": "_allow_access_control" + } + } + } + }, + { + "terms": { + "_allow_access_control.enum": {{#toJson}}access_control{{/toJson}} + } + } + ] + } + } + """ + } + } + } +} +---- +// NOTCONSOLE + +This document contains the Elasticsearch query that describes which documents the user `john@example.com` has access to. +The access control information is stored in the `access_control` field. +In this case the user has access only to documents that contain `"john@example.co"` or `"Engineering Members"` in the `_allow_access_control` field. + +The `query` field contains the DLS query we will use to create an Elasticsearch API key. +That key will ensure queries are restricted to the documents `john@example.com` has access to. + +To create the API key, we will use the {ref}/security-api-create-api-key.html[Create API Key] API. +The API call will look like this: + +[source,console] +---- +POST /_security/api_key +{ + "name": "john-api-key", + "expiration": "1d", + "role_descriptors": { + "sharepoint-online-role": { + "index": [ + { + "names": [ + "sharepoint-search-application" + ], + "privileges": [ + "read" + ], + "query": { + "template": { + "params": { + "access_control": [ + "john@example.co", + "Engineering Members" + ] + }, + "source": """ + { + "bool": { + "should": [ + { + "bool": { + "must_not": { + "exists": { + "field": "_allow_access_control" + } + } + } + }, + { + "terms": { + "_allow_access_control.enum": {{#toJson}}access_control{{/toJson}} + } + } + ] + } + } + """ + } + } + } + ], + "restriction": { + "workflows": [ + "search_application_query" + ] + } + } + } +} +---- +// TEST[skip:TODO] + +The response will look like this: + +[source,js] +---- +{ + "id": "0rCD3i-MjKsw4g9BpRIBa", + "name": "john-api-key", + "expiration": 1687881715555, + "api_key": "zTxre9L6TcmRIgd2NgLCRg", + "encoded": "Qk05dy1JZ0JhRDNyNGpLQ3MwUmk6elRzdGU5QjZUY21SSWdkMldnQ1RMZw==" +} +---- +// NOTCONSOLE + +The `api_key` field contains the API key that can be used to query the Search Application with the appropriate DLS restrictions. + +[discrete#es-dls-e2e-guide-elasticsearch-querying-multiple-indices] +===== Querying multiple indices + +This section describes how to generate an API key to query a search application that contains multiple indices with documents ingested by connectors with DLS. + +A user might have multiple identities that define which documents they are allowed to read. +In this case we want to create a single Elasticsearch API key that can be used to query only the documents this user has access to. + +Let's assume we want to create an API key that combines the following user identities: + +[source,js] +---- +GET .search-acl-filter-source1 +{ + "_id": "example.user@example.com", + "identity": { + "username": "example username", + "email": "example.user@example.com" + }, + "query": { + "template": { + "params": { + "access_control": [ + "example.user@example.com", + "source1-user-group"] + } + }, + "source": "..." + } +} +---- +// NOTCONSOLE + +[source,js] +---- +GET .search-acl-filter-source2 +{ + "_id": "example.user@example.com", + "identity": { + "username": "example username", + "email": "example.user@example.com" + }, + "query": { + "template": { + "params": { + "access_control": [ + "example.user@example.com", + "source2-user-group"] + } + }, + "source": "..." + } +} +---- +// NOTCONSOLE + +`.search-acl-filter-source1` and `.search-acl-filter-source2` define the access control identities for `source1` and `source2`. + +The following script exemplifies how to generate the Elasticsearch API key that combines multiple user identities: + +[source,js] +---- +require("dotenv").config(); +const axios = require("axios"); + +// Elasticsearch URL and creds retrieved from environment variables +const ELASTICSEARCH_URL = process.env.ELASTICSEARCH_URL; +const ELASTICSEARCH_USER = process.env.ELASTICSEARCH_USER; +const ELASTICSEARCH_PASSWORD = process.env.ELASTICSEARCH_PASSWORD; + +const config = { + auth: { + username: ELASTICSEARCH_USER, + password: ELASTICSEARCH_PASSWORD, + }, + headers: { + "Content-Type": "application/json", + }, +}; + +async function createApiKey({ + searchApplication, + userId, + indices = "", + metadata, + expiration = "1d" +}) { + try { + const indices = indices.split(","); + + let combinedQuery = { bool: { should: [] } }; + + for (const index of indices) { + const aclsIndex = `.search-acl-filter-${index}`; + const response = await axios.get( + `${ELASTICSEARCH_URL}/${aclsIndex}/_doc/${userId}`, + config + ); + combinedQuery.bool.should.push({ + bool: { + must: [ + { + term: { + "_index": index, + }, + }, + response.data._source.query.source, + ], + }, + }); + } + + if (!metadata || Object.keys(metadata).length === 0) { + metadata = { created_by: "create-api-key" }; + } + + const apiKeyBody = { + name: userId, + expiration, + role_descriptors: { + [`${searchApplication}-role`]: { + index: [ + { + names: [searchApplication], + privileges: ["read"], + query: combinedQuery, + }, + ], + restriction: { + workflows: ["search_application_query"], + }, + }, + }, + metadata, + }; + + const apiKeyResponse = await axios.post( + `${ELASTICSEARCH_URL}/_security/api_key`, + apiKeyBody, + config + ); + + console.log(apiKeyResponse.data); + return apiKeyResponse.data.encoded; + } catch (error) { + console.log(error) + } +} + +// example usage: +createApiKey({ + searchApplication: "my-search-app", + userId: "example.user@example.com", + indices: "source1,source2", + expiration: "1d", + metadata: { + application: "my-search-app", + namespace: "dev", + foo: "bar", + }, +}).then((encodedKey) => console.log(encodedKey)); + +---- +// NOTCONSOLE + +NOTE: The example combines multiple identities into a single role descriptor. This is because an Elasticsearch API key can use role restrictions only if it has a *single role descriptor*. + +[discrete#es-dls-e2e-guide-elasticsearch-api-keys-frontend-implementation] +==== Implementation in your frontend application + +If you're building a frontend application, use the `encoded` field to pass the API key to the frontend. +Your app can then use the API key to query the search application. +The workflow will look something like this: + +1. User signs in to your application. +2. Your application generates an Elasticsearch API key using the {ref}/security-api-create-api-key.html[Create API Key] API. +3. The `encoded` field is returned to the frontend application. +4. When the user searches for documents, the frontend application passes the `encoded` field to your search application's {ref}/search-application-search.html[`_search` endpoint]. +For example, you might use the https://github.com/elastic/search-application-client[Search Application client^] to make the actual queries using the API key: ++ +[source,js] +---- +const client = SearchApplicationClient(applicationName, endpoint, apiKey, params); +---- +// NOTCONSOLE + +Here's what this workflow looks like in a sequence diagram: + +[.screenshot] +image::images/dls-api-key-workflow.png[DLS API key and search application client workflow] + +[TIP] +==== +When creating an Elasticsearch API key for query Search Applications, you must include the `search_application_query` restriction. This will ensure the API key can only access the Search Application Search API. +==== + +[TIP] +==== +We recommend always setting an `expiration` time when creating an Elasticsearch API key. When `expiration` is not set, the Elasticsearch API will never expire. +==== + +[discrete#es-dls-e2e-guide-workflow-guidance] +==== Workflow guidance + +We recommend relying on the connector access control sync to automate and keep documents in sync with changes to the original content source's user permissions. + +In this workflow you will need to handle the generation of the Elasticsearch API key in the backend of your application, in response to browser sign ins. + +Once the key is generated, the backend will also need to return that key to the client (browser) to be used in subsequent search requests to your search application. + +The API key can be invalidated using the {ref}/security-api-invalidate-api-key.html[Invalidate API Key API]. +Additionally, if the user's permission changes, you'll need to update or recreate the Elasticsearch API key. + +[discrete#es-dls-e2e-guide-next-steps] +==== Next steps + +Learn how to use the Search Application client to query your Search Application. +See {ref}/search-application-client.html[Search Applications client]. + +[discrete#es-dls-e2e-guide-learn-more] +==== Learn more + +* <> +* <> +* {ref}/search-application-overview.html[Search Applications] diff --git a/docs/reference/connector/docs/dls-overview.asciidoc b/docs/reference/connector/docs/dls-overview.asciidoc new file mode 100644 index 0000000000000..ec6bb43d955c7 --- /dev/null +++ b/docs/reference/connector/docs/dls-overview.asciidoc @@ -0,0 +1,345 @@ +[#es-dls-overview] +=== How DLS works + +Document level security (DLS) enables you to control access to content at the document level. +Access to each document in an index can be managed independently, based on the identities (such as usernames, emails, groups etc.) that are allowed to view it. + +This feature works with the help of special access control documents that are indexed by a connector into a hidden Elasticsearch index, associated with the standard content index. +If your content documents have access control fields that match the criteria defined in your access control documents, Elasticsearch will apply DLS to the documents synced by the connector. + +[discrete#es-dls-overview-core-concepts] +==== Core concepts + +At a very high level, there are two essential components that enable document level security with connectors: + +* *Access control documents*: These documents define the access control policy for documents from your third party source. +They live in a hidden index named with the following pattern: `.search-acl-filter-`. +See <> for more details and an example. +* *Content documents with access control fields*: The documents that contain the synced content from your third party source must have *access control fields* that match the criteria defined in your access control documents. +These documents live in an index named with the following pattern: `search-`. +** If a content document does not have access control fields, there will be no restrictions on who can view it. +** If the access control field is present but _empty_, no identities will have access and the document will be effectively invisible. ++ +See <> for more details. + +[discrete#es-dls-overview-procedure] +==== Enabling DLS + +To enable DLS, you need to perform the following steps: + +. First *enable DLS* for your connector as part of the connector configuration. +. Run an *Access control* sync. +. This creates a hidden access control index prefixed with `.search-acl-filter-`. For example, if you named your connector index `search-sharepoint`, the access control index would be named `.search-acl-filter-search-sharepoint`. +. The <> on the hidden index define which identities are allowed to view documents with access control fields. +. The access control document uses a search template to define how to filter search results based on identities. +. Schedule recurring *Access control* syncs to update the access control documents in the hidden index. + +Note the following details about content documents and syncs: + +. Remember that for DLS to work, your *content documents* must have access control fields that match the criteria defined in your access control documents. +<> contain the actual content your users will search for. +If a content document does not have access control fields, there will be no restrictions on who can view it. +. When a user searches for content, the access control documents determine which content the user is allowed to view. +. At _search_ time documents without the `_allow_access_control` field or with allowed values in `_allow_access_control.enum` will be returned in the search results. The logic for determining whether a document has access control enabled is based on the presence or values of the `_allow_access_control*` fields. +. Run *Content* syncs to sync your third party data source to Elasticsearch. +A specific field (or fields) within these documents correlates with the query parameters in the access control documents enabling document-level security (DLS). + +[NOTE] +==== +You must enable DLS for your connector _before_ running the first content sync. +If you have already run a content sync, you'll need to delete all documents on the index, enable DLS, and run a new content sync. +==== + +[discrete#es-dls-overview-index] +==== DLS at index time + +[discrete#es-dls-overview-access-control-documents] +===== Access control documents + +These documents define the access control policy for the data indexed into Elasticsearch. +An example of an access control document is as follows: + +[source,js] +---- +{ + "_id": "example.user@example.com", + "identity": { + "username": "example username", + "email": "example.user@example.com" + }, + "query": { + "template": { + "params": { + "access_control": [ + "example.user@example.com", + "example group", + "example username"] + } + }, + "source": "..." + } +} +---- +// NOTCONSOLE + +In this example, the identity object specifies the identity of the user that this document pertains to. +The `query` object then uses a template to list the parameters that form the access control policy for this identity. +It also contains the query `source`, which will specify a query to fetch all content documents the identity has access to. +The `_id` could be, for example, the email address or the username of a user. +The exact content and structure of `identity` depends on the corresponding implementation. + +[discrete#es-dls-overview-content-documents] +===== Content documents + +Content documents contain the actual data from your 3rd party source. +A specific field (or fields) within these documents correlates with the query parameters in the access control documents enabling document-level security (DLS). +Please note, the field names used to implement DLS may vary across different connectors. +In the following example we'll use the field `_allow_access_control` for specifying the access control for a user identity. + +[source,js] +---- +{ + "_id": "some-unique-id", + "key-1": "value-1", + "key-2": "value-2", + "key-3": "value-3", + "_allow_access_control": [ + "example.user@example.com", + "example group", + "example username" + ] +} +---- +// NOTCONSOLE + +[discrete#es-dls-overview-sync-type-comparison] +===== Access control sync vs content sync + +The ingestion of documents into an Elasticsearch index is known as a sync. +DLS is managed using two types of syncs: + +* *Content sync*: Ingests content into an index that starts with `search-`. + +* *Access control sync*: Separate, additional sync which ingests access control documents into index that starts with `.search-acl-filter-`. + +During a sync, the connector ingests the documents into the relevant index based on their type (content or access control). +The access control documents determine the access control policy for the content documents. + +By leveraging DLS, you can ensure that your Elasticsearch data is securely accessible to the right users or groups, based on the permissions defined in the access control documents. + +[discrete#es-dls-overview-search-time] +==== DLS at search time + +[discrete#es-dls-overview-search-time-identity-allowed] +===== When is an identity allowed to see a content document + +A user can view a document if at least one access control element in their access control document matches an item within the document's `_allow_access_control` field. + +[discrete#es-dls-overview-search-time-example] +====== Example +This section illustrates when a user has access to certain documents depending on the access control. + +One access control document: +[source,js] +---- +{ + "_id": "example.user@example.com", + "identity": { + "username": "example username", + "email": "example.user@example.com" + }, + "query": { + "template": { + "params": { + "access_control": [ + "example.user@example.com", + "example group", + "example username"] + } + }, + "source": "..." + } +} +---- +// NOTCONSOLE + +Let's see which of the following example documents these permissions can access, and why. +[source,js] +---- +{ + "_id": "some-unique-id-1", + "_allow_access_control": [ + "example.user@example.com", + "example group", + "example username" + ] +} +---- +// NOTCONSOLE + +The user `example username` will have access to this document as he's part of the corresponding group and his username and email address are also explicitly part of `_allow_access_control`. + +[source,js] +---- +{ + "_id": "some-unique-id-2", + "_allow_access_control": [ + "example group" + ] +} +---- +// NOTCONSOLE + +The user `example username` will also have access to this document as they are part of the `example group`. + +[source,js] +---- +{ + "_id": "some-unique-id-3", + "_allow_access_control": [ + "another.user@example.com" + ] +} +---- +// NOTCONSOLE + +The user `example username` won't have access to this document because their email does not match `another.user@example.com`. + +[source,js] +---- +{ + "_id": "some-unique-id-4", + "_allow_access_control": [] +} +---- +// NOTCONSOLE + +No one will have access to this document as the `_allow_access_control` field is empty. + +[discrete#es-dls-overview-multiple-connectors] +===== Querying multiple indices + +This section illustrates how to define an Elasticsearch API key that has restricted read access to multiple indices that have DLS enabled. + +A user might have multiple identities that define which documents they are allowed to read. +We can define an Elasticsearch API key with a role descriptor for each index the user has access to. + +[discrete#es-dls-overview-multiple-connectors-example] +====== Example + +Let's assume we want to create an API key that combines the following user identities: + +[source,js] +---- +GET .search-acl-filter-source1 +{ + "_id": "example.user@example.com", + "identity": { + "username": "example username", + "email": "example.user@example.com" + }, + "query": { + "template": { + "params": { + "access_control": [ + "example.user@example.com", + "source1-user-group"] + } + }, + "source": "..." + } +} +---- +// NOTCONSOLE + +[source,js] +---- +GET .search-acl-filter-source2 +{ + "_id": "example.user@example.com", + "identity": { + "username": "example username", + "email": "example.user@example.com" + }, + "query": { + "template": { + "params": { + "access_control": [ + "example.user@example.com", + "source2-user-group"] + } + }, + "source": "..." + } +} +---- +// NOTCONSOLE + +`.search-acl-filter-source1` and `.search-acl-filter-source2` define the access control identities for `source1` and `source2`. + +You can create an Elasticsearch API key using an API call like this: + +[source,console] +---- +POST /_security/api_key +{ + "name": "my-api-key", + "role_descriptors": { + "role-source1": { + "indices": [ + { + "names": ["source1"], + "privileges": ["read"], + "query": { + "template": { + "params": { + "access_control": [ + "example.user@example.com", + "source1-user-group"] + } + }, + "source": "..." + } + } + ] + }, + "role-source2": { + "indices": [ + { + "names": ["source2"], + "privileges": ["read"], + "query": { + "template": { + "params": { + "access_control": [ + "example.user@example.com", + "source2-user-group"] + } + }, + "source": "..." + } + } + ] + } + } +} + +---- +// TEST[skip:TODO] + +[discrete#es-dls-overview-multiple-connectors-workflow-guidance] +====== Workflow guidance + +We recommend relying on the connector access control sync to automate and keep documents in sync with changes to the original content source's user permissions. + +Consider setting an `expiration` time when creating an Elasticsearch API key. When `expiration` is not set, the Elasticsearch API will never expire. + +The API key can be invalidated using the {ref}/security-api-invalidate-api-key.html[Invalidate API Key API]. +Additionally, if the user's permission changes, you'll need to update or recreate the Elasticsearch API key. + +[discrete#es-dls-overview-search-time-learn-more] +===== Learn more + +* <> +* {ref}/document-level-security.html[Elasticsearch Document Level Security^] + diff --git a/docs/reference/connector/docs/dls.asciidoc b/docs/reference/connector/docs/dls.asciidoc new file mode 100644 index 0000000000000..2e8871ea9eb87 --- /dev/null +++ b/docs/reference/connector/docs/dls.asciidoc @@ -0,0 +1,39 @@ +[#es-dls] +== Document level security + +Document level security (DLS) enables you to restrict access to documents in your Elasticsearch indices according to user and group permissions. +This ensures search results only return authorized information for users, based on their permissions. + +[discrete#es-dls-availability-prerequisites] +=== Availability & prerequisites + +Support for DLS in Elastic connectors was introduced in version *8.9.0*. + +[NOTE] +==== +This feature is in *beta* and is subject to change. +The design and code is less mature than official GA features and is being provided as-is with no warranties. +Beta features are not subject to the support SLA of official GA features. +==== + +This feature is not available for all Elastic subscription levels. +Refer to the subscriptions pages for https://www.elastic.co/subscriptions/cloud[Elastic Cloud^] and https://www.elastic.co/subscriptions[Elastic Stack^]. + +DLS is available by default when using the following Elastic connectors: + +include::_connectors-list-dls.asciidoc[] + +Note that our standalone products (App Search and Workplace Search) do not use this feature. +Workplace Search has its own permissions management system. + +[discrete#es-dls-learn-more] +=== Learn more + +DLS documentation: + +* <> +* <> +* <> + +include::dls-overview.asciidoc[] +include::dls-e2e-guide.asciidoc[] diff --git a/docs/reference/connector/docs/images/analytics-collections-dashboard.png b/docs/reference/connector/docs/images/analytics-collections-dashboard.png new file mode 100644 index 0000000000000..b99fc07bc5fab Binary files /dev/null and b/docs/reference/connector/docs/images/analytics-collections-dashboard.png differ diff --git a/docs/reference/connector/docs/images/analytics-explorer-dashboard.png b/docs/reference/connector/docs/images/analytics-explorer-dashboard.png new file mode 100644 index 0000000000000..922763585d67f Binary files /dev/null and b/docs/reference/connector/docs/images/analytics-explorer-dashboard.png differ diff --git a/docs/reference/connector/docs/images/analytics-overview-dashboard.png b/docs/reference/connector/docs/images/analytics-overview-dashboard.png new file mode 100644 index 0000000000000..c088cd3994d1e Binary files /dev/null and b/docs/reference/connector/docs/images/analytics-overview-dashboard.png differ diff --git a/docs/reference/connector/docs/images/app-search-audit-log-table.png b/docs/reference/connector/docs/images/app-search-audit-log-table.png new file mode 100644 index 0000000000000..ccf9147bdb6e8 Binary files /dev/null and b/docs/reference/connector/docs/images/app-search-audit-log-table.png differ diff --git a/docs/reference/connector/docs/images/app-search-kibana-ui.png b/docs/reference/connector/docs/images/app-search-kibana-ui.png new file mode 100644 index 0000000000000..0e6b09b7f1bba Binary files /dev/null and b/docs/reference/connector/docs/images/app-search-kibana-ui.png differ diff --git a/docs/reference/connector/docs/images/app-search-settings.png b/docs/reference/connector/docs/images/app-search-settings.png new file mode 100644 index 0000000000000..9c8c31c81a6c5 Binary files /dev/null and b/docs/reference/connector/docs/images/app-search-settings.png differ diff --git a/docs/reference/connector/docs/images/app-search-standalone-ui.png b/docs/reference/connector/docs/images/app-search-standalone-ui.png new file mode 100644 index 0000000000000..f496d831b70ad Binary files /dev/null and b/docs/reference/connector/docs/images/app-search-standalone-ui.png differ diff --git a/docs/reference/connector/docs/images/basic-rule-example.png b/docs/reference/connector/docs/images/basic-rule-example.png new file mode 100644 index 0000000000000..aa1d79bb6f274 Binary files /dev/null and b/docs/reference/connector/docs/images/basic-rule-example.png differ diff --git a/docs/reference/connector/docs/images/blog-elastic-crawler-1.png b/docs/reference/connector/docs/images/blog-elastic-crawler-1.png new file mode 100644 index 0000000000000..e2e5593c3e102 Binary files /dev/null and b/docs/reference/connector/docs/images/blog-elastic-crawler-1.png differ diff --git a/docs/reference/connector/docs/images/blog-elastic-crawler-2.jpg b/docs/reference/connector/docs/images/blog-elastic-crawler-2.jpg new file mode 100644 index 0000000000000..3bc45743afbd0 Binary files /dev/null and b/docs/reference/connector/docs/images/blog-elastic-crawler-2.jpg differ diff --git a/docs/reference/connector/docs/images/blog-elastic-crawler-3.jpg b/docs/reference/connector/docs/images/blog-elastic-crawler-3.jpg new file mode 100644 index 0000000000000..6f7f4fe5c4b6d Binary files /dev/null and b/docs/reference/connector/docs/images/blog-elastic-crawler-3.jpg differ diff --git a/docs/reference/connector/docs/images/build-a-connector-workflow.png b/docs/reference/connector/docs/images/build-a-connector-workflow.png new file mode 100644 index 0000000000000..eb51863358e9a Binary files /dev/null and b/docs/reference/connector/docs/images/build-a-connector-workflow.png differ diff --git a/docs/reference/connector/docs/images/combine-engines-indices-content-sources.png b/docs/reference/connector/docs/images/combine-engines-indices-content-sources.png new file mode 100644 index 0000000000000..072f4cefff01b Binary files /dev/null and b/docs/reference/connector/docs/images/combine-engines-indices-content-sources.png differ diff --git a/docs/reference/connector/docs/images/connectors-overview.svg b/docs/reference/connector/docs/images/connectors-overview.svg new file mode 100644 index 0000000000000..0a7fb30c61d6d --- /dev/null +++ b/docs/reference/connector/docs/images/connectors-overview.svg @@ -0,0 +1,70 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/reference/connector/docs/images/convert-connector.png b/docs/reference/connector/docs/images/convert-connector.png new file mode 100644 index 0000000000000..f07886d12d7fb Binary files /dev/null and b/docs/reference/connector/docs/images/convert-connector.png differ diff --git a/docs/reference/connector/docs/images/crawler-crawl-rules.png b/docs/reference/connector/docs/images/crawler-crawl-rules.png new file mode 100644 index 0000000000000..69c97418189d3 Binary files /dev/null and b/docs/reference/connector/docs/images/crawler-crawl-rules.png differ diff --git a/docs/reference/connector/docs/images/crawler-events-logs-viewer-by-url.png b/docs/reference/connector/docs/images/crawler-events-logs-viewer-by-url.png new file mode 100644 index 0000000000000..2f05747d49398 Binary files /dev/null and b/docs/reference/connector/docs/images/crawler-events-logs-viewer-by-url.png differ diff --git a/docs/reference/connector/docs/images/crawler-events-logs-viewer.png b/docs/reference/connector/docs/images/crawler-events-logs-viewer.png new file mode 100644 index 0000000000000..758b94e808661 Binary files /dev/null and b/docs/reference/connector/docs/images/crawler-events-logs-viewer.png differ diff --git a/docs/reference/connector/docs/images/crawler-extraction-rules-content-field.png b/docs/reference/connector/docs/images/crawler-extraction-rules-content-field.png new file mode 100644 index 0000000000000..2b7b9f3d41cd9 Binary files /dev/null and b/docs/reference/connector/docs/images/crawler-extraction-rules-content-field.png differ diff --git a/docs/reference/connector/docs/images/crawler-extraction-rules-url-filters.png b/docs/reference/connector/docs/images/crawler-extraction-rules-url-filters.png new file mode 100644 index 0000000000000..11be61bcce8fa Binary files /dev/null and b/docs/reference/connector/docs/images/crawler-extraction-rules-url-filters.png differ diff --git a/docs/reference/connector/docs/images/crawler-extraction-rules.png b/docs/reference/connector/docs/images/crawler-extraction-rules.png new file mode 100644 index 0000000000000..175f18e2eaf66 Binary files /dev/null and b/docs/reference/connector/docs/images/crawler-extraction-rules.png differ diff --git a/docs/reference/connector/docs/images/crawler-iteration-cycle.png b/docs/reference/connector/docs/images/crawler-iteration-cycle.png new file mode 100644 index 0000000000000..f013bd2ed0dcd Binary files /dev/null and b/docs/reference/connector/docs/images/crawler-iteration-cycle.png differ diff --git a/docs/reference/connector/docs/images/crawler-proxy-schematic.png b/docs/reference/connector/docs/images/crawler-proxy-schematic.png new file mode 100644 index 0000000000000..524182d2f6643 Binary files /dev/null and b/docs/reference/connector/docs/images/crawler-proxy-schematic.png differ diff --git a/docs/reference/connector/docs/images/crawler-proxy-validation.png b/docs/reference/connector/docs/images/crawler-proxy-validation.png new file mode 100644 index 0000000000000..61f268f83f209 Binary files /dev/null and b/docs/reference/connector/docs/images/crawler-proxy-validation.png differ diff --git a/docs/reference/connector/docs/images/crawler-scheduling-multiple-crawls.png b/docs/reference/connector/docs/images/crawler-scheduling-multiple-crawls.png new file mode 100644 index 0000000000000..fcddae8dd1d04 Binary files /dev/null and b/docs/reference/connector/docs/images/crawler-scheduling-multiple-crawls.png differ diff --git a/docs/reference/connector/docs/images/crawler-scheduling.png b/docs/reference/connector/docs/images/crawler-scheduling.png new file mode 100644 index 0000000000000..f67a0d6b5fb5d Binary files /dev/null and b/docs/reference/connector/docs/images/crawler-scheduling.png differ diff --git a/docs/reference/connector/docs/images/discover-data-view-analytics.png b/docs/reference/connector/docs/images/discover-data-view-analytics.png new file mode 100644 index 0000000000000..676ed40098e99 Binary files /dev/null and b/docs/reference/connector/docs/images/discover-data-view-analytics.png differ diff --git a/docs/reference/connector/docs/images/discover-lens-analytics.png b/docs/reference/connector/docs/images/discover-lens-analytics.png new file mode 100644 index 0000000000000..89701eca60bad Binary files /dev/null and b/docs/reference/connector/docs/images/discover-lens-analytics.png differ diff --git a/docs/reference/connector/docs/images/dls-api-key-workflow.png b/docs/reference/connector/docs/images/dls-api-key-workflow.png new file mode 100644 index 0000000000000..a3bae143aa57e Binary files /dev/null and b/docs/reference/connector/docs/images/dls-api-key-workflow.png differ diff --git a/docs/reference/connector/docs/images/document-enrichment-add-inference-pipeline.png b/docs/reference/connector/docs/images/document-enrichment-add-inference-pipeline.png new file mode 100644 index 0000000000000..ddcf42e24ab83 Binary files /dev/null and b/docs/reference/connector/docs/images/document-enrichment-add-inference-pipeline.png differ diff --git a/docs/reference/connector/docs/images/document-enrichment-diagram.png b/docs/reference/connector/docs/images/document-enrichment-diagram.png new file mode 100644 index 0000000000000..89ae1d45e24d4 Binary files /dev/null and b/docs/reference/connector/docs/images/document-enrichment-diagram.png differ diff --git a/docs/reference/connector/docs/images/elser-deploy-model.png b/docs/reference/connector/docs/images/elser-deploy-model.png new file mode 100644 index 0000000000000..46f5e8cc7229a Binary files /dev/null and b/docs/reference/connector/docs/images/elser-deploy-model.png differ diff --git a/docs/reference/connector/docs/images/elser-model-deployment.png b/docs/reference/connector/docs/images/elser-model-deployment.png new file mode 100644 index 0000000000000..1bcae4c85a5e1 Binary files /dev/null and b/docs/reference/connector/docs/images/elser-model-deployment.png differ diff --git a/docs/reference/connector/docs/images/elser-model-started.png b/docs/reference/connector/docs/images/elser-model-started.png new file mode 100644 index 0000000000000..c533f7b5123fb Binary files /dev/null and b/docs/reference/connector/docs/images/elser-model-started.png differ diff --git a/docs/reference/connector/docs/images/elser-pipeline-model-selection.png b/docs/reference/connector/docs/images/elser-pipeline-model-selection.png new file mode 100644 index 0000000000000..986071e77b36a Binary files /dev/null and b/docs/reference/connector/docs/images/elser-pipeline-model-selection.png differ diff --git a/docs/reference/connector/docs/images/elser-start-model.png b/docs/reference/connector/docs/images/elser-start-model.png new file mode 100644 index 0000000000000..81cdfa0eb58a0 Binary files /dev/null and b/docs/reference/connector/docs/images/elser-start-model.png differ diff --git a/docs/reference/connector/docs/images/enable-rbac-app-search.png b/docs/reference/connector/docs/images/enable-rbac-app-search.png new file mode 100644 index 0000000000000..11ef21d55f07f Binary files /dev/null and b/docs/reference/connector/docs/images/enable-rbac-app-search.png differ diff --git a/docs/reference/connector/docs/images/enable-rbac-workplace-search.png b/docs/reference/connector/docs/images/enable-rbac-workplace-search.png new file mode 100644 index 0000000000000..45205d23cddfd Binary files /dev/null and b/docs/reference/connector/docs/images/enable-rbac-workplace-search.png differ diff --git a/docs/reference/connector/docs/images/filtering-general-diagram.png b/docs/reference/connector/docs/images/filtering-general-diagram.png new file mode 100644 index 0000000000000..ce1a9367d7b57 Binary files /dev/null and b/docs/reference/connector/docs/images/filtering-general-diagram.png differ diff --git a/docs/reference/connector/docs/images/filtering-rules-zero-state.png b/docs/reference/connector/docs/images/filtering-rules-zero-state.png new file mode 100644 index 0000000000000..fa43c4cc5a71c Binary files /dev/null and b/docs/reference/connector/docs/images/filtering-rules-zero-state.png differ diff --git a/docs/reference/connector/docs/images/hybrid-architecture.png b/docs/reference/connector/docs/images/hybrid-architecture.png new file mode 100644 index 0000000000000..81d19179db3e2 Binary files /dev/null and b/docs/reference/connector/docs/images/hybrid-architecture.png differ diff --git a/docs/reference/connector/docs/images/kibana-setup-guide.png b/docs/reference/connector/docs/images/kibana-setup-guide.png new file mode 100644 index 0000000000000..2797472933102 Binary files /dev/null and b/docs/reference/connector/docs/images/kibana-setup-guide.png differ diff --git a/docs/reference/connector/docs/images/kibana-ui.png b/docs/reference/connector/docs/images/kibana-ui.png new file mode 100644 index 0000000000000..4371f3a1052aa Binary files /dev/null and b/docs/reference/connector/docs/images/kibana-ui.png differ diff --git a/docs/reference/connector/docs/images/ldap-login.png b/docs/reference/connector/docs/images/ldap-login.png new file mode 100644 index 0000000000000..b7dd2b9fce5fb Binary files /dev/null and b/docs/reference/connector/docs/images/ldap-login.png differ diff --git a/docs/reference/connector/docs/images/mongodb-connector-config.png b/docs/reference/connector/docs/images/mongodb-connector-config.png new file mode 100644 index 0000000000000..2c4d2e2158908 Binary files /dev/null and b/docs/reference/connector/docs/images/mongodb-connector-config.png differ diff --git a/docs/reference/connector/docs/images/mongodb-load-sample-data.png b/docs/reference/connector/docs/images/mongodb-load-sample-data.png new file mode 100644 index 0000000000000..f7bc9c4192b02 Binary files /dev/null and b/docs/reference/connector/docs/images/mongodb-load-sample-data.png differ diff --git a/docs/reference/connector/docs/images/mongodb-sample-document.png b/docs/reference/connector/docs/images/mongodb-sample-document.png new file mode 100644 index 0000000000000..f462c41ad751c Binary files /dev/null and b/docs/reference/connector/docs/images/mongodb-sample-document.png differ diff --git a/docs/reference/connector/docs/images/oidc-login.png b/docs/reference/connector/docs/images/oidc-login.png new file mode 100644 index 0000000000000..37753acc8a0f6 Binary files /dev/null and b/docs/reference/connector/docs/images/oidc-login.png differ diff --git a/docs/reference/connector/docs/images/pipeline-copy-customize.png b/docs/reference/connector/docs/images/pipeline-copy-customize.png new file mode 100644 index 0000000000000..1f2bf99aa4f16 Binary files /dev/null and b/docs/reference/connector/docs/images/pipeline-copy-customize.png differ diff --git a/docs/reference/connector/docs/images/pipelines-extraction-sync-rules.png b/docs/reference/connector/docs/images/pipelines-extraction-sync-rules.png new file mode 100644 index 0000000000000..1a491a6bf95bb Binary files /dev/null and b/docs/reference/connector/docs/images/pipelines-extraction-sync-rules.png differ diff --git a/docs/reference/connector/docs/images/pki-login-screen.png b/docs/reference/connector/docs/images/pki-login-screen.png new file mode 100644 index 0000000000000..9fec19564adb3 Binary files /dev/null and b/docs/reference/connector/docs/images/pki-login-screen.png differ diff --git a/docs/reference/connector/docs/images/saml-login.png b/docs/reference/connector/docs/images/saml-login.png new file mode 100644 index 0000000000000..f8d5771363efc Binary files /dev/null and b/docs/reference/connector/docs/images/saml-login.png differ diff --git a/docs/reference/connector/docs/images/search-applications-create.png b/docs/reference/connector/docs/images/search-applications-create.png new file mode 100644 index 0000000000000..cce31b985ad82 Binary files /dev/null and b/docs/reference/connector/docs/images/search-applications-create.png differ diff --git a/docs/reference/connector/docs/images/search-applications-docs-explorer.png b/docs/reference/connector/docs/images/search-applications-docs-explorer.png new file mode 100644 index 0000000000000..d9b2cfa05f986 Binary files /dev/null and b/docs/reference/connector/docs/images/search-applications-docs-explorer.png differ diff --git a/docs/reference/connector/docs/images/search-applications-unified-search.png b/docs/reference/connector/docs/images/search-applications-unified-search.png new file mode 100644 index 0000000000000..2eca235b2d968 Binary files /dev/null and b/docs/reference/connector/docs/images/search-applications-unified-search.png differ diff --git a/docs/reference/connector/docs/images/select-ingestion-method.png b/docs/reference/connector/docs/images/select-ingestion-method.png new file mode 100644 index 0000000000000..29dc3630e1237 Binary files /dev/null and b/docs/reference/connector/docs/images/select-ingestion-method.png differ diff --git a/docs/reference/connector/docs/images/self-managed-architecture.png b/docs/reference/connector/docs/images/self-managed-architecture.png new file mode 100644 index 0000000000000..c06248acf7d81 Binary files /dev/null and b/docs/reference/connector/docs/images/self-managed-architecture.png differ diff --git a/docs/reference/connector/docs/images/simple-rule-equals.png b/docs/reference/connector/docs/images/simple-rule-equals.png new file mode 100644 index 0000000000000..5dd5e43427ea5 Binary files /dev/null and b/docs/reference/connector/docs/images/simple-rule-equals.png differ diff --git a/docs/reference/connector/docs/images/simple-rule-greater.png b/docs/reference/connector/docs/images/simple-rule-greater.png new file mode 100644 index 0000000000000..5dd78897c6154 Binary files /dev/null and b/docs/reference/connector/docs/images/simple-rule-greater.png differ diff --git a/docs/reference/connector/docs/images/simple-rule-regex.png b/docs/reference/connector/docs/images/simple-rule-regex.png new file mode 100644 index 0000000000000..adb8be4965464 Binary files /dev/null and b/docs/reference/connector/docs/images/simple-rule-regex.png differ diff --git a/docs/reference/connector/docs/images/sync-rules-advanced-rules-appeared.png b/docs/reference/connector/docs/images/sync-rules-advanced-rules-appeared.png new file mode 100644 index 0000000000000..e3a49b40212d8 Binary files /dev/null and b/docs/reference/connector/docs/images/sync-rules-advanced-rules-appeared.png differ diff --git a/docs/reference/connector/docs/images/sync-rules-applied-rules-during-sync.png b/docs/reference/connector/docs/images/sync-rules-applied-rules-during-sync.png new file mode 100644 index 0000000000000..7cb713277a77b Binary files /dev/null and b/docs/reference/connector/docs/images/sync-rules-applied-rules-during-sync.png differ diff --git a/docs/reference/connector/docs/images/sync-rules-detail-view-button.png b/docs/reference/connector/docs/images/sync-rules-detail-view-button.png new file mode 100644 index 0000000000000..f24daea0d4351 Binary files /dev/null and b/docs/reference/connector/docs/images/sync-rules-detail-view-button.png differ diff --git a/docs/reference/connector/docs/images/sync-rules-draft-new-rules.png b/docs/reference/connector/docs/images/sync-rules-draft-new-rules.png new file mode 100644 index 0000000000000..c7386659b2b09 Binary files /dev/null and b/docs/reference/connector/docs/images/sync-rules-draft-new-rules.png differ diff --git a/docs/reference/connector/docs/images/sync-rules-extract-all-at-once.png b/docs/reference/connector/docs/images/sync-rules-extract-all-at-once.png new file mode 100644 index 0000000000000..4b2235bf07967 Binary files /dev/null and b/docs/reference/connector/docs/images/sync-rules-extract-all-at-once.png differ diff --git a/docs/reference/connector/docs/images/sync-rules-new-rule-applied.png b/docs/reference/connector/docs/images/sync-rules-new-rule-applied.png new file mode 100644 index 0000000000000..936b7b98b7cda Binary files /dev/null and b/docs/reference/connector/docs/images/sync-rules-new-rule-applied.png differ diff --git a/docs/reference/connector/docs/images/sync-rules-pagination.png b/docs/reference/connector/docs/images/sync-rules-pagination.png new file mode 100644 index 0000000000000..3cb52f134081c Binary files /dev/null and b/docs/reference/connector/docs/images/sync-rules-pagination.png differ diff --git a/docs/reference/connector/docs/images/sync-rules-paste-aggregation-pipeline.png b/docs/reference/connector/docs/images/sync-rules-paste-aggregation-pipeline.png new file mode 100644 index 0000000000000..697759beeb2b9 Binary files /dev/null and b/docs/reference/connector/docs/images/sync-rules-paste-aggregation-pipeline.png differ diff --git a/docs/reference/connector/docs/images/sync-rules-rules-fulfilling-properties.png b/docs/reference/connector/docs/images/sync-rules-rules-fulfilling-properties.png new file mode 100644 index 0000000000000..3245d4ca7a2ae Binary files /dev/null and b/docs/reference/connector/docs/images/sync-rules-rules-fulfilling-properties.png differ diff --git a/docs/reference/connector/docs/images/sync-rules-save-and-validate-draft.png b/docs/reference/connector/docs/images/sync-rules-save-and-validate-draft.png new file mode 100644 index 0000000000000..15c0808159a18 Binary files /dev/null and b/docs/reference/connector/docs/images/sync-rules-save-and-validate-draft.png differ diff --git a/docs/reference/connector/docs/images/sync-rules-time-dimension.png b/docs/reference/connector/docs/images/sync-rules-time-dimension.png new file mode 100644 index 0000000000000..003dd4bef1f0b Binary files /dev/null and b/docs/reference/connector/docs/images/sync-rules-time-dimension.png differ diff --git a/docs/reference/connector/docs/images/use-a-connector-workflow.png b/docs/reference/connector/docs/images/use-a-connector-workflow.png new file mode 100644 index 0000000000000..eb51863358e9a Binary files /dev/null and b/docs/reference/connector/docs/images/use-a-connector-workflow.png differ diff --git a/docs/reference/connector/docs/images/workplace-search-create-role-mapping.png b/docs/reference/connector/docs/images/workplace-search-create-role-mapping.png new file mode 100644 index 0000000000000..92c22b80ea30d Binary files /dev/null and b/docs/reference/connector/docs/images/workplace-search-create-role-mapping.png differ diff --git a/docs/reference/connector/docs/images/workplace-search-kibana-ui.png b/docs/reference/connector/docs/images/workplace-search-kibana-ui.png new file mode 100644 index 0000000000000..ea5f4fcfca4df Binary files /dev/null and b/docs/reference/connector/docs/images/workplace-search-kibana-ui.png differ diff --git a/docs/reference/connector/docs/images/workplace-search-standalone-ui.png b/docs/reference/connector/docs/images/workplace-search-standalone-ui.png new file mode 100644 index 0000000000000..b3a1e12e63fa3 Binary files /dev/null and b/docs/reference/connector/docs/images/workplace-search-standalone-ui.png differ diff --git a/docs/reference/connector/docs/index.asciidoc b/docs/reference/connector/docs/index.asciidoc new file mode 100644 index 0000000000000..dfca45f86ebce --- /dev/null +++ b/docs/reference/connector/docs/index.asciidoc @@ -0,0 +1,130 @@ +[#es-connectors] += Ingest content with Elastic connectors +++++ +Connectors +++++ + +.Connectors documentation history +**** +Please note that the connectors documentation lived in the https://www.elastic.co/guide/en/enterprise-search/8.15/connectors-references.html[Enterprise Search documentation] prior to version 8.16.0. +**** + +A _connector_ is a type of https://www.elastic.co/integrations/data-integrations[Elastic integration^] that syncs content from an original data source to an *Elasticsearch index*. +Connectors enable you to create _searchable_, read-only replicas of your data sources. + +[IMPORTANT] +==== +These connectors are focused on general content, which is non-timestamped data. +Refer to https://www.elastic.co/guide/en/cloud/current/ec-cloud-ingest-data.html[add data to {es}] if you're interested in ingesting timestamped data. +==== + +Connectors extract the original files, records, or objects; and transforms them into Elasticsearch documents. + +Many connectors are available out-of-the-box on Elastic Cloud. +You can also access the source code for these (and additional) connectors, and run them on your own infrastructure. + +* **Managed connectors** are available directly within your Elastic Cloud deployment. +* **Self-managed connectors** are self-managed on your own infrastructure. + +[discrete#es-connectors-native] +== Elastic managed connectors {ess-icon} + +_Managed connectors_ are available directly within your Elastic Cloud deployment, as a managed service. +No additional infrastructure is required. + +Please note that Elastic managed connectors were previously known as "native connectors". + +Refer to <> for details on how to configure and use Elastic managed connectors. + +.*Expand* for list of available Elastic managed connectors +[%collapsible] +==== +include::_connectors-list-native.asciidoc[] +==== + +[discrete#es-connectors-build] +== Self-managed connectors + +Self-managed connectors enable you to run connectors locally on your own infrastructure. +This means you can try out new connectors before they are available natively within Elastic Cloud, and/or customize existing connectors. + +Please note that self-managed connectors were previously known as "connector clients". + +Refer to <> for details on how to deploy self-managed connectors. + +.*Expand* for list of available self-managed connectors +[%collapsible] +==== +include::_connectors-list-clients.asciidoc[] +==== + +[discrete#es-connectors-overview-framework] +== Connector framework + +All Elastic connectors are built using our Python connector framework. +The source code is available in the {connectors-python}[`elastic/connectors`] repository on GitHub. + +The connector framework is available for developers to customize existing self-managed connectors or build their own connectors. +Refer to <> for details. + +[discrete#es-connectors-overview-diagram] +== Connectors overview diagram + +The following diagram provides a high-level overview of the Elastic connectors offering and some key facts. + +image::connectors-overview.svg[align="center",width="100%"] + +[discrete#es-connectors-overview-available-connectors] +== Available connectors and feature support + +include::_connectors-overview-table.asciidoc[] + + +:connectors-branch: {branch} + +ifeval::['{branch}' == 'master'] +:connectors-branch: main +endif::[] + +:connectors-python: https://github.com/elastic/connectors/tree/{connectors-branch} +:connectors-ruby: https://github.com/elastic/connectors-ruby/tree/{connectors-branch} + +include::connectors-refs.asciidoc[] + + +include::connectors-self-managed.asciidoc[] +include::connectors-run-from-docker.asciidoc[] +include::connectors-run-from-source.asciidoc[] +include::connectors-docker-compose-quickstart.asciidoc[] +include::postgresql-connector-client-tutorial.asciidoc[] + + +include::connectors-managed-service.asciidoc[] +include::connectors-hosted-tutorial-mongo.asciidoc[] + +include::connectors-framework.asciidoc[] + +include::connectors-usage.asciidoc[] + +include::connectors-APIs.asciidoc[] +include::connectors-API-tutorial.asciidoc[] +include::connectors-content-syncs.asciidoc[] +include::connectors-filter-extract-transform.asciidoc[] +include::connectors-content-extraction.asciidoc[] +include::sync-rules.asciidoc[] + +include::dls.asciidoc[] + + +include::connectors-management.asciidoc[] +include::connectors-scalability.asciidoc[] +include::connectors-security.asciidoc[] +include::connectors-troubleshooting.asciidoc[] +include::connectors-logs.asciidoc[] + +include::connectors-use-cases.asciidoc[] + + +include::connectors-release-notes.asciidoc[] +include::connectors-known-issues.asciidoc[] + diff --git a/docs/reference/connector/docs/postgresql-connector-client-tutorial.asciidoc b/docs/reference/connector/docs/postgresql-connector-client-tutorial.asciidoc new file mode 100644 index 0000000000000..3a3ab242a47aa --- /dev/null +++ b/docs/reference/connector/docs/postgresql-connector-client-tutorial.asciidoc @@ -0,0 +1,234 @@ +[#es-postgresql-connector-client-tutorial] +=== PostgreSQL self-managed connector tutorial +++++ +Tutorial +++++ + +This tutorial walks you through the process of creating a self-managed connector for a PostgreSQL data source. +You'll be using the <> workflow in the Kibana UI. +This means you'll be deploying the connector on your own infrastructure. +Refer to the <> for more information about this connector. + +You'll use the {connectors-python}[connector framework^] to create the connector. +In this exercise, you'll be working in both the terminal (or your IDE) and the Kibana UI. + +If you want to deploy a self-managed connector for another data source, use this tutorial as a blueprint. +Refer to the list of available <>. + +[TIP] +==== +Want to get started quickly testing a self-managed connector using Docker Compose? +Refer to this https://github.com/elastic/connectors/tree/main/scripts/stack#readme[README] in the `elastic/connectors` repo for more information. +==== + +[discrete#es-postgresql-connector-client-tutorial-prerequisites] +==== Prerequisites + +[discrete#es-postgresql-connector-client-tutorial-prerequisites-elastic] +===== Elastic prerequisites + +First, ensure you satisfy the <> for self-managed connectors. + +[discrete#es-postgresql-connector-client-tutorial-postgresql-prerequisites] +===== PostgreSQL prerequisites + +You need: + +* PostgreSQL version 11+. +* Tables must be owned by a PostgreSQL user. +* Database `superuser` privileges are required to index all database tables. + +[TIP] +==== +You should enable recording of the commit time of PostgreSQL transactions. +Otherwise, _all_ data will be indexed in every sync. +By default, `track_commit_timestamp` is `off`. + +Enable this by running the following command on the PosgreSQL server command line: + +[source,shell] +---- +ALTER SYSTEM SET track_commit_timestamp = on; +---- + +Then restart the PostgreSQL server. +==== + +[discrete#es-postgresql-connector-client-tutorial-steps] +==== Steps + +To complete this tutorial, you'll need to complete the following steps: + +. <> +. <> +. <> +. <> + +[discrete#es-postgresql-connector-client-tutorial-create-index] +==== Create an Elasticsearch index + +Elastic connectors enable you to create searchable, read-only replicas of your data sources in Elasticsearch. +The first step in setting up your self-managed connector is to create an index. + +In the {kibana-ref}[Kibana^] UI, navigate to *Search > Content > Elasticsearch indices* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. + +Create a new connector index: + +. Under *Select an ingestion method* choose *Connector*. +. Choose *PostgreSQL* from the list of connectors. +. Name your index and optionally change the language analyzer to match the human language of your data source. +(The index name you provide is automatically prefixed with `search-`.) +. Save your changes. + +The index is created and ready to configure. + +[discrete#es-postgresql-connector-client-tutorial-gather-elastic-details] +.Gather Elastic details +**** +Before you can configure the connector, you need to gather some details about your Elastic deployment: + +* *Elasticsearch endpoint*. +** If you're an Elastic Cloud user, find your deployment’s Elasticsearch endpoint in the Cloud UI under *Cloud > Deployments > > Elasticsearch*. +** If you're running your Elastic deployment and the connector service in Docker, the default Elasticsearch endpoint is `http://host.docker.internal:9200`. +* *API key.* +You'll need this key to configure the connector. +Use an existing key or create a new one. +* *Connector ID*. +Your unique connector ID is automatically generated when you create the connector. +Find this in the Kibana UI. +**** + +[discrete#es-postgresql-connector-client-tutorial-setup-connector] +==== Set up the connector + +Once you've created an index, you can set up the connector. +You will be guided through this process in the UI. + +. *Edit the name and description for the connector.* +This will help your team identify the connector. +. *Clone and edit the connector service code.* +For this example, we'll use the {connectors-python}[Python framework^]. +Follow these steps: +** Clone or fork that repository locally with the following command: `git clone https://github.com/elastic/connectors`. +** Open the `config.yml` configuration file in your editor of choice. +** Replace the values for `host`, `api_key`, and `connector_id` with the values you gathered <>. +Use the `service_type` value `postgresql` for this connector. ++ +.*Expand* to see an example `config.yml` file +[%collapsible] +==== +Replace the values for `host`, `api_key`, and `connector_id` with your own values. +Use the `service_type` value `postgresql` for this connector. +[source,yaml] +---- +elasticsearch: + host: > # Your Elasticsearch endpoint + api_key: '' # Your top-level Elasticsearch API key +... +connectors: + - + connector_id: "" + api_key: "'" # Your scoped connector index API key (optional). If not provided, the top-level API key is used. + service_type: "postgresql" + + + +# Self-managed connector settings +connector_id: '' # Your connector ID +service_type: 'postgresql' # The service type for your connector + +sources: + # mongodb: connectors.sources.mongo:MongoDataSource + # s3: connectors.sources.s3:S3DataSource + # dir: connectors.sources.directory:DirectoryDataSource + # mysql: connectors.sources.mysql:MySqlDataSource + # network_drive: connectors.sources.network_drive:NASDataSource + # google_cloud_storage: connectors.sources.google_cloud_storage:GoogleCloudStorageDataSource + # azure_blob_storage: connectors.sources.azure_blob_storage:AzureBlobStorageDataSource + postgresql: connectors.sources.postgresql:PostgreSQLDataSource + # oracle: connectors.sources.oracle:OracleDataSource + # sharepoint: connectors.sources.sharepoint:SharepointDataSource + # mssql: connectors.sources.mssql:MSSQLDataSource + # jira: connectors.sources.jira:JiraDataSource +---- +==== + +[discrete#es-postgresql-connector-client-tutorial-run-connector-service] +==== Run the connector service + +Now that you've configured the connector code, you can run the connector service. + +In your terminal or IDE: + +. `cd` into the root of your `connectors` clone/fork. +. Run the following command: `make run`. + +The connector service should now be running. +The UI will let you know that the connector has successfully connected to Elasticsearch. + +Here we're working locally. +In production setups, you'll deploy the connector service to your own infrastructure. +If you prefer to use Docker, refer to the {connectors-python}/docs/DOCKER.md[repo docs^] for instructions. + +[discrete#es-postgresql-connector-client-tutorial-sync-data-source] +==== Sync your PostgreSQL data source + +[discrete#es-postgresql-connector-client-tutorial-sync-data-source-details] +===== Enter your PostgreSQL data source details + +Once you've configured the connector, you can use it to index your data source. + +You can now enter your PostgreSQL instance details in the Kibana UI. + +Enter the following information: + +* *Host*. +Server host address for your PostgreSQL instance. +* *Port*. +Port number for your PostgreSQL instance. +* *Username*. +Username of the PostgreSQL account. +* *Password*. +Password for that user. +* *Database*. +Name of the PostgreSQL database. +* *Comma-separated list of tables*. +`*` will fetch data from all tables in the configured database. + +Once you've entered all these details, select *Save configuration*. + +[discrete#es-postgresql-connector-client-tutorial-sync-data-source-launch-sync] +===== Launch a sync + +If you navigate to the *Overview* tab in the Kibana UI, you can see the connector's _ingestion status_. +This should now have changed to *Configured*. + +It's time to launch a sync by selecting the *Sync* button. + +If you navigate to the terminal window where you're running the connector service, you should see output like the following: + +[source,shell] +---- +[FMWK][13:22:26][INFO] Fetcher +[FMWK][13:22:26][INF0] Fetcher +[FMWK][13:22:26][INFO] Fetcher +... +[FMWK][23:22:28][INF0] [oRXQwYYBLhXTs-qYpJ9i] Sync done: 3864 indexed, 0 deleted. +(27 seconds) +---- + +This confirms the connector has fetched records from your PostgreSQL table(s) and transformed them into documents in your Elasticsearch index. + +Verify your Elasticsearch documents in the *Documents* tab in the Kibana UI. + +If you're happy with the results, set a recurring sync schedule in the *Scheduling* tab. +This will ensure your _searchable_ data in Elasticsearch is always up to date with changes to your PostgreSQL data source. + +[discrete#es-postgresql-connector-client-tutorial-learn-more] +==== Learn more + +* <> +* {connectors-python}[Elastic connector framework repository^] +* <> +* <> +* <> \ No newline at end of file diff --git a/docs/reference/connector/docs/sync-rules.asciidoc b/docs/reference/connector/docs/sync-rules.asciidoc new file mode 100644 index 0000000000000..9b2a77be7db03 --- /dev/null +++ b/docs/reference/connector/docs/sync-rules.asciidoc @@ -0,0 +1,327 @@ +[#es-sync-rules] +=== Connector sync rules +++++ +Sync rules +++++ + +Use connector sync rules to help control which documents are synced between the third-party data source and Elasticsearch. +Define sync rules in the Kibana UI for each connector index, under the `Sync rules` tab for the index. + +Sync rules apply to <> and <>. + +[discrete#es-sync-rules-availability-prerequisites] +==== Availability and prerequisites + +In Elastic versions *8.8.0 and later* all connectors have support for _basic_ sync rules. + +Some connectors support _advanced_ sync rules. +Learn more in the <>. + +[discrete#es-sync-rules-types] +==== Types of sync rule + +There are two types of sync rule: + +* **Basic sync rules** - these rules are represented in a table-like view. +Basic sync rules are identical for all connectors. +* **Advanced sync rules** - these rules cover complex query-and-filter scenarios that cannot be expressed with basic sync rules. +Advanced sync rules are defined through a _source-specific_ DSL JSON snippet. + +[.screenshot] +image::images/filtering-rules-zero-state.png[Sync rules tab] + +[discrete#es-sync-rules-general-filtering] +==== General data filtering concepts + +Before discussing sync rules, it's important to establish a basic understanding of _data filtering_ concepts. +The following diagram shows that data filtering can occur in several different processes/locations. + +[.screenshot] +image::images/filtering-general-diagram.png[Filtering] + +In this documentation we will focus on remote and integration filtering. +Sync rules can be used to modify both of these. + +[discrete#es-sync-rules-general-filtering-remote] +===== Remote filtering + +Data might be filtered at its source. +We call this *remote filtering*, as the filtering process is external to Elastic. + +[discrete#es-sync-rules-general-filtering-integration] +===== Integration filtering + +*Integration filtering* acts as a bridge between the original data source and Elasticsearch. +Filtering that takes place in connectors is an example of integration filtering. + +[discrete#es-sync-rules-general-filtering-pipeline] +===== Pipeline filtering + +Finally, Elasticsearch can filter data right _before persistence_ using {ref}/ingest-pipeline-search.html[ingest pipelines]. +We will not focus on ingest pipeline filtering in this guide. + +[NOTE] +==== +Currently, basic sync rules are the only way to control _integration filtering_ for connectors. +Remember that remote filtering extends far beyond the scope of connectors alone. +For best results, collaborate with the owners and maintainers of your data source. +Ensure the source data is well-organized and optimized for the query types made by the connectors. +==== + +[discrete#es-sync-rules-overview] +==== Sync rules overview + +In most cases, your data lake will contain far more data than you want to expose to end users. +For example, you may want to search a product catalog, but not include vendor contact information, even if the two are co-located for business purposes. + +The optimal time to filter data is _early_ in the data pipeline. +There are two main reasons: + +* *Performance*: +It's more efficient to send a query to the backing data source than to obtain all the data and then filter it in the connector. +It's faster to send a smaller dataset over a network and to process it on the connector side. +* *Security*: +Query-time filtering is applied on the data source side, so the data is not sent over the network and into the connector, which limits the exposure of your data. + +In a perfect world, all filtering would be done as remote filtering. + +In practice, however, this is not always possible. +Some sources do not allow robust remote filtering. +Others do, but require special setup (building indexes on specific fields, tweaking settings, etc.) that may require attention from other stakeholders in your organization. + +With this in mind, sync rules were designed to modify both remote filtering and integration filtering. +Your goal should be to do as much remote filtering as possible, but integration is a perfectly viable fall-back. +By definition, remote filtering is applied before the data is obtained from a third-party source. +Integration filtering is applied after the data is obtained from a third-party source, but before it is ingested into the Elasticsearch index. + +[NOTE] +==== +All sync rules are applied to a given document _before_ any {ref}/ingest-pipeline-search.html[ingest pipelines] are run on that document. +Therefore, you can use ingest pipelines for any processing that must occur _after_ integration filtering has occurred. +==== + +[NOTE] +==== +If a sync rule is added, edited or removed, it will only take effect after the next full sync. +==== + +[discrete#es-sync-rules-basic] +==== Basic sync rules + +Each basic sync rules can be one of two "policies": `include` and `exclude`. +`Include` rules are used to include the documents that "match" the specified condition. +`Exclude` rules are used to exclude the documents that "match" the specified condition. + +A "match" is determined based on a condition defined by a combination of "field", "rule", and "value". + +The `Field` column should be used to define which field on a given document should be considered. + +The following rules are available in the `Rule` column: + +* `equals` - The field value is equal to the specified value. +* `starts_with` - The field value starts with the specified (string) value. +* `ends_with` - The field value ends with the specified (string) value. +* `contains` - The field value includes the specified (string) value. +* `regex` - The field value matches the specified https://en.wikipedia.org/wiki/Regular_expression[regular expression^]. +* `>` - The field value is greater than the specified value. +* `<` - The field value is less than the specified value. + +Finally, the `Value` column is dependent on: + +* the data type in the specified "field" +* which "rule" was selected. + +For example, if a value of `[A-Z]{2}` might make sense for a `regex` rule, but much less so for a `>` rule. +Similarly, you probably wouldn't have a value of `espresso` when operating on an `ip_address` field, but perhaps you would for a `beverage` field. + +[discrete#es-sync-rules-basic-examples] +===== Basic sync rules examples + +[discrete#es-sync-rules-basic-examples-1] +====== Example 1 + +Exclude all documents that have an `ID` field with the value greater than 1000. + +[.screenshot] +image::images/simple-rule-greater.png[Simple greater than rule] + +[discrete#es-sync-rules-basic-examples-2] +====== Example 2 + +Exclude all documents that have a `state` field that matches a specified regex. + +[.screenshot] +image::images/simple-rule-regex.png[Simple regex rule] + +[discrete#es-sync-rules-performance-implications] +===== Performance implications + +- If you're relying solely on basic sync rules in the integration filtering phase the connector will fetch *all* the data from the data source +- For data sources without automatic pagination, or similar optimizations, fetching all the data can lead to memory issues. +For example, loading datasets which are too big to fit in memory at once. + +[NOTE] +==== +The native MongoDB connector provided by Elastic uses pagination and therefore has optimized performance. +Keep in mind that custom community-built self-managed connectors may not have these performance optimizations. +==== + +The following diagrams illustrate the concept of pagination. +A huge data set may not fit into a connector instance's memory. +Splitting data into smaller chunks reduces the risk of out-of-memory errors. + +This diagram illustrates an entire dataset being extracted at once: +[.screenshot] +image::images/sync-rules-extract-all-at-once.png[Extract whole dataset at once] + +By comparison, this diagram illustrates a paginated dataset: + +[.screenshot] +image::images/sync-rules-pagination.png[Pagination] + +[discrete#es-sync-rules-advanced] +==== Advanced sync rules + +[IMPORTANT] +==== +Advanced sync rules overwrite any remote filtering query that could have been inferred from the basic sync rules. +If an advanced sync rule is defined, any defined basic sync rules will be used exclusively for integration filtering. +==== + +Advanced sync rules are only used in remote filtering. +You can think of advanced sync rules as a language-agnostic way to represent queries to the data source. +Therefore, these rules are highly *source-specific*. + +The following connectors support advanced sync rules: + +include::_connectors-list-advanced-rules.asciidoc[] + +Each connector supporting advanced sync rules provides its own DSL to specify rules. +Refer to the documentation for <> for details. + +[discrete#es-interplay-basic-rules-advanced-rules] +==== Combining basic and advanced sync rules + +You can also use basic sync rules and advanced sync rules together to filter a data set. + +The following diagram provides an overview of the order in which advanced sync rules, basic sync rules, and pipeline filtering, are applied to your documents: + +[.screenshot] +image::images/sync-rules-time-dimension.png[Sync Rules: What is applied when?] + +[discrete#es-example-interplay-basic-rules-advanced-rules] +===== Example + +In the following example we want to filter a data set containing apartments to only contain apartments with specific properties. +We'll use basic and advanced sync rules throughout the example. + +A sample apartment looks like this in the `.json` format: +[source, js] +---- + { + "id": 1234, + "bedrooms": 3, + "price": 1500, + "address": { + "street": "Street 123", + "government_area": "Area", + "country_information": { + "country_code": "PT", + "country": "Portugal" + } + } +} +---- +// NOTCONSOLE + +The target data set should fulfill the following conditions: + +. Every apartment should have at least *3 bedrooms* +. The apartments should not be more expensive than *1500 per month* +. The apartment with id '1234' should get included without considering the first two conditions +. Each apartment should be located in either 'Portugal' or 'Spain' + +The first 3 conditions can be handled by basic sync rules, but we'll need to use advanced sync rules for number 4. + +[discrete#es-example-interplay-basic-rules] +====== Basic sync rules examples + +To create a new basic sync rule, navigate to the 'Sync Rules' tab and select *Draft new sync rules*: + +[.screenshot] +image::images/sync-rules-draft-new-rules.png[Draft new rules] + +Afterwards you need to press the 'Save and validate draft' button to validate these rules. +Note that when saved the rules will be in _draft_ state. They won't be executed in the next sync unless they are _applied_. + +[.screenshot] +image::images/sync-rules-save-and-validate-draft.png[Save and validate draft] + +After a successful validation you can apply your rules so they'll be executed in the next sync. + +These following conditions can be covered by basic sync rules: + +1. The apartment with id '1234' should get included without considering the first two conditions +2. Every apartment should have at least three bedrooms +3. The apartments should not be more expensive than 1000/month + +[.screenshot] +image::images/sync-rules-rules-fulfilling-properties.png[Save and validate draft] + +[NOTE] +==== +Remember that order matters for basic sync rules. +You may get different results for a different ordering. +==== + +[discrete#es-example-interplay-advanced-rules] +====== Advanced sync rules example + +You want to only include apartments which are located in Portugal or Spain. +We need to use advanced sync rules here, because we're dealing with deeply nested objects. + +Let's assume that the apartment data is stored inside a MongoDB instance. +For MongoDB we support https://www.mongodb.com/docs/manual/core/aggregation-pipeline/[aggregation pipelines^] in our advanced sync rules among other things. +An aggregation pipeline to only select properties located in Portugal or Spain looks like this: +[source, js] +---- + [ + { + "$match": { + "$or": [ + { + "address.country_information.country": "Portugal" + }, + { + "address.country_information.country": "Spain" + } + ] + } + } + ] +---- +// NOTCONSOLE + +To create these advanced sync rules navigate to the sync rules creation dialog and select the 'Advanced rules' tab. +You can now paste your aggregation pipeline into the input field under `aggregate.pipeline`: + +[.screenshot] +image::images/sync-rules-paste-aggregation-pipeline.png[Paste aggregation pipeline] + +Once validated, apply these rules. +The following screenshot shows the applied sync rules, which will be executed in the next sync: + +[.screenshot] +image::images/sync-rules-advanced-rules-appeared.png[Advanced sync rules appeared] + +After a successful sync you can expand the sync details to see which rules were applied: + +[.screenshot] +image::images/sync-rules-applied-rules-during-sync.png[Applied rules during sync] + +[WARNING] +==== +Active sync rules can become invalid when changed outside of the UI. +Sync jobs with invalid rules will fail. +One workaround is to revalidate the draft rules and override the invalid active rules. +==== diff --git a/docs/reference/data-management.asciidoc b/docs/reference/data-management.asciidoc index 4245227a1524d..7ef021dc6370b 100644 --- a/docs/reference/data-management.asciidoc +++ b/docs/reference/data-management.asciidoc @@ -6,29 +6,26 @@ -- The data you store in {es} generally falls into one of two categories: -* Content: a collection of items you want to search, such as a catalog of products -* Time series data: a stream of continuously-generated timestamped data, such as log entries - -Content might be frequently updated, +* *Content*: a collection of items you want to search, such as a catalog of products +* *Time series data*: a stream of continuously-generated timestamped data, such as log entries +*Content* might be frequently updated, but the value of the content remains relatively constant over time. You want to be able to retrieve items quickly regardless of how old they are. -Time series data keeps accumulating over time, so you need strategies for +*Time series data* keeps accumulating over time, so you need strategies for balancing the value of the data against the cost of storing it. As it ages, it tends to become less important and less-frequently accessed, so you can move it to less expensive, less performant hardware. For your oldest data, what matters is that you have access to the data. It's ok if queries take longer to complete. -To help you manage your data, {es} offers you: - -* <> ({ilm-init}) to manage both indices and data streams and it is fully customisable, and -* <> which is the built-in lifecycle of data streams and addresses the most -common lifecycle management needs. +To help you manage your data, {es} offers you the following options: -preview::["The built-in data stream lifecycle is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but this feature is not subject to the support SLA of official GA features."] +* <> +* <> +* {curator-ref-current}/about.html[Elastic Curator] -**{ilm-init}** can be used to manage both indices and data streams and it allows you to: +**{ilm-init}** can be used to manage both indices and data streams. It allows you to do the following: * Define the retention period of your data. The retention period is the minimum time your data will be stored in {es}. Data older than this period can be deleted by {es}. @@ -38,12 +35,24 @@ Data older than this period can be deleted by {es}. for your older indices while reducing operating costs and maintaining search performance. * Perform <> of data stored on less-performant hardware. -**Data stream lifecycle** is less feature rich but is focused on simplicity, so it allows you to easily: +**Data stream lifecycle** is less feature rich but is focused on simplicity. It allows you to do the following: * Define the retention period of your data. The retention period is the minimum time your data will be stored in {es}. Data older than this period can be deleted by {es} at a later time. -* Improve the performance of your data stream by performing background operations that will optimise the way your data -stream is stored. +* Improve the performance of your data stream by performing background operations that will optimise the way your data stream is stored. + +**Elastic Curator** is a tool that allows you to manage your indices and snapshots using user-defined filters and predefined actions. If ILM provides the functionality to manage your index lifecycle, and you have at least a Basic license, consider using ILM in place of Curator. Many stack components make use of ILM by default. {curator-ref-current}/ilm.html[Learn more]. + +NOTE: <> is a deprecated Elasticsearch feature that allows you to manage the amount of data that is stored in your cluster, similar to the downsampling functionality of {ilm-init} and data stream lifecycle. This feature should not be used for new deployments. + +[TIP] +==== +{ilm-init} is not available on {es-serverless}. + +In an {ecloud} or self-managed environment, ILM lets you automatically transition indices through data tiers according to your performance needs and retention requirements. This allows you to balance hardware costs with performance. {es-serverless} eliminates this complexity by optimizing your cluster performance for you. + +Data stream lifecycle is an optimized lifecycle tool that lets you focus on the most common lifecycle management needs, without unnecessary hardware-centric concepts like data tiers. +==== -- include::ilm/index.asciidoc[] diff --git a/docs/reference/data-store-architecture.asciidoc b/docs/reference/data-store-architecture.asciidoc new file mode 100644 index 0000000000000..4ee75c15562ea --- /dev/null +++ b/docs/reference/data-store-architecture.asciidoc @@ -0,0 +1,18 @@ += Data store architecture + +[partintro] +-- + +{es} is a distributed document store. Instead of storing information as rows of columnar data, {es} stores complex data structures that have been serialized as JSON documents. When you have multiple {es} nodes in a cluster, stored documents are distributed across the cluster and can be accessed immediately +from any node. + +The topics in this section provides information about the architecture of {es} and how it stores and retrieves data: + +* <>: Learn about the basic building blocks of an {es} cluster, including nodes, shards, primaries, and replicas. +* <>: Learn how {es} replicates read and write operations across shards and shard copies. +* <>: Learn how {es} allocates and balances shards across nodes. +-- + +include::nodes-shards.asciidoc[] +include::docs/data-replication.asciidoc[leveloffset=-1] +include::modules/shard-ops.asciidoc[] \ No newline at end of file diff --git a/docs/reference/data-streams/data-stream-apis.asciidoc b/docs/reference/data-streams/data-stream-apis.asciidoc index c13703ab2a6ee..8b952fad59f8d 100644 --- a/docs/reference/data-streams/data-stream-apis.asciidoc +++ b/docs/reference/data-streams/data-stream-apis.asciidoc @@ -2,6 +2,12 @@ [[data-stream-apis]] == Data stream APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs]. +-- + The following APIs are available for managing <>: * <> diff --git a/docs/reference/data-streams/lifecycle/apis/delete-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/delete-lifecycle.asciidoc index 315f7fa85e45f..5222d33b5870b 100644 --- a/docs/reference/data-streams/lifecycle/apis/delete-lifecycle.asciidoc +++ b/docs/reference/data-streams/lifecycle/apis/delete-lifecycle.asciidoc @@ -4,6 +4,12 @@ Delete Data Stream Lifecycle ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs]. +-- + Deletes the <> from a set of data streams. [[delete-lifecycle-api-prereqs]] diff --git a/docs/reference/data-streams/lifecycle/apis/explain-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/explain-lifecycle.asciidoc index 2b15886ebe192..8289fb54d51bd 100644 --- a/docs/reference/data-streams/lifecycle/apis/explain-lifecycle.asciidoc +++ b/docs/reference/data-streams/lifecycle/apis/explain-lifecycle.asciidoc @@ -4,6 +4,12 @@ Explain Data Stream Lifecycle ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs]. +-- + Retrieves the current <> status for one or more data stream backing indices. [[explain-lifecycle-api-prereqs]] diff --git a/docs/reference/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc b/docs/reference/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc index f48fa1eb52daa..0fbe7de287f7b 100644 --- a/docs/reference/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc +++ b/docs/reference/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc @@ -4,6 +4,12 @@ Get Data Stream Lifecycle ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs]. +-- + Gets stats about the execution of <>. [[get-lifecycle-stats-api-prereqs]] diff --git a/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc index 6323fac1eac2f..57d63fee2ddc1 100644 --- a/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc +++ b/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc @@ -4,6 +4,12 @@ Get Data Stream Lifecycle ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs]. +-- + Gets the <> of a set of <>. [[get-lifecycle-api-prereqs]] diff --git a/docs/reference/data-streams/lifecycle/apis/put-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/put-lifecycle.asciidoc index c60c105e818ab..c5002cf4882e2 100644 --- a/docs/reference/data-streams/lifecycle/apis/put-lifecycle.asciidoc +++ b/docs/reference/data-streams/lifecycle/apis/put-lifecycle.asciidoc @@ -4,6 +4,12 @@ Put Data Stream Lifecycle ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs]. +-- + Configures the data stream <> for the targeted <>. [[put-lifecycle-api-prereqs]] diff --git a/docs/reference/data-streams/logs.asciidoc b/docs/reference/data-streams/logs.asciidoc index e870289bcf7be..3af5e09889a89 100644 --- a/docs/reference/data-streams/logs.asciidoc +++ b/docs/reference/data-streams/logs.asciidoc @@ -1,26 +1,20 @@ [[logs-data-stream]] == Logs data stream -preview::[Logs data streams and the logsdb index mode are in tech preview and may be changed or removed in the future. Don't use logs data streams or logsdb index mode in production.] +IMPORTANT: The {es} `logsdb` index mode is generally available in Elastic Cloud Hosted +and self-managed Elasticsearch as of version 8.17, and is enabled by default for +logs in https://www.elastic.co/elasticsearch/serverless[{serverless-full}]. A logs data stream is a data stream type that stores log data more efficiently. In benchmarks, log data stored in a logs data stream used ~2.5 times less disk space than a regular data -stream. The exact impact will vary depending on your data set. - -The following features are enabled in a logs data stream: - -* <>, which omits storing the `_source` field. When the document source is requested, it is synthesized from document fields upon retrieval. - -* Index sorting. This yields a lower storage footprint. By default indices are sorted by `host.name` and `@timestamp` fields at index time. - -* More space efficient compression for fields with <> enabled. +stream. The exact impact varies by data set. [discrete] [[how-to-use-logsds]] === Create a logs data stream -To create a logs data stream, set your index template `index.mode` to `logsdb`: +To create a logs data stream, set your <> `index.mode` to `logsdb`: [source,console] ---- @@ -39,10 +33,12 @@ PUT _index_template/my-index-template // TEST <1> The index mode setting. -<2> The index template priority. By default, Elasticsearch ships with an index template with a `logs-*-*` pattern with a priority of 100. You need to define a priority higher than 100 to ensure that this index template gets selected over the default index template for the `logs-*-*` pattern. See the <> for more information. +<2> The index template priority. By default, Elasticsearch ships with a `logs-*-*` index template with a priority of 100. To make sure your index template takes priority over the default `logs-*-*` template, set its `priority` to a number higher than 100. For more information, see <>. After the index template is created, new indices that use the template will be configured as a logs data stream. You can start indexing data and <>. +You can also set the index mode and adjust other template settings in <>. + //// [source,console] ---- @@ -50,3 +46,180 @@ DELETE _index_template/my-index-template ---- // TEST[continued] //// + +[[logsdb-default-settings]] + +[discrete] +[[logsdb-synthetic-source]] +=== Synthetic source + +If you have the required https://www.elastic.co/subscriptions[subscription], `logsdb` index mode uses <>, which omits storing the original `_source` +field. Instead, the document source is synthesized from doc values or stored fields upon document retrieval. + +If you don't have the required https://www.elastic.co/subscriptions[subscription], `logsdb` mode uses the original `_source` field. + +Before using synthetic source, make sure to review the <>. + +When working with multi-value fields, the `index.mapping.synthetic_source_keep` setting controls how field values +are preserved for <> reconstruction. In `logsdb`, the default value is `arrays`, +which retains both duplicate values and the order of entries. However, the exact structure of +array elements and objects is not necessarily retained. Preserving duplicates and ordering can be critical for some +log fields, such as DNS A records, HTTP headers, and log entries that represent sequential or repeated events. + +[discrete] +[[logsdb-sort-settings]] +=== Index sort settings + +In `logsdb` index mode, the following sort settings are applied by default: + +`index.sort.field`: `["host.name", "@timestamp"]`:: +Indices are sorted by `host.name` and `@timestamp` by default. The `@timestamp` field is automatically injected if it is not present. + +`index.sort.order`: `["desc", "desc"]`:: +Both `host.name` and `@timestamp` are sorted in descending (`desc`) order, prioritizing the latest data. + +`index.sort.mode`: `["min", "min"]`:: +The `min` mode sorts indices by the minimum value of multi-value fields. + +`index.sort.missing`: `["_first", "_first"]`:: +Missing values are sorted to appear `_first`. + +You can override these default sort settings. For example, to sort on different fields +and change the order, manually configure `index.sort.field` and `index.sort.order`. For more details, see +<>. + +When using the default sort settings, the `host.name` field is automatically injected into the index mappings as a `keyword` field to ensure that sorting can be applied. This guarantees that logs are efficiently sorted and retrieved based on the `host.name` and `@timestamp` fields. + +NOTE: If `subobjects` is set to `true` (default), the `host` field is mapped as an object field +named `host` with a `name` child field of type `keyword`. If `subobjects` is set to `false`, +a single `host.name` field is mapped as a `keyword` field. + +To apply different sort settings to an existing data stream, update the data stream's component templates, and then +perform or wait for a <>. + +NOTE: In `logsdb` mode, the `@timestamp` field is automatically injected if it's not already present. If you apply custom sort settings, the `@timestamp` field is injected into the mappings but is not +automatically added to the list of sort fields. + +[discrete] +[[logsdb-host-name]] +==== Existing data streams + +If you're enabling `logsdb` index mode on a data stream that already exists, make sure to check mappings and sorting. The `logsdb` mode automatically maps `host.name` as a keyword if it's included in the sort settings. If a `host.name` field already exists but has a different type, mapping errors might occur, preventing `logsdb` mode from being fully applied. + +To avoid mapping conflicts, consider these options: + +* **Adjust mappings:** Check your existing mappings to ensure that `host.name` is mapped as a keyword. + +* **Change sorting:** If needed, you can remove `host.name` from the sort settings and use a different set of fields. Sorting by `@timestamp` can be a good fallback. + +* **Switch to a different <>**: If resolving `host.name` mapping conflicts is not feasible, you can choose not to use `logsdb` mode. + +IMPORTANT: On existing data streams, `logsdb` mode is applied on <> (automatic or manual). + +[discrete] +[[logsdb-specialized-codecs]] +=== Specialized codecs + +By default, `logsdb` index mode uses the `best_compression` <>, which applies {wikipedia}/Zstd[ZSTD] +compression to stored fields. You can switch to the `default` codec for faster compression with a slightly larger storage footprint. + +The `logsdb` index mode also automatically applies specialized codecs for numeric doc values, in order to optimize storage usage. Numeric fields are +encoded using the following sequence of codecs: + +* **Delta encoding**: + Stores the difference between consecutive values instead of the actual values. + +* **Offset encoding**: + Stores the difference from a base value rather than between consecutive values. + +* **Greatest Common Divisor (GCD) encoding**: + Finds the greatest common divisor of a set of values and stores the differences as multiples of the GCD. + +* **Frame Of Reference (FOR) encoding**: + Determines the smallest number of bits required to encode a block of values and uses + bit-packing to fit such values into larger 64-bit blocks. + +Each encoding is evaluated according to heuristics determined by the data distribution. +For example, the algorithm checks whether the data is monotonically non-decreasing or +non-increasing. If so, delta encoding is applied; otherwise, the process +continues with the next encoding method (offset). + +Encoding is specific to each Lucene segment and is reapplied when segments are merged. The merged Lucene segment +might use a different encoding than the original segments, depending on the characteristics of the merged data. + +For keyword fields, **Run Length Encoding (RLE)** is applied to the ordinals, which represent positions in the Lucene +segment-level keyword dictionary. This compression is used when multiple consecutive documents share the same keyword. + +[discrete] +[[logsdb-ignored-settings]] +=== `ignore` settings + +The `logsdb` index mode uses the following `ignore` settings. You can override these settings as needed. + +[discrete] +[[logsdb-ignore-malformed]] +==== `ignore_malformed` + +By default, `logsdb` index mode sets `ignore_malformed` to `true`. With this setting, documents with malformed fields +can be indexed without causing ingestion failures. + +[discrete] +[[logs-db-ignore-above]] +==== `ignore_above` + +In `logsdb` index mode, the `index.mapping.ignore_above` setting is applied by default at the index level to ensure +efficient storage and indexing of large keyword fields.The index-level default for `ignore_above` is 8191 +_characters._ Using UTF-8 encoding, this results in a limit of 32764 bytes, depending on character encoding. + +The mapping-level `ignore_above` setting takes precedence. If a specific field has an `ignore_above` value +defined in its mapping, that value overrides the index-level `index.mapping.ignore_above` value. This default +behavior helps to optimize indexing performance by preventing excessively large string values from being indexed. + +If you need to customize the limit, you can override it at the mapping level or change the index level default. + +[discrete] +[[logs-db-ignore-limit]] +==== `ignore_dynamic_beyond_limit` + +In `logsdb` index mode, the setting `index.mapping.total_fields.ignore_dynamic_beyond_limit` is set to `true` by +default. This setting allows dynamically mapped fields to be added on top of statically defined fields, even when the total number of fields exceeds the `index.mapping.total_fields.limit`. Instead of triggering an index failure, additional dynamically mapped fields are ignored so that ingestion can continue. + +NOTE: When automatically injected, `host.name` and `@timestamp` count toward the limit of mapped fields. If `host.name` is mapped with `subobjects: true`, it has two fields. When mapped with `subobjects: false`, `host.name` has only one field. + +[discrete] +[[logsdb-nodocvalue-fields]] +=== Fields without `doc_values` + +When the `logsdb` index mode uses synthetic `_source` and `doc_values` are disabled for a field in the mapping, +{es} might set the `store` setting to `true` for that field. This ensures that the field's +data remains accessible for reconstructing the document's source when using +<>. + +For example, this adjustment occurs with text fields when `store` is `false` and no suitable multi-field is available for +reconstructing the original value. + +[discrete] +[[logsdb-settings-summary]] +=== Settings reference + +The `logsdb` index mode uses the following settings: + +* **`index.mode`**: `"logsdb"` + +* **`index.mapping.synthetic_source_keep`**: `"arrays"` + +* **`index.sort.field`**: `["host.name", "@timestamp"]` + +* **`index.sort.order`**: `["desc", "desc"]` + +* **`index.sort.mode`**: `["min", "min"]` + +* **`index.sort.missing`**: `["_first", "_first"]` + +* **`index.codec`**: `"best_compression"` + +* **`index.mapping.ignore_malformed`**: `true` + +* **`index.mapping.ignore_above`**: `8191` + +* **`index.mapping.total_fields.ignore_dynamic_beyond_limit`**: `true` diff --git a/docs/reference/data-streams/modify-data-streams-api.asciidoc b/docs/reference/data-streams/modify-data-streams-api.asciidoc index 2da869083df22..2f717f9ec3b4b 100644 --- a/docs/reference/data-streams/modify-data-streams-api.asciidoc +++ b/docs/reference/data-streams/modify-data-streams-api.asciidoc @@ -4,6 +4,12 @@ Modify data streams ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs]. +-- + Performs one or more <> modification actions in a single atomic operation. diff --git a/docs/reference/data-streams/promote-data-stream-api.asciidoc b/docs/reference/data-streams/promote-data-stream-api.asciidoc index 111c7a2256f8a..8247d092bcb29 100644 --- a/docs/reference/data-streams/promote-data-stream-api.asciidoc +++ b/docs/reference/data-streams/promote-data-stream-api.asciidoc @@ -5,6 +5,12 @@ Promote data stream ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs]. +-- + The purpose of the promote <> API is to turn a data stream that is replicated by CCR into a regular data stream. diff --git a/docs/reference/data-streams/set-up-tsds.asciidoc b/docs/reference/data-streams/set-up-tsds.asciidoc index 3a483ac351180..d082a9c4eebeb 100644 --- a/docs/reference/data-streams/set-up-tsds.asciidoc +++ b/docs/reference/data-streams/set-up-tsds.asciidoc @@ -121,7 +121,8 @@ naming scheme]. * Specify a mapping that defines your dimensions and metrics: ** One or more <> with a `time_series_dimension` value of `true`. - At least one of these dimensions must be a plain `keyword` field. + Alternatively, one or more <> fields configured as dimension containers, + provided that they will contain at least one sub-field (mapped statically or dynamically). ** One or more <>, marked using the `time_series_metric` mapping parameter. @@ -203,10 +204,9 @@ DELETE _ilm/policy/my-weather-sensor-lifecycle-policy Documents in a TSDS must include: * A `@timestamp` field -* One or more dimension fields. At least one dimension must be a `keyword` field -that matches the `index.routing_path` index setting, if specified. If not specified -explicitly, `index.routing_path` is set automatically to whichever mappings have - `time_series_dimension` set to `true`. +* One or more dimension fields. At least one dimension must match the `index.routing_path` index setting, +if specified. If not specified explicitly, `index.routing_path` is set automatically to whichever mappings have +`time_series_dimension` set to `true`. To automatically create your TSDS, submit an indexing request that targets the TSDS's name. This name must match one of your index template's @@ -285,13 +285,12 @@ POST metrics-weather_sensors-dev/_rollover Configuring a TSDS via an index template that uses component templates is a bit more complicated. Typically with component templates mappings and settings get scattered across multiple component templates. -When configuring the `index.mode` setting in a component template, the `index.routing_path` setting needs to -be defined in the same component template. Additionally the fields mentioned in the `index.routing_path` -also need to be defined in the same component template with the `time_series_dimension` attribute enabled. +If the `index.routing_path` is defined, the fields it references need to be defined in the same component +template with the `time_series_dimension` attribute enabled. -The reasons for this is that each component template needs to be valid on its own and the time series index mode -requires the `index.routing_path` setting. When configuring the `index.mode` setting in an index template, the `index.routing_path` setting is configured automatically. It is derived from -the field mappings with `time_series_dimension` attribute enabled. +The reasons for this is that each component template needs to be valid on its own. When configuring the +`index.mode` setting in an index template, the `index.routing_path` setting is configured automatically. +It is derived from the field mappings with `time_series_dimension` attribute enabled. [discrete] [[set-up-tsds-whats-next]] diff --git a/docs/reference/data-streams/tsds-reindex.asciidoc b/docs/reference/data-streams/tsds-reindex.asciidoc index 9d6594db4e779..f4d00f33c179c 100644 --- a/docs/reference/data-streams/tsds-reindex.asciidoc +++ b/docs/reference/data-streams/tsds-reindex.asciidoc @@ -202,7 +202,7 @@ POST /_component_template/destination_template POST /_index_template/2 { "index_patterns": [ - "k8s*" + "k9s*" ], "composed_of": [ "destination_template" diff --git a/docs/reference/data-streams/tsds.asciidoc b/docs/reference/data-streams/tsds.asciidoc index 01573658c33d0..1e1d56e5b4d93 100644 --- a/docs/reference/data-streams/tsds.asciidoc +++ b/docs/reference/data-streams/tsds.asciidoc @@ -17,7 +17,7 @@ metrics data. Only use a TSDS if you typically add metrics data to {es} in near real-time and `@timestamp` order. A TSDS is only intended for metrics data. For other timestamped data, such as -logs or traces, use a regular data stream. +logs or traces, use a <> or regular data stream. [discrete] [[differences-from-regular-data-stream]] @@ -109,7 +109,10 @@ parameter: * <> * <> -For a flattened field, use the `time_series_dimensions` parameter to configure an array of fields as dimensions. For details refer to <>. +For a flattened field, use the `time_series_dimensions` parameter to configure an array of fields as dimensions. +For details refer to <>. + +Dimension definitions can be simplified through <> fields. [discrete] [[time-series-metric]] @@ -294,12 +297,15 @@ When you create the matching index template for a TSDS, you must specify one or more dimensions in the `index.routing_path` setting. Each document in a TSDS must contain one or more dimensions that match the `index.routing_path` setting. -Dimensions in the `index.routing_path` setting must be plain `keyword` fields. The `index.routing_path` setting accepts wildcard patterns (for example `dim.*`) and can dynamically match new fields. However, {es} will reject any mapping -updates that add scripted, runtime, or non-dimension, non-`keyword` fields that +updates that add scripted, runtime, or non-dimension fields that match the `index.routing_path` value. +<> fields may be configured +as dimension containers. In this case, their sub-fields get included to the +routing path automatically. + TSDS documents don't support a custom `_routing` value. Similarly, you can't require a `_routing` value in mappings for a TSDS. @@ -333,4 +339,5 @@ include::tsds-index-settings.asciidoc[] include::downsampling.asciidoc[] include::downsampling-ilm.asciidoc[] include::downsampling-manual.asciidoc[] +include::downsampling-dsl.asciidoc[] include::tsds-reindex.asciidoc[] diff --git a/docs/reference/docs.asciidoc b/docs/reference/docs.asciidoc index ff2d823410a6d..ccdbaaffb2b77 100644 --- a/docs/reference/docs.asciidoc +++ b/docs/reference/docs.asciidoc @@ -1,9 +1,13 @@ [[docs]] == Document APIs -This section starts with a short introduction to {es}'s <>, followed by a detailed description of the following CRUD -APIs: +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-document[Document APIs]. +-- + +This section describes the following CRUD APIs: .Single document APIs * <> @@ -18,8 +22,6 @@ APIs: * <> * <> -include::docs/data-replication.asciidoc[] - include::docs/index_.asciidoc[] include::docs/get.asciidoc[] diff --git a/docs/reference/docs/bulk.asciidoc b/docs/reference/docs/bulk.asciidoc index 69bf3d1b7db5a..d6a555de34aef 100644 --- a/docs/reference/docs/bulk.asciidoc +++ b/docs/reference/docs/bulk.asciidoc @@ -4,6 +4,12 @@ Bulk ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-document[Document APIs]. +-- + Performs multiple indexing or delete operations in a single API call. This reduces overhead and can greatly increase indexing speed. diff --git a/docs/reference/docs/data-replication.asciidoc b/docs/reference/docs/data-replication.asciidoc index 2c1a16c81d011..6ee266070e727 100644 --- a/docs/reference/docs/data-replication.asciidoc +++ b/docs/reference/docs/data-replication.asciidoc @@ -1,6 +1,6 @@ [[docs-replication]] -=== Reading and Writing documents +=== Reading and writing documents [discrete] ==== Introduction diff --git a/docs/reference/docs/delete.asciidoc b/docs/reference/docs/delete.asciidoc index 452d7f7758bfa..046a20abdaffb 100644 --- a/docs/reference/docs/delete.asciidoc +++ b/docs/reference/docs/delete.asciidoc @@ -4,6 +4,12 @@ Delete ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-document[Document APIs]. +-- + Removes a JSON document from the specified index. [[docs-delete-api-request]] diff --git a/docs/reference/docs/get.asciidoc b/docs/reference/docs/get.asciidoc index c71215fff8d73..a3ff70fb95f6e 100644 --- a/docs/reference/docs/get.asciidoc +++ b/docs/reference/docs/get.asciidoc @@ -4,6 +4,12 @@ Get ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-document[Document APIs]. +-- + Retrieves the specified JSON document from an index. [source,console] diff --git a/docs/reference/docs/multi-get.asciidoc b/docs/reference/docs/multi-get.asciidoc index 7c3eafa9c79f3..293bd2568a34b 100644 --- a/docs/reference/docs/multi-get.asciidoc +++ b/docs/reference/docs/multi-get.asciidoc @@ -4,6 +4,12 @@ Multi get ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-document[Document APIs]. +-- + Retrieves multiple JSON documents by ID. [source,console] diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc index dc27e40ecd90b..2f6ddd344eaab 100644 --- a/docs/reference/docs/reindex.asciidoc +++ b/docs/reference/docs/reindex.asciidoc @@ -4,6 +4,12 @@ Reindex ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-document[Document APIs]. +-- + Copies documents from a source to a destination. The source can be any existing index, alias, or data stream. The destination diff --git a/docs/reference/docs/termvectors.asciidoc b/docs/reference/docs/termvectors.asciidoc index 31dfba1ac2668..d40452fb4875a 100644 --- a/docs/reference/docs/termvectors.asciidoc +++ b/docs/reference/docs/termvectors.asciidoc @@ -4,6 +4,12 @@ Term vectors ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-document[Document APIs]. +-- + Retrieves information and statistics for terms in the fields of a particular document. [source,console] diff --git a/docs/reference/docs/update-by-query.asciidoc b/docs/reference/docs/update-by-query.asciidoc index d470080fc602f..c8d68082c8ea1 100644 --- a/docs/reference/docs/update-by-query.asciidoc +++ b/docs/reference/docs/update-by-query.asciidoc @@ -4,6 +4,12 @@ Update by query ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-document[Document APIs]. +-- + Updates documents that match the specified query. If no query is specified, performs an update on every document in the data stream or index without modifying the source, which is useful for picking up mapping changes. diff --git a/docs/reference/docs/update.asciidoc b/docs/reference/docs/update.asciidoc index ca6a7e489449b..ae9ae8fe73fc6 100644 --- a/docs/reference/docs/update.asciidoc +++ b/docs/reference/docs/update.asciidoc @@ -4,6 +4,12 @@ Update ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-document[Document APIs]. +-- + Updates a document using the specified script. [[docs-update-api-request]] @@ -71,7 +77,7 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=refresh] include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=routing] `_source`:: -(Optional, list) Set to `false` to disable source retrieval (default: `true`). +(Optional, list) Set to `true` to enable source retrieval (default: `false`). You can also specify a comma-separated list of the fields you want to retrieve. `_source_excludes`:: diff --git a/docs/reference/eql/delete-async-eql-search-api.asciidoc b/docs/reference/eql/delete-async-eql-search-api.asciidoc index 2f52a1a9cb234..2783c9ac0b871 100644 --- a/docs/reference/eql/delete-async-eql-search-api.asciidoc +++ b/docs/reference/eql/delete-async-eql-search-api.asciidoc @@ -6,6 +6,12 @@ Delete async EQL search ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-eql[EQL APIs]. +-- + Deletes an <> or a <>. The API also deletes results for the search. diff --git a/docs/reference/eql/eql-apis.asciidoc b/docs/reference/eql/eql-apis.asciidoc index e8cc2b21492ae..4d6aafd2039df 100644 --- a/docs/reference/eql/eql-apis.asciidoc +++ b/docs/reference/eql/eql-apis.asciidoc @@ -1,6 +1,12 @@ [[eql-apis]] == EQL APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-eql[EQL APIs]. +-- + <> is a query language for event-based time series data, such as logs, metrics, and traces. For an overview of EQL and related tutorials, see <>. diff --git a/docs/reference/eql/eql-search-api.asciidoc b/docs/reference/eql/eql-search-api.asciidoc index d7f10f4627f6c..544e4d7325c5b 100644 --- a/docs/reference/eql/eql-search-api.asciidoc +++ b/docs/reference/eql/eql-search-api.asciidoc @@ -6,6 +6,12 @@ EQL search ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-eql[EQL APIs]. +-- + Returns search results for an <> query. EQL assumes each document in a data stream or index corresponds to an @@ -88,6 +94,53 @@ request that targets only `bar*` still returns an error. + Defaults to `true`. +`allow_partial_search_results`:: +(Optional, Boolean) + +If `false`, the request returns an error if one or more shards involved in the query are unavailable. ++ +If `true`, the query is executed only on the available shards, ignoring shard request timeouts and +<>. ++ +Defaults to `false`. ++ +To override the default for this field, set the +`xpack.eql.default_allow_partial_results` cluster setting to `true`. + + +[IMPORTANT] +==== +You can also specify this value using the `allow_partial_search_results` request body parameter. +If both parameters are specified, only the query parameter is used. +==== + + +`allow_partial_sequence_results`:: +(Optional, Boolean) + + +Used together with `allow_partial_search_results=true`, controls the behavior of sequence queries specifically +(if `allow_partial_search_results=false`, this setting has no effect). +If `true` and if some shards are unavailable, the sequences are calculated on available shards only. ++ +If `false` and if some shards are unavailable, the query only returns information about the shard failures, +but no further results. ++ +Defaults to `false`. ++ +Consider that sequences calculated with `allow_partial_search_results=true` can return incorrect results +(eg. if a <> clause matches records in unavailable shards) ++ +To override the default for this field, set the +`xpack.eql.default_allow_partial_sequence_results` cluster setting to `true`. + + +[IMPORTANT] +==== +You can also specify this value using the `allow_partial_sequence_results` request body parameter. +If both parameters are specified, only the query parameter is used. +==== + `ccs_minimize_roundtrips`:: (Optional, Boolean) If `true`, network round-trips between the local and the remote cluster are minimized when running cross-cluster search (CCS) requests. diff --git a/docs/reference/eql/get-async-eql-search-api.asciidoc b/docs/reference/eql/get-async-eql-search-api.asciidoc index 2ac4271b5b986..ff4a07811b732 100644 --- a/docs/reference/eql/get-async-eql-search-api.asciidoc +++ b/docs/reference/eql/get-async-eql-search-api.asciidoc @@ -6,6 +6,12 @@ Get async EQL search ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-eql[EQL APIs]. +-- + Returns the current status and available results for an <> or a <>. diff --git a/docs/reference/eql/get-async-eql-status-api.asciidoc b/docs/reference/eql/get-async-eql-status-api.asciidoc index 908a65773f6aa..cb3904a644921 100644 --- a/docs/reference/eql/get-async-eql-status-api.asciidoc +++ b/docs/reference/eql/get-async-eql-status-api.asciidoc @@ -5,6 +5,13 @@ ++++ Get async EQL search status ++++ + +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-eql[EQL APIs]. +-- + Returns the current status for an <> or a <> without returning results. This is a more lightweight API than diff --git a/docs/reference/esql/esql-across-clusters.asciidoc b/docs/reference/esql/esql-across-clusters.asciidoc index d13b3db1c73ea..db266fafde9d6 100644 --- a/docs/reference/esql/esql-across-clusters.asciidoc +++ b/docs/reference/esql/esql-across-clusters.asciidoc @@ -85,7 +85,7 @@ POST /_security/role/remote1 "privileges": [ "read","read_cross_cluster" ], <4> "clusters" : ["my_remote_cluster"] <5> } - ], + ], "remote_cluster": [ <6> { "privileges": [ @@ -100,15 +100,23 @@ POST /_security/role/remote1 ---- <1> The `cross_cluster_search` cluster privilege is required for the _local_ cluster. -<2> Typically, users will have permissions to read both local and remote indices. However, for cases where the role is intended to ONLY search the remote cluster, the `read` permission is still required for the local cluster. To provide read access to the local cluster, but disallow reading any indices in the local cluster, the `names` field may be an empty string. -<3> The indices allowed read access to the remote cluster. The configured <> must also allow this index to be read. -<4> The `read_cross_cluster` privilege is always required when using {esql} across clusters with the API key based security model. +<2> Typically, users will have permissions to read both local and remote indices. However, for cases where the role +is intended to ONLY search the remote cluster, the `read` permission is still required for the local cluster. +To provide read access to the local cluster, but disallow reading any indices in the local cluster, the `names` +field may be an empty string. +<3> The indices allowed read access to the remote cluster. The configured +<> must also allow this index to be read. +<4> The `read_cross_cluster` privilege is always required when using {esql} across clusters with the API key based +security model. <5> The remote clusters to which these privileges apply. -This remote cluster must be configured with a <> and connected to the remote cluster before the remote index can be queried. +This remote cluster must be configured with a <> +and connected to the remote cluster before the remote index can be queried. Verify connection using the <> API. -<6> Required to allow remote enrichment. Without this, the user cannot read from the `.enrich` indices on the remote cluster. The `remote_cluster` security privilege was introduced in version *8.15.0*. +<6> Required to allow remote enrichment. Without this, the user cannot read from the `.enrich` indices on the +remote cluster. The `remote_cluster` security privilege was introduced in version *8.15.0*. -You will then need a user or API key with the permissions you created above. The following example API call creates a user with the `remote1` role. +You will then need a user or API key with the permissions you created above. The following example API call creates +a user with the `remote1` role. [source,console] ---- @@ -119,11 +127,13 @@ POST /_security/user/remote_user } ---- -Remember that all cross-cluster requests from the local cluster are bound by the cross cluster API key’s privileges, which are controlled by the remote cluster's administrator. +Remember that all cross-cluster requests from the local cluster are bound by the cross cluster API key’s privileges, +which are controlled by the remote cluster's administrator. [TIP] ==== -Cross cluster API keys created in versions prior to 8.15.0 will need to replaced or updated to add the new permissions required for {esql} with ENRICH. +Cross cluster API keys created in versions prior to 8.15.0 will need to replaced or updated to add the new permissions +required for {esql} with ENRICH. ==== [discrete] @@ -174,6 +184,189 @@ FROM *:my-index-000001 | LIMIT 10 ---- +[discrete] +[[ccq-cluster-details]] +==== Cross-cluster metadata + +Using the `"include_ccs_metadata": true` option, users can request that +ES|QL {ccs} responses include metadata about the search on each cluster (when the response format is JSON). +Here we show an example using the async search endpoint. {ccs-cap} metadata is also present in the synchronous +search endpoint response when requested. + +[source,console] +---- +POST /_query/async?format=json +{ + "query": """ + FROM my-index-000001,cluster_one:my-index-000001,cluster_two:my-index* + | STATS COUNT(http.response.status_code) BY user.id + | LIMIT 2 + """, + "include_ccs_metadata": true +} +---- +// TEST[setup:my_index] +// TEST[s/cluster_one:my-index-000001,cluster_two:my-index//] + +Which returns: + +[source,console-result] +---- +{ + "is_running": false, + "took": 42, <1> + "columns" : [ + { + "name" : "COUNT(http.response.status_code)", + "type" : "long" + }, + { + "name" : "user.id", + "type" : "keyword" + } + ], + "values" : [ + [4, "elkbee"], + [1, "kimchy"] + ], + "_clusters": { <2> + "total": 3, + "successful": 3, + "running": 0, + "skipped": 0, + "partial": 0, + "failed": 0, + "details": { <3> + "(local)": { <4> + "status": "successful", + "indices": "blogs", + "took": 41, <5> + "_shards": { <6> + "total": 13, + "successful": 13, + "skipped": 0, + "failed": 0 + } + }, + "cluster_one": { + "status": "successful", + "indices": "cluster_one:my-index-000001", + "took": 38, + "_shards": { + "total": 4, + "successful": 4, + "skipped": 0, + "failed": 0 + } + }, + "cluster_two": { + "status": "successful", + "indices": "cluster_two:my-index*", + "took": 40, + "_shards": { + "total": 18, + "successful": 18, + "skipped": 1, + "failed": 0 + } + } + } + } +} +---- +// TEST[skip: cross-cluster testing env not set up] + +<1> How long the entire search (across all clusters) took, in milliseconds. +<2> This section of counters shows all possible cluster search states and how many cluster +searches are currently in that state. The clusters can have one of the following statuses: *running*, +*successful* (searches on all shards were successful), *skipped* (the search +failed on a cluster marked with `skip_unavailable`=`true`) or *failed* (the search +failed on a cluster marked with `skip_unavailable`=`false`). +<3> The `_clusters/details` section shows metadata about the search on each cluster. +<4> If you included indices from the local cluster you sent the request to in your {ccs}, +it is identified as "(local)". +<5> How long (in milliseconds) the search took on each cluster. This can be useful to determine +which clusters have slower response times than others. +<6> The shard details for the search on that cluster, including a count of shards that were +skipped due to the can-match phase results. Shards are skipped when they cannot have any matching data +and therefore are not included in the full ES|QL query. + + +The cross-cluster metadata can be used to determine whether any data came back from a cluster. +For instance, in the query below, the wildcard expression for `cluster-two` did not resolve +to a concrete index (or indices). The cluster is, therefore, marked as 'skipped' and the total +number of shards searched is set to zero. + +[source,console] +---- +POST /_query/async?format=json +{ + "query": """ + FROM cluster_one:my-index*,cluster_two:logs* + | STATS COUNT(http.response.status_code) BY user.id + | LIMIT 2 + """, + "include_ccs_metadata": true +} +---- +// TEST[continued] +// TEST[s/cluster_one:my-index\*,cluster_two:logs\*/my-index-000001/] + +Which returns: + +[source,console-result] +---- +{ + "is_running": false, + "took": 55, + "columns": [ + ... // not shown + ], + "values": [ + ... // not shown + ], + "_clusters": { + "total": 2, + "successful": 2, + "running": 0, + "skipped": 0, + "partial": 0, + "failed": 0, + "details": { + "cluster_one": { + "status": "successful", + "indices": "cluster_one:my-index*", + "took": 38, + "_shards": { + "total": 4, + "successful": 4, + "skipped": 0, + "failed": 0 + } + }, + "cluster_two": { + "status": "skipped", <1> + "indices": "cluster_two:logs*", + "took": 0, + "_shards": { + "total": 0, <2> + "successful": 0, + "skipped": 0, + "failed": 0 + } + } + } + } +} +---- +// TEST[skip: cross-cluster testing env not set up] + +<1> This cluster is marked as 'skipped', since there were no matching indices on that cluster. +<2> Indicates that no shards were searched (due to not having any matching indices). + + + + [discrete] [[ccq-enrich]] ==== Enrich across clusters @@ -331,8 +524,7 @@ setting. As a result, if a remote cluster specified in the request is unavailable or failed, {ccs} for {esql} queries will fail regardless of the setting. We are actively working to align the behavior of {ccs} for {esql} with other -{ccs} APIs. This includes providing detailed execution information for each cluster -in the response, such as execution time, selected target indices, and shards. +{ccs} APIs. [discrete] [[ccq-during-upgrade]] diff --git a/docs/reference/esql/esql-apis.asciidoc b/docs/reference/esql/esql-apis.asciidoc index 8586cd1ae6bce..157f4e4357e78 100644 --- a/docs/reference/esql/esql-apis.asciidoc +++ b/docs/reference/esql/esql-apis.asciidoc @@ -1,6 +1,14 @@ [[esql-apis]] == {esql} APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-esql[ES|QL APIs]. +-- + +https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-esql-query + The <> provides a powerful way to filter, transform, and analyze data stored in {es}, and in the future in other runtimes. For an overview of {esql} and related tutorials, see <>. diff --git a/docs/reference/esql/esql-async-query-api.asciidoc b/docs/reference/esql/esql-async-query-api.asciidoc index 6cd23fc524f96..8cb974cf6773b 100644 --- a/docs/reference/esql/esql-async-query-api.asciidoc +++ b/docs/reference/esql/esql-async-query-api.asciidoc @@ -4,6 +4,12 @@ {esql} async query API ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-esql[ES|QL APIs]. +-- + Runs an async <>. The async query API lets you asynchronously execute a query request, diff --git a/docs/reference/esql/esql-async-query-delete-api.asciidoc b/docs/reference/esql/esql-async-query-delete-api.asciidoc index 5cad566f7f9c0..421c59191f3bd 100644 --- a/docs/reference/esql/esql-async-query-delete-api.asciidoc +++ b/docs/reference/esql/esql-async-query-delete-api.asciidoc @@ -4,6 +4,12 @@ {esql} async query delete API ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-esql[ES|QL APIs]. +-- + The <> async query delete API is used to manually delete an async query by ID. If the query is still running, the query will be cancelled. Otherwise, the stored results are deleted. diff --git a/docs/reference/esql/esql-async-query-get-api.asciidoc b/docs/reference/esql/esql-async-query-get-api.asciidoc index ec68313b2c490..693e96861adb9 100644 --- a/docs/reference/esql/esql-async-query-get-api.asciidoc +++ b/docs/reference/esql/esql-async-query-get-api.asciidoc @@ -4,6 +4,12 @@ {esql} async query get API ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-esql[ES|QL APIs]. +-- + Returns the current status and available results for an <> or a stored results. @@ -39,6 +45,10 @@ parameter is `true`. [[esql-async-query-get-api-query-params]] ==== {api-query-parms-title} +The API accepts the same parameters as the synchronous +<>, along with the following +parameters: + `wait_for_completion_timeout`:: (Optional, <>) Timeout duration to wait for the request to finish. Defaults to no timeout, diff --git a/docs/reference/esql/esql-commands.asciidoc b/docs/reference/esql/esql-commands.asciidoc index 235113ac1394a..33e748d7eb7c1 100644 --- a/docs/reference/esql/esql-commands.asciidoc +++ b/docs/reference/esql/esql-commands.asciidoc @@ -38,12 +38,12 @@ image::images/esql/processing-command.svg[A processing command changing an input * <> * <> ifeval::["{release-state}"=="unreleased"] -* experimental:[] <> +//* experimental:[] <> endif::[] * <> * <> ifeval::["{release-state}"=="unreleased"] -* experimental:[] <> +//* experimental:[] <> endif::[] * experimental:[] <> * <> @@ -63,12 +63,12 @@ include::processing-commands/enrich.asciidoc[] include::processing-commands/eval.asciidoc[] include::processing-commands/grok.asciidoc[] ifeval::["{release-state}"=="unreleased"] -include::processing-commands/inlinestats.asciidoc[] +//include::processing-commands/inlinestats.asciidoc[] endif::[] include::processing-commands/keep.asciidoc[] include::processing-commands/limit.asciidoc[] ifeval::["{release-state}"=="unreleased"] -include::processing-commands/lookup.asciidoc[] +//include::processing-commands/lookup.asciidoc[] endif::[] include::processing-commands/mv_expand.asciidoc[] include::processing-commands/rename.asciidoc[] diff --git a/docs/reference/esql/esql-enrich-data.asciidoc b/docs/reference/esql/esql-enrich-data.asciidoc index c48118d1c367a..ad34e29f1a55b 100644 --- a/docs/reference/esql/esql-enrich-data.asciidoc +++ b/docs/reference/esql/esql-enrich-data.asciidoc @@ -138,8 +138,33 @@ include::{es-ref-dir}/ingest/apis/enrich/execute-enrich-policy.asciidoc[tag=upda include::../ingest/enrich.asciidoc[tag=update-enrich-policy] -==== Limitations +==== Enrich Policy Types and Limitations +The {esql} `ENRICH` command supports all three enrich policy types: + +`geo_match`:: +Matches enrich data to incoming documents based on a <>. +For an example, see <>. + +`match`:: +Matches enrich data to incoming documents based on a <>. +For an example, see <>. + +`range`:: +Matches a number, date, or IP address in incoming documents to a range in the +enrich index based on a <>. For an example, +see <>. + // tag::limitations[] -The {esql} `ENRICH` command only supports enrich policies of type `match`. -Furthermore, `ENRICH` only supports enriching on a column of type `keyword`. +While all three enrich policy types are supported, there are some limitations to be aware of: + +* The `geo_match` enrich policy type only supports the `intersects` spatial relation. +* It is required that the `match_field` in the `ENRICH` command is of the correct type. +For example, if the enrich policy is of type `geo_match`, the `match_field` in the `ENRICH` +command must be of type `geo_point` or `geo_shape`. +Likewise, a `range` enrich policy requires a `match_field` of type `integer`, `long`, `date`, or `ip`, +depending on the type of the range field in the original enrich index. +* However, this constraint is relaxed for `range` policies when the `match_field` is of type `KEYWORD`. +In this case the field values will be parsed during query execution, row by row. +If any value fails to parse, the output values for that row will be set to `null`, +an appropriate warning will be produced and the query will continue to execute. // end::limitations[] diff --git a/docs/reference/esql/esql-functions-operators.asciidoc b/docs/reference/esql/esql-functions-operators.asciidoc index 61f2ac6566d27..9743fd59f5e98 100644 --- a/docs/reference/esql/esql-functions-operators.asciidoc +++ b/docs/reference/esql/esql-functions-operators.asciidoc @@ -48,6 +48,12 @@ include::functions/ip-functions.asciidoc[tag=ip_list] include::functions/math-functions.asciidoc[tag=math_list] ==== +.*Search functions* +[%collapsible] +==== +include::functions/search-functions.asciidoc[tag=search_list] +==== + .*Spatial functions* [%collapsible] ==== @@ -89,6 +95,7 @@ include::functions/conditional-functions-and-expressions.asciidoc[] include::functions/date-time-functions.asciidoc[] include::functions/ip-functions.asciidoc[] include::functions/math-functions.asciidoc[] +include::functions/search-functions.asciidoc[] include::functions/spatial-functions.asciidoc[] include::functions/string-functions.asciidoc[] include::functions/type-conversion-functions.asciidoc[] diff --git a/docs/reference/esql/esql-kibana.asciidoc b/docs/reference/esql/esql-kibana.asciidoc index 5da8b9323cc20..87dd4d87fa8e3 100644 --- a/docs/reference/esql/esql-kibana.asciidoc +++ b/docs/reference/esql/esql-kibana.asciidoc @@ -9,9 +9,9 @@ You can use {esql} in {kib} to query and aggregate your data, create visualizations, and set up alerts. This guide shows you how to use {esql} in Kibana. To follow along with the -queries, load the "Sample web logs" sample data set by clicking *Try sample -data* from the {kib} Home, selecting *Other sample data sets*, and clicking *Add -data* on the *Sample web logs* card. +queries, load the "Sample web logs" sample data set by selecting **Sample Data** +from the **Integrations** page in {kib}, selecting *Other sample data sets*, +and clicking *Add data* on the *Sample web logs* card. [discrete] [[esql-kibana-enable]] @@ -30,9 +30,7 @@ However, users will be able to access existing {esql} artifacts like saved searc // tag::esql-mode[] To get started with {esql} in Discover, open the main menu and select -*Discover*. Next, from the Data views menu, select *Language: ES|QL*. - -image::images/esql/esql-data-view-menu.png[align="center",width=33%] +*Discover*. Next, select *Try ES|QL* from the application menu bar. // end::esql-mode[] [discrete] @@ -54,8 +52,9 @@ A source command can be followed by one or more <>. In this query, the processing command is <>. `LIMIT` limits the number of rows that are retrieved. -TIP: Click the help icon (image:images/esql/esql-icon-help.svg[Static,20]) to open the -in-product reference documentation for all commands and functions. +TIP: Click the **ES|QL help** button to open the +in-product reference documentation for all commands and functions or to get +recommended queries that will help you get started. // tag::autocomplete[] To make it easier to write queries, auto-complete offers suggestions with @@ -76,7 +75,7 @@ FROM kibana_sample_data_logs | LIMIT 10 ==== [discrete] -==== Expand the query bar +==== Make your query readable For readability, you can put each processing command on a new line. The following query is identical to the previous one: @@ -87,15 +86,12 @@ FROM kibana_sample_data_logs | LIMIT 10 ---- -// tag::compact[] -To make it easier to write multi-line queries, click the double-headed arrow -button (image:images/esql/esql-icon-expand-query-bar.svg[]) to expand the query -bar: +You can do that using the **Add line breaks on pipes** button from the query editor's footer. -image::images/esql/esql-expanded-query-bar.png[align="center"] +image::https://images.contentstack.io/v3/assets/bltefdd0b53724fa2ce/bltd5554518309e10f6/672d153cfeb8f9d479ebcc6e/esql-line-breakdown.gif[Automatic line breaks for ES|QL queries] -To return to a compact query bar, click the minimize editor button -(image:images/esql/esql-icon-minimize-query-bar.svg[]). +// tag::compact[] +You can adjust the editor's height by dragging its bottom border to your liking. // end::compact[] [discrete] @@ -110,14 +106,27 @@ detailed warning, expand the query bar, and click *warnings*. ==== Query history You can reuse your recent {esql} queries in the query bar. -In the query bar click *Show recent queries*: - -image::images/esql/esql-discover-show-recent-query.png[align="center",size="50%"] +In the query bar, click *Show recent queries*. You can then scroll through your recent queries: image::images/esql/esql-discover-query-history.png[align="center",size="50%"] +[discrete] +[[esql-kibana-starred-queries]] +==== Starred queries + +From the query history, you can mark some queries as favorite to find and access them faster later. + +In the query bar, click *Show recent queries*. + +From the **Recent** tab, you can star any queries you want. + +In the **Starred** tab, find all the queries you have previously starred. + +image::images/esql/esql-discover-query-starred.png[align="center",size="50%"] + + [discrete] [[esql-kibana-results-table]] === The results table @@ -171,14 +180,44 @@ FROM kibana_sample_data_logs [[esql-kibana-time-filter]] === Time filtering -To display data within a specified time range, use the -{kibana-ref}/set-time-filter.html[time filter]. The time filter is only enabled -when the indices you're querying have a field called `@timestamp`. +To display data within a specified time range, you can use the standard time filter, +custom time parameters, or a WHERE command. -If your indices do not have a timestamp field called `@timestamp`, you can limit -the time range using the <> command and the <> function. +[discrete] +==== Standard time filter +The standard {kibana-ref}/set-time-filter.html[time filter] is enabled +when the indices you're querying have a field named `@timestamp`. + +[discrete] +==== Custom time parameters +If your indices do not have a field named `@timestamp`, you can use +the `?_tstart` and `?_tend` parameters to specify a time range. These parameters +work with any timestamp field and automatically sync with the {kibana-ref}/set-time-filter.html[time filter]. + +[source,esql] +---- +FROM my_index +| WHERE custom_timestamp >= ?_tstart AND custom_timestamp < ?_tend +---- + +You can also use the `?_tstart` and `?_tend` parameters with the <> function +to create auto-incrementing time buckets in {esql} <>. +For example: + +[source,esql] +---- +FROM kibana_sample_data_logs +| STATS average_bytes = AVG(bytes) BY BUCKET(@timestamp, 50, ?_tstart, ?_tend) +---- + +This example uses `50` buckets, which is the maximum number of buckets. + +[discrete] +==== WHERE command +You can also limit the time range using the <> command and the <> function. For example, if the timestamp field is called `timestamp`, to query the last 15 minutes of data: + [source,esql] ---- FROM kibana_sample_data_logs @@ -190,8 +229,9 @@ FROM kibana_sample_data_logs === Analyze and visualize data Between the query bar and the results table, Discover shows a date histogram -visualization. If the indices you're querying do not contain a `@timestamp` -field, the histogram is not shown. +visualization. By default, if the indices you're querying do not contain a `@timestamp` +field, the histogram is not shown. But you can use a custom time field with the `?_tstart` +and `?_tend` parameters to enable it. The visualization adapts to the query. A query's nature determines the type of visualization. For example, this query aggregates the total number of bytes per @@ -220,7 +260,7 @@ save button (image:images/esql/esql-icon-save-visualization.svg[]). Once saved to a dashboard, you'll be taken to the Dashboards page. You can continue to make changes to the visualization. Click the options button in the top-right (image:images/esql/esql-icon-options.svg[]) and -select *Edit ESQL visualization* to open the in-line editor: +select *Edit ES|QL visualization* to open the in-line editor: image::images/esql/esql-kibana-edit-on-dashboard.png[align="center",width=66%] diff --git a/docs/reference/esql/esql-language.asciidoc b/docs/reference/esql/esql-language.asciidoc index a7c0e5e01a867..151ca803bf2eb 100644 --- a/docs/reference/esql/esql-language.asciidoc +++ b/docs/reference/esql/esql-language.asciidoc @@ -14,6 +14,7 @@ Detailed reference documentation for the {esql} language: * <> * <> * <> +* <> include::esql-syntax.asciidoc[] include::esql-commands.asciidoc[] @@ -23,3 +24,4 @@ include::multivalued-fields.asciidoc[] include::esql-process-data-with-dissect-grok.asciidoc[] include::esql-enrich-data.asciidoc[] include::implicit-casting.asciidoc[] +include::time-spans.asciidoc[] diff --git a/docs/reference/esql/esql-limitations.asciidoc b/docs/reference/esql/esql-limitations.asciidoc index 8accc7550edbb..b8522409687a9 100644 --- a/docs/reference/esql/esql-limitations.asciidoc +++ b/docs/reference/esql/esql-limitations.asciidoc @@ -25,6 +25,9 @@ include::processing-commands/limit.asciidoc[tag=limitation] * `alias` * `boolean` * `date` +* `date_nanos` (Tech Preview) +** The following functions don't yet support date nanos: `bucket`, `date_format`, `date_parse`, `date_diff`, `date_extract` +** You can use `to_datetime` to cast to millisecond dates to use unsupported functions * `double` (`float`, `half_float`, `scaled_float` are represented as `double`) * `ip` * `keyword` family including `keyword`, `constant_keyword`, and `wildcard` @@ -50,7 +53,6 @@ include::processing-commands/limit.asciidoc[tag=limitation] ** `position` ** `aggregate_metric_double` * Date/time -** `date_nanos` ** `date_range` * Other types ** `binary` @@ -102,11 +104,19 @@ is currently experimental. [discrete] [[esql-limitations-full-text-search]] -=== Full-text search is not supported +=== Full-text search + +experimental:[] {esql}'s support for <> +is currently in Technical Preview. One limitation of full-text search is that +it is necessary to use the search function, like <>, in a <> command +directly after the <> source command, or close enough to it. +Otherwise, the query will fail with a validation error. +Another limitation is that any <> command containing a full-text search function +cannot also use disjunctions (`OR`). Because of <>, -full-text search is not yet supported. Queries on `text` fields are like queries -on `keyword` fields: they are case-sensitive and need to match the full string. +queries on `text` fields are like queries on `keyword` fields: they are +case-sensitive and need to match the full string. For example, after indexing a field of type `text` with the value `Elasticsearch query language`, the following `WHERE` clause does not match because the `LIKE` diff --git a/docs/reference/esql/esql-process-data-with-dissect-grok.asciidoc b/docs/reference/esql/esql-process-data-with-dissect-grok.asciidoc index 87748fee4f202..e626e058a4e56 100644 --- a/docs/reference/esql/esql-process-data-with-dissect-grok.asciidoc +++ b/docs/reference/esql/esql-process-data-with-dissect-grok.asciidoc @@ -40,7 +40,7 @@ delimiter-based pattern, and extracts the specified keys as columns. For example, the following pattern: [source,txt] ---- -%{clientip} [%{@timestamp}] %{status} +%{clientip} [%{@timestamp}] %{status} ---- matches a log line of this format: @@ -76,8 +76,8 @@ ignore certain fields, append fields, skip over padding, etc. ===== Terminology dissect pattern:: -the set of fields and delimiters describing the textual -format. Also known as a dissection. +the set of fields and delimiters describing the textual +format. Also known as a dissection. The dissection is described using a set of `%{}` sections: `%{a} - %{b} - %{c}` @@ -91,14 +91,14 @@ Any set of characters other than `%{`, `'not }'`, or `}` is a delimiter. key:: + -- -the text between the `%{` and `}`, exclusive of the `?`, `+`, `&` prefixes -and the ordinal suffix. +the text between the `%{` and `}`, exclusive of the `?`, `+`, `&` prefixes +and the ordinal suffix. Examples: -* `%{?aaa}` - the key is `aaa` -* `%{+bbb/3}` - the key is `bbb` -* `%{&ccc}` - the key is `ccc` +* `%{?aaa}` - the key is `aaa` +* `%{+bbb/3}` - the key is `bbb` +* `%{&ccc}` - the key is `ccc` -- [[esql-dissect-examples]] @@ -218,7 +218,7 @@ Putting it together as an {esql} query: [source.merge.styled,esql] ---- -include::{esql-specs}/docs.csv-spec[tag=grokWithEscape] +include::{esql-specs}/docs.csv-spec[tag=grokWithEscapeTripleQuotes] ---- `GROK` adds the following columns to the input table: @@ -239,15 +239,24 @@ with a `\`. For example, in the earlier pattern: %{IP:ip} \[%{TIMESTAMP_ISO8601:@timestamp}\] %{GREEDYDATA:status} ---- -In {esql} queries, the backslash character itself is a special character that +In {esql} queries, when using single quotes for strings, the backslash character itself is a special character that needs to be escaped with another `\`. For this example, the corresponding {esql} query becomes: [source.merge.styled,esql] ---- include::{esql-specs}/docs.csv-spec[tag=grokWithEscape] ---- + +For this reason, in general it is more convenient to use triple quotes `"""` for GROK patterns, +that do not require escaping for backslash. + +[source.merge.styled,esql] +---- +include::{esql-specs}/docs.csv-spec[tag=grokWithEscapeTripleQuotes] +---- ==== + [[esql-grok-patterns]] ===== Grok patterns @@ -318,4 +327,4 @@ as the `GROK` command. The `GROK` command does not support configuring <>, or <>. The `GROK` command is not subject to <>. -// end::grok-limitations[] \ No newline at end of file +// end::grok-limitations[] diff --git a/docs/reference/esql/esql-query-api.asciidoc b/docs/reference/esql/esql-query-api.asciidoc index c8c735b73d2a4..e5b29a5ca25f1 100644 --- a/docs/reference/esql/esql-query-api.asciidoc +++ b/docs/reference/esql/esql-query-api.asciidoc @@ -4,6 +4,12 @@ {esql} query API ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-esql[ES|QL APIs]. +-- + Returns search results for an <> query. [source,console] @@ -67,6 +73,11 @@ precedence. `false`. The API only supports this parameter for CBOR, JSON, SMILE, and YAML responses. See <>. +`include_ccs_metadata`:: +(Optional, boolean) If `true`, cross-cluster searches will include metadata about the query +on each cluster. Defaults to `false`. The API only supports this parameter for CBOR, JSON, SMILE, +and YAML responses. See <>. + `locale`:: (Optional, string) Returns results (especially dates) formatted per the conventions of the locale. For syntax, refer to <>. @@ -79,13 +90,16 @@ For syntax, refer to <>. (Optional, boolean) If provided and `true` the response will include an extra `profile` object with information about how the query was executed. It provides insight into the performance of each part of the query. This is for human debugging as the object's format might change at any time. +Think of this like https://www.postgresql.org/docs/current/sql-explain.html[EXPLAIN ANALYZE] or +https://en.wikipedia.org/wiki/Query_plan[EXPLAIN PLAN]. `query`:: (Required, string) {esql} query to run. For syntax, refer to <>. + ifeval::["{release-state}"=="unreleased"] -`table`:: -(Optional, object) Named "table" parameters that can be referenced by the <> command. +//`table`:: +//(Optional, object) Named "table" parameters that can be referenced by the <> command. endif::[] [discrete] @@ -106,7 +120,16 @@ returned if `drop_null_columns` is sent with the request. (array of arrays) Values for the search results. +`_clusters`:: +(object) +Metadata about clusters involved in the execution of a cross-cluster query. Only returned (1) for +cross-cluster searches and (2) when `include_ccs_metadata` is sent in the body and set to `true` +and (3) when `format` of the response is set to JSON (the default), CBOR, SMILE, or YAML. +See <> for more information. + `profile`:: (object) Profile describing the execution of the query. Only returned if `profile` was sent in the body. -The object itself is for human debugging and can change at any time. +The object itself is for human debugging and can change at any time. Think of this like +https://www.postgresql.org/docs/current/sql-explain.html[EXPLAIN ANALYZE] or +https://en.wikipedia.org/wiki/Query_plan[EXPLAIN PLAN]. diff --git a/docs/reference/esql/esql-rest.asciidoc b/docs/reference/esql/esql-rest.asciidoc index 2c8c5e81e273d..1cd808b2aef44 100644 --- a/docs/reference/esql/esql-rest.asciidoc +++ b/docs/reference/esql/esql-rest.asciidoc @@ -192,6 +192,7 @@ Which returns: [source,console-result] ---- { + "took": 28, "columns": [ {"name": "author", "type": "text"}, {"name": "name", "type": "text"}, @@ -206,6 +207,7 @@ Which returns: ] } ---- +// TESTRESPONSE[s/"took": 28/"took": "$body.took"/] [discrete] [[esql-locale-param]] @@ -233,6 +235,7 @@ POST /_query } ---- // TEST[setup:library] +// TEST[skip:This can output a warning, and asciidoc doesn't support allowed_warnings] [discrete] [[esql-rest-params]] @@ -384,12 +387,13 @@ GET /_query/async/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUT // TEST[skip: no access to query ID - may return response values] If the response's `is_running` value is `false`, the query has finished -and the results are returned. +and the results are returned, along with the `took` time for the query. [source,console-result] ---- { "is_running": false, + "took": 48, "columns": ... } ---- diff --git a/docs/reference/esql/esql-syntax.asciidoc b/docs/reference/esql/esql-syntax.asciidoc index c7f741d064310..ba1c4ca820381 100644 --- a/docs/reference/esql/esql-syntax.asciidoc +++ b/docs/reference/esql/esql-syntax.asciidoc @@ -157,21 +157,15 @@ FROM employees ==== Timespan literals Datetime intervals and timespans can be expressed using timespan literals. -Timespan literals are a combination of a number and a qualifier. These -qualifiers are supported: - -* `millisecond`/`milliseconds`/`ms` -* `second`/`seconds`/`sec`/`s` -* `minute`/`minutes`/`min` -* `hour`/`hours`/`h` -* `day`/`days`/`d` -* `week`/`weeks`/`w` -* `month`/`months`/`mo` -* `quarter`/`quarters`/`q` -* `year`/`years`/`yr`/`y` +Timespan literals are a combination of a number and a temporal unit. The +supported temporal units are listed in <>. +More examples of the usages of time spans can be found in +<>. + Timespan literals are not whitespace sensitive. These expressions are all valid: * `1day` * `1 day` * `1 day` + diff --git a/docs/reference/esql/functions/aggregation-functions.asciidoc b/docs/reference/esql/functions/aggregation-functions.asciidoc index 7cdc42ea6cbf9..24b42a6efd831 100644 --- a/docs/reference/esql/functions/aggregation-functions.asciidoc +++ b/docs/reference/esql/functions/aggregation-functions.asciidoc @@ -17,10 +17,12 @@ The <> command supports these aggregate functions: * <> * <> * experimental:[] <> +* experimental:[] <> +* <> * <> * <> * <> -* experimental:[] <> +* <> // end::agg_list[] include::layout/avg.asciidoc[] @@ -32,6 +34,8 @@ include::layout/median_absolute_deviation.asciidoc[] include::layout/min.asciidoc[] include::layout/percentile.asciidoc[] include::layout/st_centroid_agg.asciidoc[] +include::layout/st_extent_agg.asciidoc[] +include::layout/std_dev.asciidoc[] include::layout/sum.asciidoc[] include::layout/top.asciidoc[] include::layout/values.asciidoc[] diff --git a/docs/reference/esql/functions/binary.asciidoc b/docs/reference/esql/functions/binary.asciidoc index 72d466ae83d11..59bdadecc4923 100644 --- a/docs/reference/esql/functions/binary.asciidoc +++ b/docs/reference/esql/functions/binary.asciidoc @@ -87,6 +87,7 @@ Supported types: include::types/greater_than_or_equal.asciidoc[] +[[esql-add]] ==== Add `+` [.text-center] image::esql/functions/signature/add.svg[Embedded,opts=inline] @@ -98,6 +99,7 @@ Supported types: include::types/add.asciidoc[] +[[esql-subtract]] ==== Subtract `-` [.text-center] image::esql/functions/signature/sub.svg[Embedded,opts=inline] diff --git a/docs/reference/esql/functions/description/bit_length.asciidoc b/docs/reference/esql/functions/description/bit_length.asciidoc new file mode 100644 index 0000000000000..3a3dd80d2bb0f --- /dev/null +++ b/docs/reference/esql/functions/description/bit_length.asciidoc @@ -0,0 +1,7 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Returns the bit length of a string. + +NOTE: All strings are in UTF-8, so a single character can use multiple bytes. diff --git a/docs/reference/esql/functions/description/byte_length.asciidoc b/docs/reference/esql/functions/description/byte_length.asciidoc new file mode 100644 index 0000000000000..c2150806e09ac --- /dev/null +++ b/docs/reference/esql/functions/description/byte_length.asciidoc @@ -0,0 +1,7 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Returns the byte length of a string. + +NOTE: All strings are in UTF-8, so a single character can use multiple bytes. diff --git a/docs/reference/esql/functions/description/categorize.asciidoc b/docs/reference/esql/functions/description/categorize.asciidoc new file mode 100644 index 0000000000000..c956066ad53f3 --- /dev/null +++ b/docs/reference/esql/functions/description/categorize.asciidoc @@ -0,0 +1,11 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Groups text messages into categories of similarly formatted text values. + +`CATEGORIZE` has the following limitations: + +* can't be used within other expressions +* can't be used with multiple groupings +* can't be used or referenced within aggregate functions diff --git a/docs/reference/esql/functions/description/hash.asciidoc b/docs/reference/esql/functions/description/hash.asciidoc new file mode 100644 index 0000000000000..e074915c5132a --- /dev/null +++ b/docs/reference/esql/functions/description/hash.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Computes the hash of the input using various algorithms such as MD5, SHA, SHA-224, SHA-256, SHA-384, SHA-512. diff --git a/docs/reference/esql/functions/description/hypot.asciidoc b/docs/reference/esql/functions/description/hypot.asciidoc new file mode 100644 index 0000000000000..5162f0d9ef98f --- /dev/null +++ b/docs/reference/esql/functions/description/hypot.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Returns the hypotenuse of two numbers. The input can be any numeric values, the return value is always a double. Hypotenuses of infinities are null. diff --git a/docs/reference/esql/functions/description/kql.asciidoc b/docs/reference/esql/functions/description/kql.asciidoc new file mode 100644 index 0000000000000..e1fe411e6689c --- /dev/null +++ b/docs/reference/esql/functions/description/kql.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Performs a KQL query. Returns true if the provided KQL query string matches the row. diff --git a/docs/reference/esql/functions/description/length.asciidoc b/docs/reference/esql/functions/description/length.asciidoc index bf976e3d6e507..91525fda0c086 100644 --- a/docs/reference/esql/functions/description/length.asciidoc +++ b/docs/reference/esql/functions/description/length.asciidoc @@ -3,3 +3,5 @@ *Description* Returns the character length of a string. + +NOTE: All strings are in UTF-8, so a single character can use multiple bytes. diff --git a/docs/reference/esql/functions/description/match.asciidoc b/docs/reference/esql/functions/description/match.asciidoc new file mode 100644 index 0000000000000..25f0571878d47 --- /dev/null +++ b/docs/reference/esql/functions/description/match.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Performs a <> on the specified field. Returns true if the provided query matches the row. diff --git a/docs/reference/esql/functions/description/mv_first.asciidoc b/docs/reference/esql/functions/description/mv_first.asciidoc index 99223e2c02d9f..13c433ce209d0 100644 --- a/docs/reference/esql/functions/description/mv_first.asciidoc +++ b/docs/reference/esql/functions/description/mv_first.asciidoc @@ -2,4 +2,10 @@ *Description* -Converts a multivalued expression into a single valued column containing the first value. This is most useful when reading from a function that emits multivalued columns in a known order like <>. The order that <> are read from underlying storage is not guaranteed. It is *frequently* ascending, but don't rely on that. If you need the minimum value use <> instead of `MV_FIRST`. `MV_MIN` has optimizations for sorted values so there isn't a performance benefit to `MV_FIRST`. +Converts a multivalued expression into a single valued column containing the first value. This is most useful when reading from a function that emits multivalued columns in a known order like <>. + +The order that <> are read from +underlying storage is not guaranteed. It is *frequently* ascending, but don't +rely on that. If you need the minimum value use <> instead of +`MV_FIRST`. `MV_MIN` has optimizations for sorted values so there isn't a +performance benefit to `MV_FIRST`. diff --git a/docs/reference/esql/functions/description/mv_last.asciidoc b/docs/reference/esql/functions/description/mv_last.asciidoc index 4b4b4336588d1..beba7b5a402c9 100644 --- a/docs/reference/esql/functions/description/mv_last.asciidoc +++ b/docs/reference/esql/functions/description/mv_last.asciidoc @@ -2,4 +2,10 @@ *Description* -Converts a multivalue expression into a single valued column containing the last value. This is most useful when reading from a function that emits multivalued columns in a known order like <>. The order that <> are read from underlying storage is not guaranteed. It is *frequently* ascending, but don't rely on that. If you need the maximum value use <> instead of `MV_LAST`. `MV_MAX` has optimizations for sorted values so there isn't a performance benefit to `MV_LAST`. +Converts a multivalue expression into a single valued column containing the last value. This is most useful when reading from a function that emits multivalued columns in a known order like <>. + +The order that <> are read from +underlying storage is not guaranteed. It is *frequently* ascending, but don't +rely on that. If you need the maximum value use <> instead of +`MV_LAST`. `MV_MAX` has optimizations for sorted values so there isn't a +performance benefit to `MV_LAST`. diff --git a/docs/reference/esql/functions/description/mv_slice.asciidoc b/docs/reference/esql/functions/description/mv_slice.asciidoc index 24d3183b6f40e..98438ae097fe7 100644 --- a/docs/reference/esql/functions/description/mv_slice.asciidoc +++ b/docs/reference/esql/functions/description/mv_slice.asciidoc @@ -2,4 +2,8 @@ *Description* -Returns a subset of the multivalued field using the start and end index values. +Returns a subset of the multivalued field using the start and end index values. This is most useful when reading from a function that emits multivalued columns in a known order like <> or <>. + +The order that <> are read from +underlying storage is not guaranteed. It is *frequently* ascending, but don't +rely on that. diff --git a/docs/reference/esql/functions/description/qstr.asciidoc b/docs/reference/esql/functions/description/qstr.asciidoc new file mode 100644 index 0000000000000..d9dbe364f607a --- /dev/null +++ b/docs/reference/esql/functions/description/qstr.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Performs a <>. Returns true if the provided query string matches the row. diff --git a/docs/reference/esql/functions/description/reverse.asciidoc b/docs/reference/esql/functions/description/reverse.asciidoc new file mode 100644 index 0000000000000..72ea3a96f97e7 --- /dev/null +++ b/docs/reference/esql/functions/description/reverse.asciidoc @@ -0,0 +1,9 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Returns a new string representing the input string in reverse order. + +NOTE: If Elasticsearch is running with a JDK version less than 20 then this will not properly reverse Grapheme Clusters. +Elastic Cloud and the JDK bundled with Elasticsearch all use newer JDKs. But if you've explicitly shifted to an older jdk +then you'll see things like "👍🏽😊" be reversed to "🏽👍😊" instead of the correct "😊👍🏽". diff --git a/docs/reference/esql/functions/description/st_envelope.asciidoc b/docs/reference/esql/functions/description/st_envelope.asciidoc new file mode 100644 index 0000000000000..6b7cf8d97538a --- /dev/null +++ b/docs/reference/esql/functions/description/st_envelope.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Determines the minimum bounding box of the supplied geometry. diff --git a/docs/reference/esql/functions/description/st_extent_agg.asciidoc b/docs/reference/esql/functions/description/st_extent_agg.asciidoc new file mode 100644 index 0000000000000..a9e1acfb0e6fb --- /dev/null +++ b/docs/reference/esql/functions/description/st_extent_agg.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Calculate the spatial extent over a field with geometry type. Returns a bounding box for all values of the field. diff --git a/docs/reference/esql/functions/description/st_xmax.asciidoc b/docs/reference/esql/functions/description/st_xmax.asciidoc new file mode 100644 index 0000000000000..f33ec590bf2d4 --- /dev/null +++ b/docs/reference/esql/functions/description/st_xmax.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Extracts the maximum value of the `x` coordinates from the supplied geometry. If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the maximum `longitude` value. diff --git a/docs/reference/esql/functions/description/st_xmin.asciidoc b/docs/reference/esql/functions/description/st_xmin.asciidoc new file mode 100644 index 0000000000000..b06cbfacde7bf --- /dev/null +++ b/docs/reference/esql/functions/description/st_xmin.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Extracts the minimum value of the `x` coordinates from the supplied geometry. If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the minimum `longitude` value. diff --git a/docs/reference/esql/functions/description/st_ymax.asciidoc b/docs/reference/esql/functions/description/st_ymax.asciidoc new file mode 100644 index 0000000000000..f9475dd967562 --- /dev/null +++ b/docs/reference/esql/functions/description/st_ymax.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Extracts the maximum value of the `y` coordinates from the supplied geometry. If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the maximum `latitude` value. diff --git a/docs/reference/esql/functions/description/st_ymin.asciidoc b/docs/reference/esql/functions/description/st_ymin.asciidoc new file mode 100644 index 0000000000000..7228c63a16030 --- /dev/null +++ b/docs/reference/esql/functions/description/st_ymin.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Extracts the minimum value of the `y` coordinates from the supplied geometry. If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the minimum `latitude` value. diff --git a/docs/reference/esql/functions/description/std_dev.asciidoc b/docs/reference/esql/functions/description/std_dev.asciidoc new file mode 100644 index 0000000000000..b78ddd7dbba13 --- /dev/null +++ b/docs/reference/esql/functions/description/std_dev.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +The standard deviation of a numeric field. diff --git a/docs/reference/esql/functions/description/term.asciidoc b/docs/reference/esql/functions/description/term.asciidoc new file mode 100644 index 0000000000000..c43aeb25a0ef7 --- /dev/null +++ b/docs/reference/esql/functions/description/term.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Performs a Term query on the specified field. Returns true if the provided term matches the row. diff --git a/docs/reference/esql/functions/examples/bit_length.asciidoc b/docs/reference/esql/functions/examples/bit_length.asciidoc new file mode 100644 index 0000000000000..a99f6f664e79e --- /dev/null +++ b/docs/reference/esql/functions/examples/bit_length.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/docs.csv-spec[tag=bitLength] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/docs.csv-spec[tag=bitLength-result] +|=== + diff --git a/docs/reference/esql/functions/examples/bucket.asciidoc b/docs/reference/esql/functions/examples/bucket.asciidoc index 4afea30660339..264efc191748f 100644 --- a/docs/reference/esql/functions/examples/bucket.asciidoc +++ b/docs/reference/esql/functions/examples/bucket.asciidoc @@ -116,4 +116,18 @@ include::{esql-specs}/bucket.csv-spec[tag=reuseGroupingFunctionWithExpression] |=== include::{esql-specs}/bucket.csv-spec[tag=reuseGroupingFunctionWithExpression-result] |=== +Sometimes you need to change the start value of each bucket by a given duration (similar to date histogram +aggregation's <> parameter). To do so, you will need to +take into account how the language handles expressions within the `STATS` command: if these contain functions or +arithmetic operators, a virtual `EVAL` is inserted before and/or after the `STATS` command. Consequently, a double +compensation is needed to adjust the bucketed date value before the aggregation and then again after. For instance, +inserting a negative offset of `1 hour` to buckets of `1 year` looks like this: +[source.merge.styled,esql] +---- +include::{esql-specs}/bucket.csv-spec[tag=bucketWithOffset] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/bucket.csv-spec[tag=bucketWithOffset-result] +|=== diff --git a/docs/reference/esql/functions/examples/byte_length.asciidoc b/docs/reference/esql/functions/examples/byte_length.asciidoc new file mode 100644 index 0000000000000..d6b557fcd2e76 --- /dev/null +++ b/docs/reference/esql/functions/examples/byte_length.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/eval.csv-spec[tag=byteLength] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/eval.csv-spec[tag=byteLength-result] +|=== + diff --git a/docs/reference/esql/functions/examples/categorize.asciidoc b/docs/reference/esql/functions/examples/categorize.asciidoc new file mode 100644 index 0000000000000..4167be6910c89 --- /dev/null +++ b/docs/reference/esql/functions/examples/categorize.asciidoc @@ -0,0 +1,14 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +This example categorizes server logs messages into categories and aggregates their counts. +[source.merge.styled,esql] +---- +include::{esql-specs}/docs.csv-spec[tag=docsCategorize] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/docs.csv-spec[tag=docsCategorize-result] +|=== + diff --git a/docs/reference/esql/functions/examples/count.asciidoc b/docs/reference/esql/functions/examples/count.asciidoc index fb696b51e054c..33ed054d3d1e2 100644 --- a/docs/reference/esql/functions/examples/count.asciidoc +++ b/docs/reference/esql/functions/examples/count.asciidoc @@ -37,7 +37,7 @@ include::{esql-specs}/stats.csv-spec[tag=count-where] |=== include::{esql-specs}/stats.csv-spec[tag=count-where-result] |=== -To count the same stream of data based on two different expressions use the pattern `COUNT( OR NULL)` +To count the same stream of data based on two different expressions use the pattern `COUNT( OR NULL)`. This builds on the three-valued logic ({wikipedia}/Three-valued_logic[3VL]) of the language: `TRUE OR NULL` is `TRUE`, but `FALSE OR NULL` is `NULL`, plus the way COUNT handles `NULL`s: `COUNT(TRUE)` and `COUNT(FALSE)` are both 1, but `COUNT(NULL)` is 0. [source.merge.styled,esql] ---- include::{esql-specs}/stats.csv-spec[tag=count-or-null] diff --git a/docs/reference/esql/functions/examples/date_diff.asciidoc b/docs/reference/esql/functions/examples/date_diff.asciidoc index f85bdf480c1c3..f75add7b80501 100644 --- a/docs/reference/esql/functions/examples/date_diff.asciidoc +++ b/docs/reference/esql/functions/examples/date_diff.asciidoc @@ -1,6 +1,6 @@ // This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. -*Example* +*Examples* [source.merge.styled,esql] ---- @@ -10,4 +10,15 @@ include::{esql-specs}/date.csv-spec[tag=docsDateDiff] |=== include::{esql-specs}/date.csv-spec[tag=docsDateDiff-result] |=== +When subtracting in calendar units - like year, month a.s.o. - only the fully elapsed units are counted. +To avoid this and obtain also remainders, simply switch to the next smaller unit and do the date math accordingly. + +[source.merge.styled,esql] +---- +include::{esql-specs}/date.csv-spec[tag=evalDateDiffYearForDocs] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/date.csv-spec[tag=evalDateDiffYearForDocs-result] +|=== diff --git a/docs/reference/esql/functions/examples/hypot.asciidoc b/docs/reference/esql/functions/examples/hypot.asciidoc new file mode 100644 index 0000000000000..6dbcc62e8755e --- /dev/null +++ b/docs/reference/esql/functions/examples/hypot.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/math.csv-spec[tag=hypot] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/math.csv-spec[tag=hypot-result] +|=== + diff --git a/docs/reference/esql/functions/examples/kql.asciidoc b/docs/reference/esql/functions/examples/kql.asciidoc new file mode 100644 index 0000000000000..1f8518aeec394 --- /dev/null +++ b/docs/reference/esql/functions/examples/kql.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/kql-function.csv-spec[tag=kql-with-field] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/kql-function.csv-spec[tag=kql-with-field-result] +|=== + diff --git a/docs/reference/esql/functions/examples/match.asciidoc b/docs/reference/esql/functions/examples/match.asciidoc new file mode 100644 index 0000000000000..3f31d68ea9abb --- /dev/null +++ b/docs/reference/esql/functions/examples/match.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/match-function.csv-spec[tag=match-with-field] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/match-function.csv-spec[tag=match-with-field-result] +|=== + diff --git a/docs/reference/esql/functions/examples/qstr.asciidoc b/docs/reference/esql/functions/examples/qstr.asciidoc new file mode 100644 index 0000000000000..003373c84c029 --- /dev/null +++ b/docs/reference/esql/functions/examples/qstr.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/qstr-function.csv-spec[tag=qstr-with-field] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/qstr-function.csv-spec[tag=qstr-with-field-result] +|=== + diff --git a/docs/reference/esql/functions/examples/reverse.asciidoc b/docs/reference/esql/functions/examples/reverse.asciidoc new file mode 100644 index 0000000000000..67c8af077b174 --- /dev/null +++ b/docs/reference/esql/functions/examples/reverse.asciidoc @@ -0,0 +1,22 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Examples* + +[source.merge.styled,esql] +---- +include::{esql-specs}/string.csv-spec[tag=reverse] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/string.csv-spec[tag=reverse-result] +|=== +`REVERSE` works with unicode, too! It keeps unicode grapheme clusters together during reversal. +[source.merge.styled,esql] +---- +include::{esql-specs}/string.csv-spec[tag=reverseEmoji] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/string.csv-spec[tag=reverseEmoji-result] +|=== + diff --git a/docs/reference/esql/functions/examples/st_envelope.asciidoc b/docs/reference/esql/functions/examples/st_envelope.asciidoc new file mode 100644 index 0000000000000..df8c0ad5607fa --- /dev/null +++ b/docs/reference/esql/functions/examples/st_envelope.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_envelope] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_envelope-result] +|=== + diff --git a/docs/reference/esql/functions/examples/st_extent_agg.asciidoc b/docs/reference/esql/functions/examples/st_extent_agg.asciidoc new file mode 100644 index 0000000000000..179be82103641 --- /dev/null +++ b/docs/reference/esql/functions/examples/st_extent_agg.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/spatial.csv-spec[tag=st_extent_agg-airports] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/spatial.csv-spec[tag=st_extent_agg-airports-result] +|=== + diff --git a/docs/reference/esql/functions/examples/st_xmax.asciidoc b/docs/reference/esql/functions/examples/st_xmax.asciidoc new file mode 100644 index 0000000000000..5bba1761cf29c --- /dev/null +++ b/docs/reference/esql/functions/examples/st_xmax.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_x_y_min_max] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_x_y_min_max-result] +|=== + diff --git a/docs/reference/esql/functions/examples/st_xmin.asciidoc b/docs/reference/esql/functions/examples/st_xmin.asciidoc new file mode 100644 index 0000000000000..5bba1761cf29c --- /dev/null +++ b/docs/reference/esql/functions/examples/st_xmin.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_x_y_min_max] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_x_y_min_max-result] +|=== + diff --git a/docs/reference/esql/functions/examples/st_ymax.asciidoc b/docs/reference/esql/functions/examples/st_ymax.asciidoc new file mode 100644 index 0000000000000..5bba1761cf29c --- /dev/null +++ b/docs/reference/esql/functions/examples/st_ymax.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_x_y_min_max] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_x_y_min_max-result] +|=== + diff --git a/docs/reference/esql/functions/examples/st_ymin.asciidoc b/docs/reference/esql/functions/examples/st_ymin.asciidoc new file mode 100644 index 0000000000000..5bba1761cf29c --- /dev/null +++ b/docs/reference/esql/functions/examples/st_ymin.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_x_y_min_max] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/spatial_shapes.csv-spec[tag=st_x_y_min_max-result] +|=== + diff --git a/docs/reference/esql/functions/examples/std_dev.asciidoc b/docs/reference/esql/functions/examples/std_dev.asciidoc new file mode 100644 index 0000000000000..2e6dc996aae9a --- /dev/null +++ b/docs/reference/esql/functions/examples/std_dev.asciidoc @@ -0,0 +1,22 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Examples* + +[source.merge.styled,esql] +---- +include::{esql-specs}/stats.csv-spec[tag=stdev] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/stats.csv-spec[tag=stdev-result] +|=== +The expression can use inline functions. For example, to calculate the standard deviation of each employee's maximum salary changes, first use `MV_MAX` on each row, and then use `STD_DEV` on the result +[source.merge.styled,esql] +---- +include::{esql-specs}/stats.csv-spec[tag=docsStatsStdDevNestedExpression] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/stats.csv-spec[tag=docsStatsStdDevNestedExpression-result] +|=== + diff --git a/docs/reference/esql/functions/examples/term.asciidoc b/docs/reference/esql/functions/examples/term.asciidoc new file mode 100644 index 0000000000000..b9d57f366294b --- /dev/null +++ b/docs/reference/esql/functions/examples/term.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/term-function.csv-spec[tag=term-with-field] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/term-function.csv-spec[tag=term-with-field-result] +|=== + diff --git a/docs/reference/esql/functions/grouping-functions.asciidoc b/docs/reference/esql/functions/grouping-functions.asciidoc index ed0caf5ec2a4c..839320ce23392 100644 --- a/docs/reference/esql/functions/grouping-functions.asciidoc +++ b/docs/reference/esql/functions/grouping-functions.asciidoc @@ -9,6 +9,8 @@ The <> command supports these grouping functions: // tag::group_list[] * <> +* experimental:[] <> // end::group_list[] include::layout/bucket.asciidoc[] +include::layout/categorize.asciidoc[] diff --git a/docs/reference/esql/functions/kibana/definition/abs.json b/docs/reference/esql/functions/kibana/definition/abs.json index 82c3c205d7512..e875d8200715d 100644 --- a/docs/reference/esql/functions/kibana/definition/abs.json +++ b/docs/reference/esql/functions/kibana/definition/abs.json @@ -56,5 +56,7 @@ "examples" : [ "ROW number = -1.0 \n| EVAL abs_number = ABS(number)", "FROM employees\n| KEEP first_name, last_name, height\n| EVAL abs_height = ABS(0.0 - height)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/acos.json b/docs/reference/esql/functions/kibana/definition/acos.json index 6a6ab59278639..f86b1a5552a6f 100644 --- a/docs/reference/esql/functions/kibana/definition/acos.json +++ b/docs/reference/esql/functions/kibana/definition/acos.json @@ -55,5 +55,7 @@ ], "examples" : [ "ROW a=.9\n| EVAL acos=ACOS(a)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/add.json b/docs/reference/esql/functions/kibana/definition/add.json index 0932a76966560..cfb4755a93d59 100644 --- a/docs/reference/esql/functions/kibana/definition/add.json +++ b/docs/reference/esql/functions/kibana/definition/add.json @@ -40,6 +40,42 @@ "variadic" : false, "returnType" : "date" }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date_nanos", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "date_period", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date_nanos", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "time_duration", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, { "params" : [ { @@ -58,6 +94,24 @@ "variadic" : false, "returnType" : "date" }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date_period", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "date_nanos", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, { "params" : [ { @@ -256,6 +310,24 @@ "variadic" : false, "returnType" : "date" }, + { + "params" : [ + { + "name" : "lhs", + "type" : "time_duration", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "date_nanos", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, { "params" : [ { @@ -292,5 +364,7 @@ "variadic" : false, "returnType" : "unsigned_long" } - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/asin.json b/docs/reference/esql/functions/kibana/definition/asin.json index f5ebb817fff33..4063ffcba6140 100644 --- a/docs/reference/esql/functions/kibana/definition/asin.json +++ b/docs/reference/esql/functions/kibana/definition/asin.json @@ -55,5 +55,7 @@ ], "examples" : [ "ROW a=.9\n| EVAL asin=ASIN(a)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/atan.json b/docs/reference/esql/functions/kibana/definition/atan.json index 654a48b8ca76d..1b2a3a1860bc2 100644 --- a/docs/reference/esql/functions/kibana/definition/atan.json +++ b/docs/reference/esql/functions/kibana/definition/atan.json @@ -55,5 +55,7 @@ ], "examples" : [ "ROW a=12.9\n| EVAL atan=ATAN(a)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/atan2.json b/docs/reference/esql/functions/kibana/definition/atan2.json index 63940831241f7..9b67c07d8d73b 100644 --- a/docs/reference/esql/functions/kibana/definition/atan2.json +++ b/docs/reference/esql/functions/kibana/definition/atan2.json @@ -295,5 +295,7 @@ ], "examples" : [ "ROW y=12.9, x=.6\n| EVAL atan2=ATAN2(y, x)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/avg.json b/docs/reference/esql/functions/kibana/definition/avg.json index eb0be684a468e..41dad3eb31613 100644 --- a/docs/reference/esql/functions/kibana/definition/avg.json +++ b/docs/reference/esql/functions/kibana/definition/avg.json @@ -44,5 +44,7 @@ "examples" : [ "FROM employees\n| STATS AVG(height)", "FROM employees\n| STATS avg_salary_change = ROUND(AVG(MV_AVG(salary_change)), 10)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/bit_length.json b/docs/reference/esql/functions/kibana/definition/bit_length.json new file mode 100644 index 0000000000000..0c75b76cdbbfb --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/bit_length.json @@ -0,0 +1,38 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "bit_length", + "description" : "Returns the bit length of a string.", + "note" : "All strings are in UTF-8, so a single character can use multiple bytes.", + "signatures" : [ + { + "params" : [ + { + "name" : "string", + "type" : "keyword", + "optional" : false, + "description" : "String expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "integer" + }, + { + "params" : [ + { + "name" : "string", + "type" : "text", + "optional" : false, + "description" : "String expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "integer" + } + ], + "examples" : [ + "FROM airports\n| WHERE country == \"India\"\n| KEEP city\n| EVAL fn_length = LENGTH(city), fn_bit_length = BIT_LENGTH(city)" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/bucket.json b/docs/reference/esql/functions/kibana/definition/bucket.json index 94214a3a4f047..3d96de05c8407 100644 --- a/docs/reference/esql/functions/kibana/definition/bucket.json +++ b/docs/reference/esql/functions/kibana/definition/bucket.json @@ -310,6 +310,312 @@ "variadic" : false, "returnType" : "date" }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "date_period", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + }, + { + "name" : "from", + "type" : "date", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "date", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + }, + { + "name" : "from", + "type" : "date", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "keyword", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + }, + { + "name" : "from", + "type" : "date", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "text", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + }, + { + "name" : "from", + "type" : "keyword", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "date", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + }, + { + "name" : "from", + "type" : "keyword", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "keyword", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + }, + { + "name" : "from", + "type" : "keyword", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "text", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + }, + { + "name" : "from", + "type" : "text", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "date", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + }, + { + "name" : "from", + "type" : "text", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "keyword", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + }, + { + "name" : "from", + "type" : "text", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "text", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "time_duration", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, { "params" : [ { @@ -1292,6 +1598,9 @@ "FROM employees\n| WHERE hire_date >= \"1985-01-01T00:00:00Z\" AND hire_date < \"1986-01-01T00:00:00Z\"\n| STATS c = COUNT(1) BY b = BUCKET(salary, 5000.)\n| SORT b", "FROM sample_data \n| WHERE @timestamp >= NOW() - 1 day and @timestamp < NOW()\n| STATS COUNT(*) BY bucket = BUCKET(@timestamp, 25, NOW() - 1 day, NOW())", "FROM employees\n| WHERE hire_date >= \"1985-01-01T00:00:00Z\" AND hire_date < \"1986-01-01T00:00:00Z\"\n| STATS AVG(salary) BY bucket = BUCKET(hire_date, 20, \"1985-01-01T00:00:00Z\", \"1986-01-01T00:00:00Z\")\n| SORT bucket", - "FROM employees\n| STATS s1 = b1 + 1, s2 = BUCKET(salary / 1000 + 999, 50.) + 2 BY b1 = BUCKET(salary / 100 + 99, 50.), b2 = BUCKET(salary / 1000 + 999, 50.)\n| SORT b1, b2\n| KEEP s1, b1, s2, b2" - ] + "FROM employees\n| STATS s1 = b1 + 1, s2 = BUCKET(salary / 1000 + 999, 50.) + 2 BY b1 = BUCKET(salary / 100 + 99, 50.), b2 = BUCKET(salary / 1000 + 999, 50.)\n| SORT b1, b2\n| KEEP s1, b1, s2, b2", + "FROM employees \n| STATS dates = VALUES(birth_date) BY b = BUCKET(birth_date + 1 HOUR, 1 YEAR) - 1 HOUR\n| EVAL d_count = MV_COUNT(dates)\n| SORT d_count\n| LIMIT 3" + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/byte_length.json b/docs/reference/esql/functions/kibana/definition/byte_length.json new file mode 100644 index 0000000000000..60f439b9d8133 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/byte_length.json @@ -0,0 +1,38 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "byte_length", + "description" : "Returns the byte length of a string.", + "note" : "All strings are in UTF-8, so a single character can use multiple bytes.", + "signatures" : [ + { + "params" : [ + { + "name" : "string", + "type" : "keyword", + "optional" : false, + "description" : "String expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "integer" + }, + { + "params" : [ + { + "name" : "string", + "type" : "text", + "optional" : false, + "description" : "String expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "integer" + } + ], + "examples" : [ + "FROM airports\n| WHERE country == \"India\"\n| KEEP city\n| EVAL fn_length = LENGTH(city), fn_byte_length = BYTE_LENGTH(city)" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/case.json b/docs/reference/esql/functions/kibana/definition/case.json index ab10460f48b25..51693d9d30660 100644 --- a/docs/reference/esql/functions/kibana/definition/case.json +++ b/docs/reference/esql/functions/kibana/definition/case.json @@ -172,6 +172,48 @@ "variadic" : true, "returnType" : "date" }, + { + "params" : [ + { + "name" : "condition", + "type" : "boolean", + "optional" : false, + "description" : "A condition." + }, + { + "name" : "trueValue", + "type" : "date_nanos", + "optional" : false, + "description" : "The value that's returned when the corresponding condition is the first to evaluate to `true`. The default value is returned when no condition matches." + } + ], + "variadic" : true, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "condition", + "type" : "boolean", + "optional" : false, + "description" : "A condition." + }, + { + "name" : "trueValue", + "type" : "date_nanos", + "optional" : false, + "description" : "The value that's returned when the corresponding condition is the first to evaluate to `true`. The default value is returned when no condition matches." + }, + { + "name" : "elseValue", + "type" : "date_nanos", + "optional" : true, + "description" : "The value that's returned when no condition evaluates to `true`." + } + ], + "variadic" : true, + "returnType" : "date_nanos" + }, { "params" : [ { @@ -424,6 +466,30 @@ "variadic" : true, "returnType" : "keyword" }, + { + "params" : [ + { + "name" : "condition", + "type" : "boolean", + "optional" : false, + "description" : "A condition." + }, + { + "name" : "trueValue", + "type" : "keyword", + "optional" : false, + "description" : "The value that's returned when the corresponding condition is the first to evaluate to `true`. The default value is returned when no condition matches." + }, + { + "name" : "elseValue", + "type" : "text", + "optional" : true, + "description" : "The value that's returned when no condition evaluates to `true`." + } + ], + "variadic" : true, + "returnType" : "keyword" + }, { "params" : [ { @@ -482,7 +548,31 @@ } ], "variadic" : true, - "returnType" : "text" + "returnType" : "keyword" + }, + { + "params" : [ + { + "name" : "condition", + "type" : "boolean", + "optional" : false, + "description" : "A condition." + }, + { + "name" : "trueValue", + "type" : "text", + "optional" : false, + "description" : "The value that's returned when the corresponding condition is the first to evaluate to `true`. The default value is returned when no condition matches." + }, + { + "name" : "elseValue", + "type" : "keyword", + "optional" : true, + "description" : "The value that's returned when no condition evaluates to `true`." + } + ], + "variadic" : true, + "returnType" : "keyword" }, { "params" : [ @@ -506,7 +596,7 @@ } ], "variadic" : true, - "returnType" : "text" + "returnType" : "keyword" }, { "params" : [ @@ -597,5 +687,7 @@ "FROM employees\n| EVAL type = CASE(\n languages <= 1, \"monolingual\",\n languages <= 2, \"bilingual\",\n \"polyglot\")\n| KEEP emp_no, languages, type", "FROM sample_data\n| EVAL successful = CASE(\n STARTS_WITH(message, \"Connected to\"), 1,\n message == \"Connection error\", 0\n )\n| STATS success_rate = AVG(successful)", "FROM sample_data\n| EVAL error = CASE(message LIKE \"*error*\", 1, 0)\n| EVAL hour = DATE_TRUNC(1 hour, @timestamp)\n| STATS error_rate = AVG(error) by hour\n| SORT hour" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/categorize.json b/docs/reference/esql/functions/kibana/definition/categorize.json new file mode 100644 index 0000000000000..ed5fa15232b85 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/categorize.json @@ -0,0 +1,37 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "categorize", + "description" : "Groups text messages into categories of similarly formatted text values.", + "signatures" : [ + { + "params" : [ + { + "name" : "field", + "type" : "keyword", + "optional" : false, + "description" : "Expression to categorize" + } + ], + "variadic" : false, + "returnType" : "keyword" + }, + { + "params" : [ + { + "name" : "field", + "type" : "text", + "optional" : false, + "description" : "Expression to categorize" + } + ], + "variadic" : false, + "returnType" : "keyword" + } + ], + "examples" : [ + "FROM sample_data\n| STATS count=COUNT() BY category=CATEGORIZE(message)" + ], + "preview" : true, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/cbrt.json b/docs/reference/esql/functions/kibana/definition/cbrt.json index 600174e17ca0c..ce23551f75e5f 100644 --- a/docs/reference/esql/functions/kibana/definition/cbrt.json +++ b/docs/reference/esql/functions/kibana/definition/cbrt.json @@ -55,5 +55,7 @@ ], "examples" : [ "ROW d = 1000.0\n| EVAL c = cbrt(d)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/ceil.json b/docs/reference/esql/functions/kibana/definition/ceil.json index b8ac9ad55f31a..15e7bafd1fa5d 100644 --- a/docs/reference/esql/functions/kibana/definition/ceil.json +++ b/docs/reference/esql/functions/kibana/definition/ceil.json @@ -56,5 +56,7 @@ ], "examples" : [ "ROW a=1.8\n| EVAL a=CEIL(a)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/cidr_match.json b/docs/reference/esql/functions/kibana/definition/cidr_match.json index dfa333c26941e..dc5d6773f69e6 100644 --- a/docs/reference/esql/functions/kibana/definition/cidr_match.json +++ b/docs/reference/esql/functions/kibana/definition/cidr_match.json @@ -43,5 +43,7 @@ ], "examples" : [ "FROM hosts \n| WHERE CIDR_MATCH(ip1, \"127.0.0.2/32\", \"127.0.0.3/32\") \n| KEEP card, host, ip0, ip1" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/coalesce.json b/docs/reference/esql/functions/kibana/definition/coalesce.json index 2459a4d51bb2d..c929323397c9b 100644 --- a/docs/reference/esql/functions/kibana/definition/coalesce.json +++ b/docs/reference/esql/functions/kibana/definition/coalesce.json @@ -88,6 +88,24 @@ "variadic" : true, "returnType" : "date" }, + { + "params" : [ + { + "name" : "first", + "type" : "date_nanos", + "optional" : false, + "description" : "Expression to evaluate." + }, + { + "name" : "rest", + "type" : "date_nanos", + "optional" : true, + "description" : "Other expression to evaluate." + } + ], + "variadic" : true, + "returnType" : "date_nanos" + }, { "params" : [ { @@ -242,7 +260,7 @@ } ], "variadic" : true, - "returnType" : "text" + "returnType" : "keyword" }, { "params" : [ @@ -260,7 +278,7 @@ } ], "variadic" : true, - "returnType" : "text" + "returnType" : "keyword" }, { "params" : [ @@ -283,5 +301,7 @@ ], "examples" : [ "ROW a=null, b=\"b\"\n| EVAL COALESCE(a, b)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/concat.json b/docs/reference/esql/functions/kibana/definition/concat.json index c12a5cdbf7a33..d3e9f0852bc13 100644 --- a/docs/reference/esql/functions/kibana/definition/concat.json +++ b/docs/reference/esql/functions/kibana/definition/concat.json @@ -79,5 +79,7 @@ ], "examples" : [ "FROM employees\n| KEEP first_name, last_name\n| EVAL fullname = CONCAT(first_name, \" \", last_name)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/cos.json b/docs/reference/esql/functions/kibana/definition/cos.json index c7757fbd4071d..1b9ca2f1d4867 100644 --- a/docs/reference/esql/functions/kibana/definition/cos.json +++ b/docs/reference/esql/functions/kibana/definition/cos.json @@ -55,5 +55,7 @@ ], "examples" : [ "ROW a=1.8 \n| EVAL cos=COS(a)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/cosh.json b/docs/reference/esql/functions/kibana/definition/cosh.json index dca261d971c40..77d9853d34e44 100644 --- a/docs/reference/esql/functions/kibana/definition/cosh.json +++ b/docs/reference/esql/functions/kibana/definition/cosh.json @@ -55,5 +55,7 @@ ], "examples" : [ "ROW a=1.8 \n| EVAL cosh=COSH(a)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/count.json b/docs/reference/esql/functions/kibana/definition/count.json index 2a15fb3bdd335..329a18c4d9d01 100644 --- a/docs/reference/esql/functions/kibana/definition/count.json +++ b/docs/reference/esql/functions/kibana/definition/count.json @@ -151,9 +151,11 @@ ], "examples" : [ "FROM employees\n| STATS COUNT(height)", - "FROM employees \n| STATS count = COUNT(*) BY languages \n| SORT languages DESC", + "FROM employees\n| STATS count = COUNT(*) BY languages\n| SORT languages DESC", "ROW words=\"foo;bar;baz;qux;quux;foo\"\n| STATS word_count = COUNT(SPLIT(words, \";\"))", "ROW n=1\n| WHERE n < 0\n| STATS COUNT(n)", "ROW n=1\n| STATS COUNT(n > 0 OR NULL), COUNT(n < 0 OR NULL)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/count_distinct.json b/docs/reference/esql/functions/kibana/definition/count_distinct.json index f6a148783ba42..54b99ee84ce2d 100644 --- a/docs/reference/esql/functions/kibana/definition/count_distinct.json +++ b/docs/reference/esql/functions/kibana/definition/count_distinct.json @@ -136,6 +136,72 @@ "variadic" : false, "returnType" : "long" }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Column or literal for which to count the number of distinct values." + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Column or literal for which to count the number of distinct values." + }, + { + "name" : "precision", + "type" : "integer", + "optional" : true, + "description" : "Precision threshold. Refer to <>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000." + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Column or literal for which to count the number of distinct values." + }, + { + "name" : "precision", + "type" : "long", + "optional" : true, + "description" : "Precision threshold. Refer to <>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000." + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Column or literal for which to count the number of distinct values." + }, + { + "name" : "precision", + "type" : "unsigned_long", + "optional" : true, + "description" : "Precision threshold. Refer to <>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000." + } + ], + "variadic" : false, + "returnType" : "long" + }, { "params" : [ { @@ -603,5 +669,7 @@ "FROM hosts\n| STATS COUNT_DISTINCT(ip0), COUNT_DISTINCT(ip1)", "FROM hosts\n| STATS COUNT_DISTINCT(ip0, 80000), COUNT_DISTINCT(ip1, 5)", "ROW words=\"foo;bar;baz;qux;quux;foo\"\n| STATS distinct_word_count = COUNT_DISTINCT(SPLIT(words, \";\"))" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/date_diff.json b/docs/reference/esql/functions/kibana/definition/date_diff.json index d6589f041075d..d32028d455348 100644 --- a/docs/reference/esql/functions/kibana/definition/date_diff.json +++ b/docs/reference/esql/functions/kibana/definition/date_diff.json @@ -54,6 +54,9 @@ } ], "examples" : [ - "ROW date1 = TO_DATETIME(\"2023-12-02T11:00:00.000Z\"), date2 = TO_DATETIME(\"2023-12-02T11:00:00.001Z\")\n| EVAL dd_ms = DATE_DIFF(\"microseconds\", date1, date2)" - ] + "ROW date1 = TO_DATETIME(\"2023-12-02T11:00:00.000Z\"), date2 = TO_DATETIME(\"2023-12-02T11:00:00.001Z\")\n| EVAL dd_ms = DATE_DIFF(\"microseconds\", date1, date2)", + "ROW end_23=TO_DATETIME(\"2023-12-31T23:59:59.999Z\"),\n start_24=TO_DATETIME(\"2024-01-01T00:00:00.000Z\"),\n end_24=TO_DATETIME(\"2024-12-31T23:59:59.999\")\n| EVAL end23_to_start24=DATE_DIFF(\"year\", end_23, start_24)\n| EVAL end23_to_end24=DATE_DIFF(\"year\", end_23, end_24)\n| EVAL start_to_end_24=DATE_DIFF(\"year\", start_24, end_24)" + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/date_extract.json b/docs/reference/esql/functions/kibana/definition/date_extract.json index 557f0e0a47e54..c6dc6583f324d 100644 --- a/docs/reference/esql/functions/kibana/definition/date_extract.json +++ b/docs/reference/esql/functions/kibana/definition/date_extract.json @@ -44,5 +44,7 @@ "examples" : [ "ROW date = DATE_PARSE(\"yyyy-MM-dd\", \"2022-05-06\")\n| EVAL year = DATE_EXTRACT(\"year\", date)", "FROM sample_data\n| WHERE DATE_EXTRACT(\"hour_of_day\", @timestamp) < 9 AND DATE_EXTRACT(\"hour_of_day\", @timestamp) >= 17" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/date_format.json b/docs/reference/esql/functions/kibana/definition/date_format.json index 7bd01d7f4ef31..6e2738fafb964 100644 --- a/docs/reference/esql/functions/kibana/definition/date_format.json +++ b/docs/reference/esql/functions/kibana/definition/date_format.json @@ -42,6 +42,8 @@ } ], "examples" : [ - "FROM employees\n| KEEP first_name, last_name, hire_date\n| EVAL hired = DATE_FORMAT(\"YYYY-MM-dd\", hire_date)" - ] + "FROM employees\n| KEEP first_name, last_name, hire_date\n| EVAL hired = DATE_FORMAT(\"yyyy-MM-dd\", hire_date)" + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/date_parse.json b/docs/reference/esql/functions/kibana/definition/date_parse.json index 9400340750c2a..77247b332c6f6 100644 --- a/docs/reference/esql/functions/kibana/definition/date_parse.json +++ b/docs/reference/esql/functions/kibana/definition/date_parse.json @@ -79,5 +79,7 @@ ], "examples" : [ "ROW date_string = \"2022-05-06\"\n| EVAL date = DATE_PARSE(\"yyyy-MM-dd\", date_string)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/date_trunc.json b/docs/reference/esql/functions/kibana/definition/date_trunc.json index bd3f362d1670b..cdda984a0ce7e 100644 --- a/docs/reference/esql/functions/kibana/definition/date_trunc.json +++ b/docs/reference/esql/functions/kibana/definition/date_trunc.json @@ -22,6 +22,24 @@ "variadic" : false, "returnType" : "date" }, + { + "params" : [ + { + "name" : "interval", + "type" : "date_period", + "optional" : false, + "description" : "Interval; expressed using the timespan literal syntax." + }, + { + "name" : "date", + "type" : "date_nanos", + "optional" : false, + "description" : "Date expression" + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, { "params" : [ { @@ -39,11 +57,31 @@ ], "variadic" : false, "returnType" : "date" + }, + { + "params" : [ + { + "name" : "interval", + "type" : "time_duration", + "optional" : false, + "description" : "Interval; expressed using the timespan literal syntax." + }, + { + "name" : "date", + "type" : "date_nanos", + "optional" : false, + "description" : "Date expression" + } + ], + "variadic" : false, + "returnType" : "date_nanos" } ], "examples" : [ "FROM employees\n| KEEP first_name, last_name, hire_date\n| EVAL year_hired = DATE_TRUNC(1 year, hire_date)", "FROM employees\n| EVAL year = DATE_TRUNC(1 year, hire_date)\n| STATS hires = COUNT(emp_no) BY year\n| SORT year", "FROM sample_data\n| EVAL error = CASE(message LIKE \"*error*\", 1, 0)\n| EVAL hour = DATE_TRUNC(1 hour, @timestamp)\n| STATS error_rate = AVG(error) by hour\n| SORT hour" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/div.json b/docs/reference/esql/functions/kibana/definition/div.json index 8bd2c33720d5f..9132f32a27778 100644 --- a/docs/reference/esql/functions/kibana/definition/div.json +++ b/docs/reference/esql/functions/kibana/definition/div.json @@ -185,5 +185,7 @@ "variadic" : false, "returnType" : "unsigned_long" } - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/e.json b/docs/reference/esql/functions/kibana/definition/e.json index 6bd0197797e59..e5941b50c00a6 100644 --- a/docs/reference/esql/functions/kibana/definition/e.json +++ b/docs/reference/esql/functions/kibana/definition/e.json @@ -11,5 +11,7 @@ ], "examples" : [ "ROW E()" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/ends_with.json b/docs/reference/esql/functions/kibana/definition/ends_with.json index 754d2c965287b..5c2f721c33442 100644 --- a/docs/reference/esql/functions/kibana/definition/ends_with.json +++ b/docs/reference/esql/functions/kibana/definition/ends_with.json @@ -79,5 +79,7 @@ ], "examples" : [ "FROM employees\n| KEEP last_name\n| EVAL ln_E = ENDS_WITH(last_name, \"d\")" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/equals.json b/docs/reference/esql/functions/kibana/definition/equals.json index eca80ccdbf657..40f3d54ba597a 100644 --- a/docs/reference/esql/functions/kibana/definition/equals.json +++ b/docs/reference/esql/functions/kibana/definition/equals.json @@ -77,6 +77,60 @@ "variadic" : false, "returnType" : "boolean" }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, { "params" : [ { @@ -401,5 +455,7 @@ "variadic" : false, "returnType" : "boolean" } - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/exp.json b/docs/reference/esql/functions/kibana/definition/exp.json index 02f7433ede779..4df40ce467d80 100644 --- a/docs/reference/esql/functions/kibana/definition/exp.json +++ b/docs/reference/esql/functions/kibana/definition/exp.json @@ -55,5 +55,7 @@ ], "examples" : [ "ROW d = 5.0\n| EVAL s = EXP(d)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/floor.json b/docs/reference/esql/functions/kibana/definition/floor.json index b61588559a928..1d1c961bfe2b2 100644 --- a/docs/reference/esql/functions/kibana/definition/floor.json +++ b/docs/reference/esql/functions/kibana/definition/floor.json @@ -56,5 +56,7 @@ ], "examples" : [ "ROW a=1.8\n| EVAL a=FLOOR(a)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/from_base64.json b/docs/reference/esql/functions/kibana/definition/from_base64.json index 4f09b52bb06f6..7580b817031a5 100644 --- a/docs/reference/esql/functions/kibana/definition/from_base64.json +++ b/docs/reference/esql/functions/kibana/definition/from_base64.json @@ -31,5 +31,7 @@ ], "examples" : [ "row a = \"ZWxhc3RpYw==\" \n| eval d = from_base64(a)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/greater_than.json b/docs/reference/esql/functions/kibana/definition/greater_than.json index 7831b0f41cd9d..ea2c0fb1212c7 100644 --- a/docs/reference/esql/functions/kibana/definition/greater_than.json +++ b/docs/reference/esql/functions/kibana/definition/greater_than.json @@ -23,6 +23,60 @@ "variadic" : false, "returnType" : "boolean" }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, { "params" : [ { @@ -311,5 +365,7 @@ "variadic" : false, "returnType" : "boolean" } - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/greater_than_or_equal.json b/docs/reference/esql/functions/kibana/definition/greater_than_or_equal.json index b6a40a838c393..7e1feb37e87b0 100644 --- a/docs/reference/esql/functions/kibana/definition/greater_than_or_equal.json +++ b/docs/reference/esql/functions/kibana/definition/greater_than_or_equal.json @@ -23,6 +23,60 @@ "variadic" : false, "returnType" : "boolean" }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, { "params" : [ { @@ -311,5 +365,7 @@ "variadic" : false, "returnType" : "boolean" } - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/greatest.json b/docs/reference/esql/functions/kibana/definition/greatest.json index 15c9f58d32d3e..077100317dfca 100644 --- a/docs/reference/esql/functions/kibana/definition/greatest.json +++ b/docs/reference/esql/functions/kibana/definition/greatest.json @@ -35,6 +35,42 @@ "variadic" : true, "returnType" : "boolean" }, + { + "params" : [ + { + "name" : "first", + "type" : "date", + "optional" : false, + "description" : "First of the columns to evaluate." + }, + { + "name" : "rest", + "type" : "date", + "optional" : true, + "description" : "The rest of the columns to evaluate." + } + ], + "variadic" : true, + "returnType" : "date" + }, + { + "params" : [ + { + "name" : "first", + "type" : "date_nanos", + "optional" : false, + "description" : "First of the columns to evaluate." + }, + { + "name" : "rest", + "type" : "date_nanos", + "optional" : true, + "description" : "The rest of the columns to evaluate." + } + ], + "variadic" : true, + "returnType" : "date_nanos" + }, { "params" : [ { @@ -171,7 +207,7 @@ } ], "variadic" : true, - "returnType" : "text" + "returnType" : "keyword" }, { "params" : [ @@ -189,7 +225,7 @@ } ], "variadic" : true, - "returnType" : "text" + "returnType" : "keyword" }, { "params" : [ @@ -212,5 +248,7 @@ ], "examples" : [ "ROW a = 10, b = 20\n| EVAL g = GREATEST(a, b)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/hash.json b/docs/reference/esql/functions/kibana/definition/hash.json new file mode 100644 index 0000000000000..17a60cf45acfe --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/hash.json @@ -0,0 +1,82 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "hash", + "description" : "Computes the hash of the input using various algorithms such as MD5, SHA, SHA-224, SHA-256, SHA-384, SHA-512.", + "signatures" : [ + { + "params" : [ + { + "name" : "algorithm", + "type" : "keyword", + "optional" : false, + "description" : "Hash algorithm to use." + }, + { + "name" : "input", + "type" : "keyword", + "optional" : false, + "description" : "Input to hash." + } + ], + "variadic" : false, + "returnType" : "keyword" + }, + { + "params" : [ + { + "name" : "algorithm", + "type" : "keyword", + "optional" : false, + "description" : "Hash algorithm to use." + }, + { + "name" : "input", + "type" : "text", + "optional" : false, + "description" : "Input to hash." + } + ], + "variadic" : false, + "returnType" : "keyword" + }, + { + "params" : [ + { + "name" : "algorithm", + "type" : "text", + "optional" : false, + "description" : "Hash algorithm to use." + }, + { + "name" : "input", + "type" : "keyword", + "optional" : false, + "description" : "Input to hash." + } + ], + "variadic" : false, + "returnType" : "keyword" + }, + { + "params" : [ + { + "name" : "algorithm", + "type" : "text", + "optional" : false, + "description" : "Hash algorithm to use." + }, + { + "name" : "input", + "type" : "text", + "optional" : false, + "description" : "Input to hash." + } + ], + "variadic" : false, + "returnType" : "keyword" + } + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/hypot.json b/docs/reference/esql/functions/kibana/definition/hypot.json new file mode 100644 index 0000000000000..06971f07a3585 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/hypot.json @@ -0,0 +1,301 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "hypot", + "description" : "Returns the hypotenuse of two numbers. The input can be any numeric values, the return value is always a double.\nHypotenuses of infinities are null.", + "signatures" : [ + { + "params" : [ + { + "name" : "number1", + "type" : "double", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "double", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "double", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "integer", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "double", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "double", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "unsigned_long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "integer", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "double", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "integer", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "integer", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "integer", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "integer", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "unsigned_long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "double", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "integer", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "unsigned_long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "unsigned_long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "double", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "unsigned_long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "integer", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "unsigned_long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "unsigned_long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "unsigned_long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + } + ], + "examples" : [ + "ROW a = 3.0, b = 4.0\n| EVAL c = HYPOT(a, b)" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/in.json b/docs/reference/esql/functions/kibana/definition/in.json index abf3bd64e2822..46fca7ced5a9f 100644 --- a/docs/reference/esql/functions/kibana/definition/in.json +++ b/docs/reference/esql/functions/kibana/definition/in.json @@ -259,5 +259,7 @@ ], "examples" : [ "ROW a = 1, b = 4, c = 3\n| WHERE c-a IN (3, b / 2, a)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/ip_prefix.json b/docs/reference/esql/functions/kibana/definition/ip_prefix.json index 00c3cf75a949e..bbf0702b325e9 100644 --- a/docs/reference/esql/functions/kibana/definition/ip_prefix.json +++ b/docs/reference/esql/functions/kibana/definition/ip_prefix.json @@ -31,5 +31,7 @@ ], "examples" : [ "row ip4 = to_ip(\"1.2.3.4\"), ip6 = to_ip(\"fe80::cae2:65ff:fece:feb9\")\n| eval ip4_prefix = ip_prefix(ip4, 24, 0), ip6_prefix = ip_prefix(ip6, 0, 112);" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/kql.json b/docs/reference/esql/functions/kibana/definition/kql.json new file mode 100644 index 0000000000000..6960681fbbf0d --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/kql.json @@ -0,0 +1,37 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "kql", + "description" : "Performs a KQL query. Returns true if the provided KQL query string matches the row.", + "signatures" : [ + { + "params" : [ + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Query string in KQL query string format." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "query", + "type" : "text", + "optional" : false, + "description" : "Query string in KQL query string format." + } + ], + "variadic" : false, + "returnType" : "boolean" + } + ], + "examples" : [ + "FROM books \n| WHERE KQL(\"author: Faulkner\")\n| KEEP book_no, author \n| SORT book_no \n| LIMIT 5;" + ], + "preview" : true, + "snapshot_only" : true +} diff --git a/docs/reference/esql/functions/kibana/definition/least.json b/docs/reference/esql/functions/kibana/definition/least.json index 0b922ad6ad3c2..18ec65c60f475 100644 --- a/docs/reference/esql/functions/kibana/definition/least.json +++ b/docs/reference/esql/functions/kibana/definition/least.json @@ -34,6 +34,42 @@ "variadic" : true, "returnType" : "boolean" }, + { + "params" : [ + { + "name" : "first", + "type" : "date", + "optional" : false, + "description" : "First of the columns to evaluate." + }, + { + "name" : "rest", + "type" : "date", + "optional" : true, + "description" : "The rest of the columns to evaluate." + } + ], + "variadic" : true, + "returnType" : "date" + }, + { + "params" : [ + { + "name" : "first", + "type" : "date_nanos", + "optional" : false, + "description" : "First of the columns to evaluate." + }, + { + "name" : "rest", + "type" : "date_nanos", + "optional" : true, + "description" : "The rest of the columns to evaluate." + } + ], + "variadic" : true, + "returnType" : "date_nanos" + }, { "params" : [ { @@ -170,7 +206,7 @@ } ], "variadic" : true, - "returnType" : "text" + "returnType" : "keyword" }, { "params" : [ @@ -188,7 +224,7 @@ } ], "variadic" : true, - "returnType" : "text" + "returnType" : "keyword" }, { "params" : [ @@ -211,5 +247,7 @@ ], "examples" : [ "ROW a = 10, b = 20\n| EVAL l = LEAST(a, b)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/left.json b/docs/reference/esql/functions/kibana/definition/left.json index bcda92b887bb0..9e226d6cde59b 100644 --- a/docs/reference/esql/functions/kibana/definition/left.json +++ b/docs/reference/esql/functions/kibana/definition/left.json @@ -43,5 +43,7 @@ ], "examples" : [ "FROM employees\n| KEEP last_name\n| EVAL left = LEFT(last_name, 3)\n| SORT last_name ASC\n| LIMIT 5" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/length.json b/docs/reference/esql/functions/kibana/definition/length.json index 121b938c815d5..bc26acde744f5 100644 --- a/docs/reference/esql/functions/kibana/definition/length.json +++ b/docs/reference/esql/functions/kibana/definition/length.json @@ -3,6 +3,7 @@ "type" : "eval", "name" : "length", "description" : "Returns the character length of a string.", + "note" : "All strings are in UTF-8, so a single character can use multiple bytes.", "signatures" : [ { "params" : [ @@ -30,6 +31,8 @@ } ], "examples" : [ - "FROM employees\n| KEEP first_name, last_name\n| EVAL fn_length = LENGTH(first_name)" - ] + "FROM airports\n| WHERE country == \"India\"\n| KEEP city\n| EVAL fn_length = LENGTH(city)" + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/less_than.json b/docs/reference/esql/functions/kibana/definition/less_than.json index bf6b9c5c08774..71aae4d759ecf 100644 --- a/docs/reference/esql/functions/kibana/definition/less_than.json +++ b/docs/reference/esql/functions/kibana/definition/less_than.json @@ -23,6 +23,60 @@ "variadic" : false, "returnType" : "boolean" }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, { "params" : [ { @@ -311,5 +365,7 @@ "variadic" : false, "returnType" : "boolean" } - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/less_than_or_equal.json b/docs/reference/esql/functions/kibana/definition/less_than_or_equal.json index 4e57161887141..f119b7ab2eb12 100644 --- a/docs/reference/esql/functions/kibana/definition/less_than_or_equal.json +++ b/docs/reference/esql/functions/kibana/definition/less_than_or_equal.json @@ -23,6 +23,60 @@ "variadic" : false, "returnType" : "boolean" }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, { "params" : [ { @@ -311,5 +365,7 @@ "variadic" : false, "returnType" : "boolean" } - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/like.json b/docs/reference/esql/functions/kibana/definition/like.json index 9a215ff88e399..f375c697bd60d 100644 --- a/docs/reference/esql/functions/kibana/definition/like.json +++ b/docs/reference/esql/functions/kibana/definition/like.json @@ -42,6 +42,8 @@ } ], "examples" : [ - "FROM employees\n| WHERE first_name LIKE \"?b*\"\n| KEEP first_name, last_name" - ] + "FROM employees\n| WHERE first_name LIKE \"\"\"?b*\"\"\"\n| KEEP first_name, last_name" + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/locate.json b/docs/reference/esql/functions/kibana/definition/locate.json index a9ddc8c52368a..6278cb5e33a00 100644 --- a/docs/reference/esql/functions/kibana/definition/locate.json +++ b/docs/reference/esql/functions/kibana/definition/locate.json @@ -175,5 +175,7 @@ ], "examples" : [ "row a = \"hello\"\n| eval a_ll = locate(a, \"ll\")" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/log.json b/docs/reference/esql/functions/kibana/definition/log.json index 0edafefc4dd1a..369503c1b8e4a 100644 --- a/docs/reference/esql/functions/kibana/definition/log.json +++ b/docs/reference/esql/functions/kibana/definition/log.json @@ -344,5 +344,7 @@ "examples" : [ "ROW base = 2.0, value = 8.0\n| EVAL s = LOG(base, value)", "row value = 100\n| EVAL s = LOG(value);" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/log10.json b/docs/reference/esql/functions/kibana/definition/log10.json index ca506b0df33e2..a958e5caec8a8 100644 --- a/docs/reference/esql/functions/kibana/definition/log10.json +++ b/docs/reference/esql/functions/kibana/definition/log10.json @@ -55,5 +55,7 @@ ], "examples" : [ "ROW d = 1000.0 \n| EVAL s = LOG10(d)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/ltrim.json b/docs/reference/esql/functions/kibana/definition/ltrim.json index 8b248634a0259..6d992b9db7b2c 100644 --- a/docs/reference/esql/functions/kibana/definition/ltrim.json +++ b/docs/reference/esql/functions/kibana/definition/ltrim.json @@ -26,10 +26,12 @@ } ], "variadic" : false, - "returnType" : "text" + "returnType" : "keyword" } ], "examples" : [ "ROW message = \" some text \", color = \" red \"\n| EVAL message = LTRIM(message)\n| EVAL color = LTRIM(color)\n| EVAL message = CONCAT(\"'\", message, \"'\")\n| EVAL color = CONCAT(\"'\", color, \"'\")" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/match.json b/docs/reference/esql/functions/kibana/definition/match.json new file mode 100644 index 0000000000000..7f2a8239cc0d0 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/match.json @@ -0,0 +1,535 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "match", + "description" : "Performs a <> on the specified field. Returns true if the provided query matches the row.", + "signatures" : [ + { + "params" : [ + { + "name" : "field", + "type" : "boolean", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "boolean", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "boolean", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "date", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "date_nanos", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "double", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "double", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "double", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "integer", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "double", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "double", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "long", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "integer", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "double", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "integer", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "integer", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "integer", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "integer", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "long", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "ip", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "ip", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "ip", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "keyword", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "double", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "integer", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "long", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "text", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "unsigned_long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "double", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "unsigned_long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "integer", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "unsigned_long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "unsigned_long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "long", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "unsigned_long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "unsigned_long", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "version", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "version", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "version", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + } + ], + "examples" : [ + "FROM books \n| WHERE MATCH(author, \"Faulkner\")\n| KEEP book_no, author \n| SORT book_no \n| LIMIT 5;" + ], + "preview" : true, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/match_operator.json b/docs/reference/esql/functions/kibana/definition/match_operator.json new file mode 100644 index 0000000000000..44233bbddb653 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/match_operator.json @@ -0,0 +1,535 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "operator", + "name" : "match_operator", + "description" : "Performs a <> on the specified field. Returns true if the provided query matches the row.", + "signatures" : [ + { + "params" : [ + { + "name" : "field", + "type" : "boolean", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "boolean", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "boolean", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "date", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "date_nanos", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "double", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "double", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "double", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "integer", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "double", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "double", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "long", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "integer", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "double", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "integer", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "integer", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "integer", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "integer", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "long", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "ip", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "ip", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "ip", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "keyword", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "double", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "integer", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "long", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "text", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "unsigned_long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "double", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "unsigned_long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "integer", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "unsigned_long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "unsigned_long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "long", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "unsigned_long", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "unsigned_long", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "version", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "version", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "version", + "optional" : false, + "description" : "Value to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + } + ], + "examples" : [ + "FROM books \n| WHERE MATCH(author, \"Faulkner\")\n| KEEP book_no, author \n| SORT book_no \n| LIMIT 5;" + ], + "preview" : true, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/max.json b/docs/reference/esql/functions/kibana/definition/max.json index b13d367d37345..7f3d2215ee099 100644 --- a/docs/reference/esql/functions/kibana/definition/max.json +++ b/docs/reference/esql/functions/kibana/definition/max.json @@ -28,6 +28,18 @@ "variadic" : false, "returnType" : "date" }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, { "params" : [ { @@ -98,7 +110,7 @@ } ], "variadic" : false, - "returnType" : "text" + "returnType" : "keyword" }, { "params" : [ @@ -116,5 +128,7 @@ "examples" : [ "FROM employees\n| STATS MAX(languages)", "FROM employees\n| STATS max_avg_salary_change = MAX(MV_AVG(salary_change))" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/median.json b/docs/reference/esql/functions/kibana/definition/median.json index 4887a4497e813..c29ae46927163 100644 --- a/docs/reference/esql/functions/kibana/definition/median.json +++ b/docs/reference/esql/functions/kibana/definition/median.json @@ -45,5 +45,7 @@ "examples" : [ "FROM employees\n| STATS MEDIAN(salary), PERCENTILE(salary, 50)", "FROM employees\n| STATS median_max_salary_change = MEDIAN(MV_MAX(salary_change))" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/median_absolute_deviation.json b/docs/reference/esql/functions/kibana/definition/median_absolute_deviation.json index 4a8b1cd30611f..06fc77d782346 100644 --- a/docs/reference/esql/functions/kibana/definition/median_absolute_deviation.json +++ b/docs/reference/esql/functions/kibana/definition/median_absolute_deviation.json @@ -45,5 +45,7 @@ "examples" : [ "FROM employees\n| STATS MEDIAN(salary), MEDIAN_ABSOLUTE_DEVIATION(salary)", "FROM employees\n| STATS m_a_d_max_salary_change = MEDIAN_ABSOLUTE_DEVIATION(MV_MAX(salary_change))" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/min.json b/docs/reference/esql/functions/kibana/definition/min.json index 338ed10d67b2e..74e3fd8208f1b 100644 --- a/docs/reference/esql/functions/kibana/definition/min.json +++ b/docs/reference/esql/functions/kibana/definition/min.json @@ -28,6 +28,18 @@ "variadic" : false, "returnType" : "date" }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, { "params" : [ { @@ -98,7 +110,7 @@ } ], "variadic" : false, - "returnType" : "text" + "returnType" : "keyword" }, { "params" : [ @@ -116,5 +128,7 @@ "examples" : [ "FROM employees\n| STATS MIN(languages)", "FROM employees\n| STATS min_avg_salary_change = MIN(MV_AVG(salary_change))" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/mod.json b/docs/reference/esql/functions/kibana/definition/mod.json index c43f697127249..51e4c55e41528 100644 --- a/docs/reference/esql/functions/kibana/definition/mod.json +++ b/docs/reference/esql/functions/kibana/definition/mod.json @@ -184,5 +184,7 @@ "variadic" : false, "returnType" : "unsigned_long" } - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/mul.json b/docs/reference/esql/functions/kibana/definition/mul.json index 9e42a55bdffef..961330cd23661 100644 --- a/docs/reference/esql/functions/kibana/definition/mul.json +++ b/docs/reference/esql/functions/kibana/definition/mul.json @@ -184,5 +184,7 @@ "variadic" : false, "returnType" : "unsigned_long" } - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/mv_append.json b/docs/reference/esql/functions/kibana/definition/mv_append.json index 3365226141f8f..81c1b777be498 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_append.json +++ b/docs/reference/esql/functions/kibana/definition/mv_append.json @@ -218,7 +218,7 @@ } ], "variadic" : false, - "returnType" : "text" + "returnType" : "keyword" }, { "params" : [ @@ -238,5 +238,7 @@ "variadic" : false, "returnType" : "version" } - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/mv_avg.json b/docs/reference/esql/functions/kibana/definition/mv_avg.json index 52a37cc291dca..a25d38c909e66 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_avg.json +++ b/docs/reference/esql/functions/kibana/definition/mv_avg.json @@ -55,5 +55,7 @@ ], "examples" : [ "ROW a=[3, 5, 1, 6]\n| EVAL avg_a = MV_AVG(a)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/mv_concat.json b/docs/reference/esql/functions/kibana/definition/mv_concat.json index 9cf458c847cf6..88b1107a9f401 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_concat.json +++ b/docs/reference/esql/functions/kibana/definition/mv_concat.json @@ -80,5 +80,7 @@ "examples" : [ "ROW a=[\"foo\", \"zoo\", \"bar\"]\n| EVAL j = MV_CONCAT(a, \", \")", "ROW a=[10, 9, 8]\n| EVAL j = MV_CONCAT(TO_STRING(a), \", \")" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/mv_count.json b/docs/reference/esql/functions/kibana/definition/mv_count.json index f125327314f4e..90ace2525f710 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_count.json +++ b/docs/reference/esql/functions/kibana/definition/mv_count.json @@ -52,6 +52,18 @@ "variadic" : false, "returnType" : "integer" }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Multivalue expression." + } + ], + "variadic" : false, + "returnType" : "integer" + }, { "params" : [ { @@ -175,5 +187,7 @@ ], "examples" : [ "ROW a=[\"foo\", \"zoo\", \"bar\"]\n| EVAL count_a = MV_COUNT(a)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/mv_dedupe.json b/docs/reference/esql/functions/kibana/definition/mv_dedupe.json index 7d66e3dcc0b9b..ce2c96dbc1757 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_dedupe.json +++ b/docs/reference/esql/functions/kibana/definition/mv_dedupe.json @@ -53,6 +53,18 @@ "variadic" : false, "returnType" : "date" }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Multivalue expression." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, { "params" : [ { @@ -147,7 +159,7 @@ } ], "variadic" : false, - "returnType" : "text" + "returnType" : "keyword" }, { "params" : [ @@ -164,5 +176,7 @@ ], "examples" : [ "ROW a=[\"foo\", \"foo\", \"bar\", \"foo\"]\n| EVAL dedupe_a = MV_DEDUPE(a)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/mv_first.json b/docs/reference/esql/functions/kibana/definition/mv_first.json index de6e642068517..552f568c9b171 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_first.json +++ b/docs/reference/esql/functions/kibana/definition/mv_first.json @@ -2,7 +2,7 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", "type" : "eval", "name" : "mv_first", - "description" : "Converts a multivalued expression into a single valued column containing the\nfirst value. This is most useful when reading from a function that emits\nmultivalued columns in a known order like <>.\n\nThe order that <> are read from\nunderlying storage is not guaranteed. It is *frequently* ascending, but don't\nrely on that. If you need the minimum value use <> instead of\n`MV_FIRST`. `MV_MIN` has optimizations for sorted values so there isn't a\nperformance benefit to `MV_FIRST`.", + "description" : "Converts a multivalued expression into a single valued column containing the\nfirst value. This is most useful when reading from a function that emits\nmultivalued columns in a known order like <>.", "signatures" : [ { "params" : [ @@ -52,6 +52,18 @@ "variadic" : false, "returnType" : "date" }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Multivalue expression." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, { "params" : [ { @@ -146,7 +158,7 @@ } ], "variadic" : false, - "returnType" : "text" + "returnType" : "keyword" }, { "params" : [ @@ -175,5 +187,7 @@ ], "examples" : [ "ROW a=\"foo;bar;baz\"\n| EVAL first_a = MV_FIRST(SPLIT(a, \";\"))" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/mv_last.json b/docs/reference/esql/functions/kibana/definition/mv_last.json index ea1293e7acfec..78d7b348a6042 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_last.json +++ b/docs/reference/esql/functions/kibana/definition/mv_last.json @@ -2,7 +2,7 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", "type" : "eval", "name" : "mv_last", - "description" : "Converts a multivalue expression into a single valued column containing the last\nvalue. This is most useful when reading from a function that emits multivalued\ncolumns in a known order like <>.\n\nThe order that <> are read from\nunderlying storage is not guaranteed. It is *frequently* ascending, but don't\nrely on that. If you need the maximum value use <> instead of\n`MV_LAST`. `MV_MAX` has optimizations for sorted values so there isn't a\nperformance benefit to `MV_LAST`.", + "description" : "Converts a multivalue expression into a single valued column containing the last\nvalue. This is most useful when reading from a function that emits multivalued\ncolumns in a known order like <>.", "signatures" : [ { "params" : [ @@ -52,6 +52,18 @@ "variadic" : false, "returnType" : "date" }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Multivalue expression." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, { "params" : [ { @@ -146,7 +158,7 @@ } ], "variadic" : false, - "returnType" : "text" + "returnType" : "keyword" }, { "params" : [ @@ -175,5 +187,7 @@ ], "examples" : [ "ROW a=\"foo;bar;baz\"\n| EVAL last_a = MV_LAST(SPLIT(a, \";\"))" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/mv_max.json b/docs/reference/esql/functions/kibana/definition/mv_max.json index eb25369f78f77..a1e55c58cff70 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_max.json +++ b/docs/reference/esql/functions/kibana/definition/mv_max.json @@ -28,6 +28,18 @@ "variadic" : false, "returnType" : "date" }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Multivalue expression." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, { "params" : [ { @@ -98,7 +110,7 @@ } ], "variadic" : false, - "returnType" : "text" + "returnType" : "keyword" }, { "params" : [ @@ -128,5 +140,7 @@ "examples" : [ "ROW a=[3, 5, 1]\n| EVAL max_a = MV_MAX(a)", "ROW a=[\"foo\", \"zoo\", \"bar\"]\n| EVAL max_a = MV_MAX(a)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/mv_median.json b/docs/reference/esql/functions/kibana/definition/mv_median.json index ea713476e506d..fe95e1999f6a9 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_median.json +++ b/docs/reference/esql/functions/kibana/definition/mv_median.json @@ -56,5 +56,7 @@ "examples" : [ "ROW a=[3, 5, 1]\n| EVAL median_a = MV_MEDIAN(a)", "ROW a=[3, 7, 1, 6]\n| EVAL median_a = MV_MEDIAN(a)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/mv_median_absolute_deviation.json b/docs/reference/esql/functions/kibana/definition/mv_median_absolute_deviation.json index d6f1174a4e259..af16e9ab0d3bb 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_median_absolute_deviation.json +++ b/docs/reference/esql/functions/kibana/definition/mv_median_absolute_deviation.json @@ -56,5 +56,7 @@ ], "examples" : [ "ROW values = [0, 2, 5, 6]\n| EVAL median_absolute_deviation = MV_MEDIAN_ABSOLUTE_DEVIATION(values), median = MV_MEDIAN(values)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/mv_min.json b/docs/reference/esql/functions/kibana/definition/mv_min.json index 87ad94338492e..7998ca4eda94e 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_min.json +++ b/docs/reference/esql/functions/kibana/definition/mv_min.json @@ -28,6 +28,18 @@ "variadic" : false, "returnType" : "date" }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Multivalue expression." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, { "params" : [ { @@ -98,7 +110,7 @@ } ], "variadic" : false, - "returnType" : "text" + "returnType" : "keyword" }, { "params" : [ @@ -128,5 +140,7 @@ "examples" : [ "ROW a=[2, 1]\n| EVAL min_a = MV_MIN(a)", "ROW a=[\"foo\", \"bar\"]\n| EVAL min_a = MV_MIN(a)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/mv_percentile.json b/docs/reference/esql/functions/kibana/definition/mv_percentile.json index dad611122f0db..8e4cec705f48d 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_percentile.json +++ b/docs/reference/esql/functions/kibana/definition/mv_percentile.json @@ -169,5 +169,7 @@ ], "examples" : [ "ROW values = [5, 5, 10, 12, 5000]\n| EVAL p50 = MV_PERCENTILE(values, 50), median = MV_MEDIAN(values)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/mv_pseries_weighted_sum.json b/docs/reference/esql/functions/kibana/definition/mv_pseries_weighted_sum.json index 626f7befbb12e..f96a6dfb5a43a 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_pseries_weighted_sum.json +++ b/docs/reference/esql/functions/kibana/definition/mv_pseries_weighted_sum.json @@ -25,5 +25,7 @@ ], "examples" : [ "ROW a = [70.0, 45.0, 21.0, 21.0, 21.0]\n| EVAL sum = MV_PSERIES_WEIGHTED_SUM(a, 1.5)\n| KEEP sum" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/mv_slice.json b/docs/reference/esql/functions/kibana/definition/mv_slice.json index ff52467b7d84a..df4d48145fac6 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_slice.json +++ b/docs/reference/esql/functions/kibana/definition/mv_slice.json @@ -2,7 +2,7 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", "type" : "eval", "name" : "mv_slice", - "description" : "Returns a subset of the multivalued field using the start and end index values.", + "description" : "Returns a subset of the multivalued field using the start and end index values.\nThis is most useful when reading from a function that emits multivalued columns\nin a known order like <> or <>.", "signatures" : [ { "params" : [ @@ -100,6 +100,30 @@ "variadic" : false, "returnType" : "date" }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Multivalue expression. If `null`, the function returns `null`." + }, + { + "name" : "start", + "type" : "integer", + "optional" : false, + "description" : "Start position. If `null`, the function returns `null`. The start argument can be negative. An index of -1 is used to specify the last value in the list." + }, + { + "name" : "end", + "type" : "integer", + "optional" : true, + "description" : "End position(included). Optional; if omitted, the position at `start` is returned. The end argument can be negative. An index of -1 is used to specify the last value in the list." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, { "params" : [ { @@ -290,7 +314,7 @@ } ], "variadic" : false, - "returnType" : "text" + "returnType" : "keyword" }, { "params" : [ @@ -320,5 +344,7 @@ "examples" : [ "row a = [1, 2, 2, 3]\n| eval a1 = mv_slice(a, 1), a2 = mv_slice(a, 2, 3)", "row a = [1, 2, 2, 3]\n| eval a1 = mv_slice(a, -2), a2 = mv_slice(a, -3, -1)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/mv_sort.json b/docs/reference/esql/functions/kibana/definition/mv_sort.json index d2bbd2c0fdbf4..072c05743af33 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_sort.json +++ b/docs/reference/esql/functions/kibana/definition/mv_sort.json @@ -40,6 +40,24 @@ "variadic" : false, "returnType" : "date" }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Multivalue expression. If `null`, the function returns `null`." + }, + { + "name" : "order", + "type" : "keyword", + "optional" : true, + "description" : "Sort order. The valid options are ASC and DESC, the default is ASC." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, { "params" : [ { @@ -146,7 +164,7 @@ } ], "variadic" : false, - "returnType" : "text" + "returnType" : "keyword" }, { "params" : [ @@ -169,5 +187,7 @@ ], "examples" : [ "ROW a = [4, 2, -3, 2]\n| EVAL sa = mv_sort(a), sd = mv_sort(a, \"DESC\")" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/mv_sum.json b/docs/reference/esql/functions/kibana/definition/mv_sum.json index f87be161b6eed..6ccbcec3c61b4 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_sum.json +++ b/docs/reference/esql/functions/kibana/definition/mv_sum.json @@ -55,5 +55,7 @@ ], "examples" : [ "ROW a=[3, 5, 6]\n| EVAL sum_a = MV_SUM(a)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/mv_zip.json b/docs/reference/esql/functions/kibana/definition/mv_zip.json index b6e5c86a3f0b8..e292cd27ee738 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_zip.json +++ b/docs/reference/esql/functions/kibana/definition/mv_zip.json @@ -271,5 +271,7 @@ ], "examples" : [ "ROW a = [\"x\", \"y\", \"z\"], b = [\"1\", \"2\"]\n| EVAL c = mv_zip(a, b, \"-\")\n| KEEP a, b, c" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/neg.json b/docs/reference/esql/functions/kibana/definition/neg.json index 6a7e86fd8e1a8..5d5f519f8fe50 100644 --- a/docs/reference/esql/functions/kibana/definition/neg.json +++ b/docs/reference/esql/functions/kibana/definition/neg.json @@ -64,5 +64,7 @@ "variadic" : false, "returnType" : "time_duration" } - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/not_equals.json b/docs/reference/esql/functions/kibana/definition/not_equals.json index 4b4d22a5abef4..d35a5b43ec238 100644 --- a/docs/reference/esql/functions/kibana/definition/not_equals.json +++ b/docs/reference/esql/functions/kibana/definition/not_equals.json @@ -77,6 +77,60 @@ "variadic" : false, "returnType" : "boolean" }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + }, + { + "name" : "rhs", + "type" : "date_nanos", + "optional" : false, + "description" : "An expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, { "params" : [ { @@ -401,5 +455,7 @@ "variadic" : false, "returnType" : "boolean" } - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/now.json b/docs/reference/esql/functions/kibana/definition/now.json index 1a2fc3a1dc42a..26ef362995185 100644 --- a/docs/reference/esql/functions/kibana/definition/now.json +++ b/docs/reference/esql/functions/kibana/definition/now.json @@ -12,5 +12,7 @@ "examples" : [ "ROW current_date = NOW()", "FROM sample_data\n| WHERE @timestamp > NOW() - 1 hour" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/percentile.json b/docs/reference/esql/functions/kibana/definition/percentile.json index 5d8a8371ed454..ea7a950b33c88 100644 --- a/docs/reference/esql/functions/kibana/definition/percentile.json +++ b/docs/reference/esql/functions/kibana/definition/percentile.json @@ -170,5 +170,7 @@ "examples" : [ "FROM employees\n| STATS p0 = PERCENTILE(salary, 0)\n , p50 = PERCENTILE(salary, 50)\n , p99 = PERCENTILE(salary, 99)", "FROM employees\n| STATS p80_max_salary_change = PERCENTILE(MV_MAX(salary_change), 80)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/pi.json b/docs/reference/esql/functions/kibana/definition/pi.json index e872762e2680f..98d9451a2b929 100644 --- a/docs/reference/esql/functions/kibana/definition/pi.json +++ b/docs/reference/esql/functions/kibana/definition/pi.json @@ -11,5 +11,7 @@ ], "examples" : [ "ROW PI()" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/pow.json b/docs/reference/esql/functions/kibana/definition/pow.json index bef8dee142b23..35fe080d82f79 100644 --- a/docs/reference/esql/functions/kibana/definition/pow.json +++ b/docs/reference/esql/functions/kibana/definition/pow.json @@ -297,5 +297,7 @@ "examples" : [ "ROW base = 2.0, exponent = 2\n| EVAL result = POW(base, exponent)", "ROW base = 4, exponent = 0.5\n| EVAL s = POW(base, exponent)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/qstr.json b/docs/reference/esql/functions/kibana/definition/qstr.json new file mode 100644 index 0000000000000..3b091bfe2e13b --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/qstr.json @@ -0,0 +1,37 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "qstr", + "description" : "Performs a <>. Returns true if the provided query string matches the row.", + "signatures" : [ + { + "params" : [ + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Query string in Lucene query string format." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "query", + "type" : "text", + "optional" : false, + "description" : "Query string in Lucene query string format." + } + ], + "variadic" : false, + "returnType" : "boolean" + } + ], + "examples" : [ + "FROM books \n| WHERE QSTR(\"author: Faulkner\")\n| KEEP book_no, author \n| SORT book_no \n| LIMIT 5;" + ], + "preview" : true, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/repeat.json b/docs/reference/esql/functions/kibana/definition/repeat.json index 2ac94e12c1796..b8660b6362e30 100644 --- a/docs/reference/esql/functions/kibana/definition/repeat.json +++ b/docs/reference/esql/functions/kibana/definition/repeat.json @@ -43,5 +43,7 @@ ], "examples" : [ "ROW a = \"Hello!\"\n| EVAL triple_a = REPEAT(a, 3);" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/replace.json b/docs/reference/esql/functions/kibana/definition/replace.json index 761d4faaae2dc..b5d1a1af7e8f4 100644 --- a/docs/reference/esql/functions/kibana/definition/replace.json +++ b/docs/reference/esql/functions/kibana/definition/replace.json @@ -199,5 +199,7 @@ ], "examples" : [ "ROW str = \"Hello World\"\n| EVAL str = REPLACE(str, \"World\", \"Universe\")\n| KEEP str" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/reverse.json b/docs/reference/esql/functions/kibana/definition/reverse.json new file mode 100644 index 0000000000000..518ec380a1356 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/reverse.json @@ -0,0 +1,39 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "reverse", + "description" : "Returns a new string representing the input string in reverse order.", + "note" : "If Elasticsearch is running with a JDK version less than 20 then this will not properly reverse Grapheme Clusters.\nElastic Cloud and the JDK bundled with Elasticsearch all use newer JDKs. But if you've explicitly shifted to an older jdk\nthen you'll see things like \"\uD83D\uDC4D\uD83C\uDFFD\uD83D\uDE0A\" be reversed to \"\uD83C\uDFFD\uD83D\uDC4D\uD83D\uDE0A\" instead of the correct \"\uD83D\uDE0A\uD83D\uDC4D\uD83C\uDFFD\".", + "signatures" : [ + { + "params" : [ + { + "name" : "str", + "type" : "keyword", + "optional" : false, + "description" : "String expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "keyword" + }, + { + "params" : [ + { + "name" : "str", + "type" : "text", + "optional" : false, + "description" : "String expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "keyword" + } + ], + "examples" : [ + "ROW message = \"Some Text\" | EVAL message_reversed = REVERSE(message);", + "ROW bending_arts = \"💧🪨🔥💨\" | EVAL bending_arts_reversed = REVERSE(bending_arts);" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/right.json b/docs/reference/esql/functions/kibana/definition/right.json index 4ac4c66384f56..1211877a2f902 100644 --- a/docs/reference/esql/functions/kibana/definition/right.json +++ b/docs/reference/esql/functions/kibana/definition/right.json @@ -43,5 +43,7 @@ ], "examples" : [ "FROM employees\n| KEEP last_name\n| EVAL right = RIGHT(last_name, 3)\n| SORT last_name ASC\n| LIMIT 5" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/rlike.json b/docs/reference/esql/functions/kibana/definition/rlike.json index 09bd1e033fe10..7a328293383bb 100644 --- a/docs/reference/esql/functions/kibana/definition/rlike.json +++ b/docs/reference/esql/functions/kibana/definition/rlike.json @@ -42,6 +42,8 @@ } ], "examples" : [ - "FROM employees\n| WHERE first_name RLIKE \".leja.*\"\n| KEEP first_name, last_name" - ] + "FROM employees\n| WHERE first_name RLIKE \"\"\".leja.*\"\"\"\n| KEEP first_name, last_name" + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/round.json b/docs/reference/esql/functions/kibana/definition/round.json index 4149f72db7ee6..4f4ddd36daa05 100644 --- a/docs/reference/esql/functions/kibana/definition/round.json +++ b/docs/reference/esql/functions/kibana/definition/round.json @@ -109,5 +109,7 @@ ], "examples" : [ "FROM employees\n| KEEP first_name, last_name, height\n| EVAL height_ft = ROUND(height * 3.281, 1)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/rtrim.json b/docs/reference/esql/functions/kibana/definition/rtrim.json index 0bc96dfef6a4f..9c8a7578ed789 100644 --- a/docs/reference/esql/functions/kibana/definition/rtrim.json +++ b/docs/reference/esql/functions/kibana/definition/rtrim.json @@ -26,10 +26,12 @@ } ], "variadic" : false, - "returnType" : "text" + "returnType" : "keyword" } ], "examples" : [ "ROW message = \" some text \", color = \" red \"\n| EVAL message = RTRIM(message)\n| EVAL color = RTRIM(color)\n| EVAL message = CONCAT(\"'\", message, \"'\")\n| EVAL color = CONCAT(\"'\", color, \"'\")" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/signum.json b/docs/reference/esql/functions/kibana/definition/signum.json index b8343283f457e..0902826651eff 100644 --- a/docs/reference/esql/functions/kibana/definition/signum.json +++ b/docs/reference/esql/functions/kibana/definition/signum.json @@ -55,5 +55,7 @@ ], "examples" : [ "ROW d = 100.0\n| EVAL s = SIGNUM(d)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/sin.json b/docs/reference/esql/functions/kibana/definition/sin.json index ce46fa66a2acc..0d4f4e5c5cf13 100644 --- a/docs/reference/esql/functions/kibana/definition/sin.json +++ b/docs/reference/esql/functions/kibana/definition/sin.json @@ -55,5 +55,7 @@ ], "examples" : [ "ROW a=1.8 \n| EVAL sin=SIN(a)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/sinh.json b/docs/reference/esql/functions/kibana/definition/sinh.json index e773e95e5e9e1..b20db3328775b 100644 --- a/docs/reference/esql/functions/kibana/definition/sinh.json +++ b/docs/reference/esql/functions/kibana/definition/sinh.json @@ -55,5 +55,7 @@ ], "examples" : [ "ROW a=1.8 \n| EVAL sinh=SINH(a)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/space.json b/docs/reference/esql/functions/kibana/definition/space.json index acf7466284d3b..7a819d3ccd213 100644 --- a/docs/reference/esql/functions/kibana/definition/space.json +++ b/docs/reference/esql/functions/kibana/definition/space.json @@ -19,5 +19,7 @@ ], "examples" : [ "ROW message = CONCAT(\"Hello\", SPACE(1), \"World!\");" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/split.json b/docs/reference/esql/functions/kibana/definition/split.json index e32a8d15f77ca..862cf7a09f19c 100644 --- a/docs/reference/esql/functions/kibana/definition/split.json +++ b/docs/reference/esql/functions/kibana/definition/split.json @@ -79,5 +79,7 @@ ], "examples" : [ "ROW words=\"foo;bar;baz;qux;quux;corge\"\n| EVAL word = SPLIT(words, \";\")" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/sqrt.json b/docs/reference/esql/functions/kibana/definition/sqrt.json index 7d9111036402d..11620a465c45f 100644 --- a/docs/reference/esql/functions/kibana/definition/sqrt.json +++ b/docs/reference/esql/functions/kibana/definition/sqrt.json @@ -55,5 +55,7 @@ ], "examples" : [ "ROW d = 100.0\n| EVAL s = SQRT(d)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/st_centroid_agg.json b/docs/reference/esql/functions/kibana/definition/st_centroid_agg.json index b01f91526709a..62bc97475432f 100644 --- a/docs/reference/esql/functions/kibana/definition/st_centroid_agg.json +++ b/docs/reference/esql/functions/kibana/definition/st_centroid_agg.json @@ -31,5 +31,7 @@ ], "examples" : [ "FROM airports\n| STATS centroid=ST_CENTROID_AGG(location)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/st_contains.json b/docs/reference/esql/functions/kibana/definition/st_contains.json index 1ef76e46f371a..0c1b61c563402 100644 --- a/docs/reference/esql/functions/kibana/definition/st_contains.json +++ b/docs/reference/esql/functions/kibana/definition/st_contains.json @@ -151,5 +151,7 @@ ], "examples" : [ "FROM airport_city_boundaries\n| WHERE ST_CONTAINS(city_boundary, TO_GEOSHAPE(\"POLYGON((109.35 18.3, 109.45 18.3, 109.45 18.4, 109.35 18.4, 109.35 18.3))\"))\n| KEEP abbrev, airport, region, city, city_location" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/st_disjoint.json b/docs/reference/esql/functions/kibana/definition/st_disjoint.json index e408a0f98fe6c..0c4f6e7c36f53 100644 --- a/docs/reference/esql/functions/kibana/definition/st_disjoint.json +++ b/docs/reference/esql/functions/kibana/definition/st_disjoint.json @@ -151,5 +151,7 @@ ], "examples" : [ "FROM airport_city_boundaries\n| WHERE ST_DISJOINT(city_boundary, TO_GEOSHAPE(\"POLYGON((-10 -60, 120 -60, 120 60, -10 60, -10 -60))\"))\n| KEEP abbrev, airport, region, city, city_location" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/st_distance.json b/docs/reference/esql/functions/kibana/definition/st_distance.json index 448e0d54051da..2fb8f341d12f4 100644 --- a/docs/reference/esql/functions/kibana/definition/st_distance.json +++ b/docs/reference/esql/functions/kibana/definition/st_distance.json @@ -43,5 +43,7 @@ ], "examples" : [ "FROM airports\n| WHERE abbrev == \"CPH\"\n| EVAL distance = ST_DISTANCE(location, city_location)\n| KEEP abbrev, name, location, city_location, distance" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/st_envelope.json b/docs/reference/esql/functions/kibana/definition/st_envelope.json new file mode 100644 index 0000000000000..6c00dda265ac7 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/st_envelope.json @@ -0,0 +1,61 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "st_envelope", + "description" : "Determines the minimum bounding box of the supplied geometry.", + "signatures" : [ + { + "params" : [ + { + "name" : "geometry", + "type" : "cartesian_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "cartesian_shape" + }, + { + "params" : [ + { + "name" : "geometry", + "type" : "cartesian_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "cartesian_shape" + }, + { + "params" : [ + { + "name" : "geometry", + "type" : "geo_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "geo_shape" + }, + { + "params" : [ + { + "name" : "geometry", + "type" : "geo_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "geo_shape" + } + ], + "examples" : [ + "FROM airport_city_boundaries\n| WHERE abbrev == \"CPH\"\n| EVAL envelope = ST_ENVELOPE(city_boundary)\n| KEEP abbrev, airport, envelope" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/st_extent_agg.json b/docs/reference/esql/functions/kibana/definition/st_extent_agg.json new file mode 100644 index 0000000000000..19afcc59e38a4 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/st_extent_agg.json @@ -0,0 +1,61 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "agg", + "name" : "st_extent_agg", + "description" : "Calculate the spatial extent over a field with geometry type. Returns a bounding box for all values of the field.", + "signatures" : [ + { + "params" : [ + { + "name" : "field", + "type" : "cartesian_point", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "cartesian_shape" + }, + { + "params" : [ + { + "name" : "field", + "type" : "cartesian_shape", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "cartesian_shape" + }, + { + "params" : [ + { + "name" : "field", + "type" : "geo_point", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "geo_shape" + }, + { + "params" : [ + { + "name" : "field", + "type" : "geo_shape", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "geo_shape" + } + ], + "examples" : [ + "FROM airports\n| WHERE country == \"India\"\n| STATS extent = ST_EXTENT_AGG(location)" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/st_intersects.json b/docs/reference/esql/functions/kibana/definition/st_intersects.json index 2f9f255ab1870..51237a0b1cee1 100644 --- a/docs/reference/esql/functions/kibana/definition/st_intersects.json +++ b/docs/reference/esql/functions/kibana/definition/st_intersects.json @@ -151,5 +151,7 @@ ], "examples" : [ "FROM airports\n| WHERE ST_INTERSECTS(location, TO_GEOSHAPE(\"POLYGON((42 14, 43 14, 43 15, 42 15, 42 14))\"))" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/st_within.json b/docs/reference/esql/functions/kibana/definition/st_within.json index e0cdf62fe0f98..d877cf4e8dfbc 100644 --- a/docs/reference/esql/functions/kibana/definition/st_within.json +++ b/docs/reference/esql/functions/kibana/definition/st_within.json @@ -151,5 +151,7 @@ ], "examples" : [ "FROM airport_city_boundaries\n| WHERE ST_WITHIN(city_boundary, TO_GEOSHAPE(\"POLYGON((109.1 18.15, 109.6 18.15, 109.6 18.65, 109.1 18.65, 109.1 18.15))\"))\n| KEEP abbrev, airport, region, city, city_location" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/st_x.json b/docs/reference/esql/functions/kibana/definition/st_x.json index c3554a2ee808b..f434f1467c0bf 100644 --- a/docs/reference/esql/functions/kibana/definition/st_x.json +++ b/docs/reference/esql/functions/kibana/definition/st_x.json @@ -31,5 +31,7 @@ ], "examples" : [ "ROW point = TO_GEOPOINT(\"POINT(42.97109629958868 14.7552534006536)\")\n| EVAL x = ST_X(point), y = ST_Y(point)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/st_xmax.json b/docs/reference/esql/functions/kibana/definition/st_xmax.json new file mode 100644 index 0000000000000..7be22617c0992 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/st_xmax.json @@ -0,0 +1,61 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "st_xmax", + "description" : "Extracts the maximum value of the `x` coordinates from the supplied geometry.\nIf the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the maximum `longitude` value.", + "signatures" : [ + { + "params" : [ + { + "name" : "point", + "type" : "cartesian_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "cartesian_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "geo_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "geo_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + } + ], + "examples" : [ + "FROM airport_city_boundaries\n| WHERE abbrev == \"CPH\"\n| EVAL envelope = ST_ENVELOPE(city_boundary)\n| EVAL xmin = ST_XMIN(envelope), xmax = ST_XMAX(envelope), ymin = ST_YMIN(envelope), ymax = ST_YMAX(envelope)\n| KEEP abbrev, airport, xmin, xmax, ymin, ymax" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/st_xmin.json b/docs/reference/esql/functions/kibana/definition/st_xmin.json new file mode 100644 index 0000000000000..8052fdb861cea --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/st_xmin.json @@ -0,0 +1,61 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "st_xmin", + "description" : "Extracts the minimum value of the `x` coordinates from the supplied geometry.\nIf the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the minimum `longitude` value.", + "signatures" : [ + { + "params" : [ + { + "name" : "point", + "type" : "cartesian_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "cartesian_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "geo_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "geo_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + } + ], + "examples" : [ + "FROM airport_city_boundaries\n| WHERE abbrev == \"CPH\"\n| EVAL envelope = ST_ENVELOPE(city_boundary)\n| EVAL xmin = ST_XMIN(envelope), xmax = ST_XMAX(envelope), ymin = ST_YMIN(envelope), ymax = ST_YMAX(envelope)\n| KEEP abbrev, airport, xmin, xmax, ymin, ymax" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/st_y.json b/docs/reference/esql/functions/kibana/definition/st_y.json index 2966ae04f75e4..ca7d817a36ff0 100644 --- a/docs/reference/esql/functions/kibana/definition/st_y.json +++ b/docs/reference/esql/functions/kibana/definition/st_y.json @@ -31,5 +31,7 @@ ], "examples" : [ "ROW point = TO_GEOPOINT(\"POINT(42.97109629958868 14.7552534006536)\")\n| EVAL x = ST_X(point), y = ST_Y(point)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/st_ymax.json b/docs/reference/esql/functions/kibana/definition/st_ymax.json new file mode 100644 index 0000000000000..1a53f7388ea56 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/st_ymax.json @@ -0,0 +1,61 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "st_ymax", + "description" : "Extracts the maximum value of the `y` coordinates from the supplied geometry.\nIf the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the maximum `latitude` value.", + "signatures" : [ + { + "params" : [ + { + "name" : "point", + "type" : "cartesian_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "cartesian_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "geo_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "geo_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + } + ], + "examples" : [ + "FROM airport_city_boundaries\n| WHERE abbrev == \"CPH\"\n| EVAL envelope = ST_ENVELOPE(city_boundary)\n| EVAL xmin = ST_XMIN(envelope), xmax = ST_XMAX(envelope), ymin = ST_YMIN(envelope), ymax = ST_YMAX(envelope)\n| KEEP abbrev, airport, xmin, xmax, ymin, ymax" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/st_ymin.json b/docs/reference/esql/functions/kibana/definition/st_ymin.json new file mode 100644 index 0000000000000..e11722a8f9c07 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/st_ymin.json @@ -0,0 +1,61 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "st_ymin", + "description" : "Extracts the minimum value of the `y` coordinates from the supplied geometry.\nIf the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the minimum `latitude` value.", + "signatures" : [ + { + "params" : [ + { + "name" : "point", + "type" : "cartesian_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "cartesian_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "geo_point", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "point", + "type" : "geo_shape", + "optional" : false, + "description" : "Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + } + ], + "examples" : [ + "FROM airport_city_boundaries\n| WHERE abbrev == \"CPH\"\n| EVAL envelope = ST_ENVELOPE(city_boundary)\n| EVAL xmin = ST_XMIN(envelope), xmax = ST_XMAX(envelope), ymin = ST_YMIN(envelope), ymax = ST_YMAX(envelope)\n| KEEP abbrev, airport, xmin, xmax, ymin, ymax" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/starts_with.json b/docs/reference/esql/functions/kibana/definition/starts_with.json index bfa3e7b0e5e42..0e5dc076a5689 100644 --- a/docs/reference/esql/functions/kibana/definition/starts_with.json +++ b/docs/reference/esql/functions/kibana/definition/starts_with.json @@ -79,5 +79,7 @@ ], "examples" : [ "FROM employees\n| KEEP last_name\n| EVAL ln_S = STARTS_WITH(last_name, \"B\")" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/std_dev.json b/docs/reference/esql/functions/kibana/definition/std_dev.json new file mode 100644 index 0000000000000..f31d3345421d9 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/std_dev.json @@ -0,0 +1,50 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "agg", + "name" : "std_dev", + "description" : "The standard deviation of a numeric field.", + "signatures" : [ + { + "params" : [ + { + "name" : "number", + "type" : "double", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "integer", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "long", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "double" + } + ], + "examples" : [ + "FROM employees\n| STATS STD_DEV(height)", + "FROM employees\n| STATS stddev_salary_change = STD_DEV(MV_MAX(salary_change))" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/sub.json b/docs/reference/esql/functions/kibana/definition/sub.json index 37e3852865e7f..608b5eb1009a7 100644 --- a/docs/reference/esql/functions/kibana/definition/sub.json +++ b/docs/reference/esql/functions/kibana/definition/sub.json @@ -40,6 +40,60 @@ "variadic" : false, "returnType" : "date" }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date_nanos", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "date_period", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date_nanos", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "time_duration", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "lhs", + "type" : "date_period", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "date_nanos", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, { "params" : [ { @@ -220,6 +274,24 @@ "variadic" : false, "returnType" : "long" }, + { + "params" : [ + { + "name" : "lhs", + "type" : "time_duration", + "optional" : false, + "description" : "A numeric value or a date time value." + }, + { + "name" : "rhs", + "type" : "date_nanos", + "optional" : false, + "description" : "A numeric value or a date time value." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, { "params" : [ { @@ -256,5 +328,7 @@ "variadic" : false, "returnType" : "unsigned_long" } - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/substring.json b/docs/reference/esql/functions/kibana/definition/substring.json index b38b545822a90..1998eb0428482 100644 --- a/docs/reference/esql/functions/kibana/definition/substring.json +++ b/docs/reference/esql/functions/kibana/definition/substring.json @@ -57,5 +57,7 @@ "FROM employees\n| KEEP last_name\n| EVAL ln_sub = SUBSTRING(last_name, 1, 3)", "FROM employees\n| KEEP last_name\n| EVAL ln_sub = SUBSTRING(last_name, -3, 3)", "FROM employees\n| KEEP last_name\n| EVAL ln_sub = SUBSTRING(last_name, 2)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/sum.json b/docs/reference/esql/functions/kibana/definition/sum.json index b9235b6ba04de..385a2e4d41399 100644 --- a/docs/reference/esql/functions/kibana/definition/sum.json +++ b/docs/reference/esql/functions/kibana/definition/sum.json @@ -44,5 +44,7 @@ "examples" : [ "FROM employees\n| STATS SUM(languages)", "FROM employees\n| STATS total_salary_changes = SUM(MV_MAX(salary_change))" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/tan.json b/docs/reference/esql/functions/kibana/definition/tan.json index f5452f310a995..76fab2ff6ab1e 100644 --- a/docs/reference/esql/functions/kibana/definition/tan.json +++ b/docs/reference/esql/functions/kibana/definition/tan.json @@ -55,5 +55,7 @@ ], "examples" : [ "ROW a=1.8 \n| EVAL tan=TAN(a)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/tanh.json b/docs/reference/esql/functions/kibana/definition/tanh.json index 081d606b64217..f56e053a3d4ee 100644 --- a/docs/reference/esql/functions/kibana/definition/tanh.json +++ b/docs/reference/esql/functions/kibana/definition/tanh.json @@ -55,5 +55,7 @@ ], "examples" : [ "ROW a=1.8 \n| EVAL tanh=TANH(a)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/tau.json b/docs/reference/esql/functions/kibana/definition/tau.json index 0f2602dc766ef..1dbb2e892ee60 100644 --- a/docs/reference/esql/functions/kibana/definition/tau.json +++ b/docs/reference/esql/functions/kibana/definition/tau.json @@ -11,5 +11,7 @@ ], "examples" : [ "ROW TAU()" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/term.json b/docs/reference/esql/functions/kibana/definition/term.json new file mode 100644 index 0000000000000..b0f129afd239c --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/term.json @@ -0,0 +1,85 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "term", + "description" : "Performs a Term query on the specified field. Returns true if the provided term matches the row.", + "signatures" : [ + { + "params" : [ + { + "name" : "field", + "type" : "keyword", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Term you wish to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "keyword", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "text", + "optional" : false, + "description" : "Term you wish to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "text", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Term you wish to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "text", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "text", + "optional" : false, + "description" : "Term you wish to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + } + ], + "examples" : [ + "FROM books \n| WHERE TERM(author, \"gabriel\") \n| KEEP book_no, title\n| LIMIT 3;" + ], + "preview" : true, + "snapshot_only" : true +} diff --git a/docs/reference/esql/functions/kibana/definition/to_base64.json b/docs/reference/esql/functions/kibana/definition/to_base64.json index 074a644bf1bab..84ace22073ab7 100644 --- a/docs/reference/esql/functions/kibana/definition/to_base64.json +++ b/docs/reference/esql/functions/kibana/definition/to_base64.json @@ -31,5 +31,7 @@ ], "examples" : [ "row a = \"elastic\" \n| eval e = to_base64(a)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/to_boolean.json b/docs/reference/esql/functions/kibana/definition/to_boolean.json index 7955c94c095b0..3a403caaace6c 100644 --- a/docs/reference/esql/functions/kibana/definition/to_boolean.json +++ b/docs/reference/esql/functions/kibana/definition/to_boolean.json @@ -91,5 +91,7 @@ ], "examples" : [ "ROW str = [\"true\", \"TRuE\", \"false\", \"\", \"yes\", \"1\"]\n| EVAL bool = TO_BOOLEAN(str)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/to_cartesianpoint.json b/docs/reference/esql/functions/kibana/definition/to_cartesianpoint.json index fdf606ec8ac58..7f6e99f88b777 100644 --- a/docs/reference/esql/functions/kibana/definition/to_cartesianpoint.json +++ b/docs/reference/esql/functions/kibana/definition/to_cartesianpoint.json @@ -43,5 +43,7 @@ ], "examples" : [ "ROW wkt = [\"POINT(4297.11 -1475.53)\", \"POINT(7580.93 2272.77)\"]\n| MV_EXPAND wkt\n| EVAL pt = TO_CARTESIANPOINT(wkt)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/to_cartesianshape.json b/docs/reference/esql/functions/kibana/definition/to_cartesianshape.json index 084ee5288beac..284288068b415 100644 --- a/docs/reference/esql/functions/kibana/definition/to_cartesianshape.json +++ b/docs/reference/esql/functions/kibana/definition/to_cartesianshape.json @@ -55,5 +55,7 @@ ], "examples" : [ "ROW wkt = [\"POINT(4297.11 -1475.53)\", \"POLYGON ((3339584.72 1118889.97, 4452779.63 4865942.27, 2226389.81 4865942.27, 1113194.90 2273030.92, 3339584.72 1118889.97))\"]\n| MV_EXPAND wkt\n| EVAL geom = TO_CARTESIANSHAPE(wkt)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/to_dateperiod.json b/docs/reference/esql/functions/kibana/definition/to_dateperiod.json index dc9176f4cc0b0..2940a08a7a100 100644 --- a/docs/reference/esql/functions/kibana/definition/to_dateperiod.json +++ b/docs/reference/esql/functions/kibana/definition/to_dateperiod.json @@ -43,5 +43,7 @@ ], "examples" : [ "row x = \"2024-01-01\"::datetime | eval y = x + \"3 DAYS\"::date_period, z = x - to_dateperiod(\"3 days\");" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/to_datetime.json b/docs/reference/esql/functions/kibana/definition/to_datetime.json index 032e8e1cbda34..8f9ecbd139d32 100644 --- a/docs/reference/esql/functions/kibana/definition/to_datetime.json +++ b/docs/reference/esql/functions/kibana/definition/to_datetime.json @@ -17,6 +17,18 @@ "variadic" : false, "returnType" : "date" }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Input value. The input can be a single- or multi-valued column or an expression." + } + ], + "variadic" : false, + "returnType" : "date" + }, { "params" : [ { @@ -93,5 +105,7 @@ "examples" : [ "ROW string = [\"1953-09-02T00:00:00.000Z\", \"1964-06-02T00:00:00.000Z\", \"1964-06-02 00:00:00\"]\n| EVAL datetime = TO_DATETIME(string)", "ROW int = [0, 1]\n| EVAL dt = TO_DATETIME(int)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/to_degrees.json b/docs/reference/esql/functions/kibana/definition/to_degrees.json index ea4b71d6c1419..6aefca5c5d4bc 100644 --- a/docs/reference/esql/functions/kibana/definition/to_degrees.json +++ b/docs/reference/esql/functions/kibana/definition/to_degrees.json @@ -55,5 +55,7 @@ ], "examples" : [ "ROW rad = [1.57, 3.14, 4.71]\n| EVAL deg = TO_DEGREES(rad)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/to_double.json b/docs/reference/esql/functions/kibana/definition/to_double.json index ae7e4832bfb3c..27565e616d6ed 100644 --- a/docs/reference/esql/functions/kibana/definition/to_double.json +++ b/docs/reference/esql/functions/kibana/definition/to_double.json @@ -139,5 +139,7 @@ ], "examples" : [ "ROW str1 = \"5.20128E11\", str2 = \"foo\"\n| EVAL dbl = TO_DOUBLE(\"520128000000\"), dbl1 = TO_DOUBLE(str1), dbl2 = TO_DOUBLE(str2)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/to_geopoint.json b/docs/reference/esql/functions/kibana/definition/to_geopoint.json index bf84008b556cc..e2ba492e39e9e 100644 --- a/docs/reference/esql/functions/kibana/definition/to_geopoint.json +++ b/docs/reference/esql/functions/kibana/definition/to_geopoint.json @@ -43,5 +43,7 @@ ], "examples" : [ "ROW wkt = \"POINT(42.97109630194 14.7552534413725)\"\n| EVAL pt = TO_GEOPOINT(wkt)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/to_geoshape.json b/docs/reference/esql/functions/kibana/definition/to_geoshape.json index 0f4749ed7d9ab..53316cc0f7b84 100644 --- a/docs/reference/esql/functions/kibana/definition/to_geoshape.json +++ b/docs/reference/esql/functions/kibana/definition/to_geoshape.json @@ -55,5 +55,7 @@ ], "examples" : [ "ROW wkt = \"POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))\"\n| EVAL geom = TO_GEOSHAPE(wkt)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/to_integer.json b/docs/reference/esql/functions/kibana/definition/to_integer.json index 5150d12936711..93691ac4e94ef 100644 --- a/docs/reference/esql/functions/kibana/definition/to_integer.json +++ b/docs/reference/esql/functions/kibana/definition/to_integer.json @@ -115,5 +115,7 @@ ], "examples" : [ "ROW long = [5013792, 2147483647, 501379200000]\n| EVAL int = TO_INTEGER(long)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/to_ip.json b/docs/reference/esql/functions/kibana/definition/to_ip.json index 36da766fd1ad0..dfc8e97d283f9 100644 --- a/docs/reference/esql/functions/kibana/definition/to_ip.json +++ b/docs/reference/esql/functions/kibana/definition/to_ip.json @@ -43,5 +43,7 @@ ], "examples" : [ "ROW str1 = \"1.1.1.1\", str2 = \"foo\"\n| EVAL ip1 = TO_IP(str1), ip2 = TO_IP(str2)\n| WHERE CIDR_MATCH(ip1, \"1.0.0.0/8\")" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/to_long.json b/docs/reference/esql/functions/kibana/definition/to_long.json index 5fd4bce34e7e0..eb1ce7220c3f9 100644 --- a/docs/reference/esql/functions/kibana/definition/to_long.json +++ b/docs/reference/esql/functions/kibana/definition/to_long.json @@ -52,6 +52,18 @@ "variadic" : false, "returnType" : "long" }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Input value. The input can be a single- or multi-valued column or an expression." + } + ], + "variadic" : false, + "returnType" : "long" + }, { "params" : [ { @@ -127,5 +139,7 @@ ], "examples" : [ "ROW str1 = \"2147483648\", str2 = \"2147483648.2\", str3 = \"foo\"\n| EVAL long1 = TO_LONG(str1), long2 = TO_LONG(str2), long3 = TO_LONG(str3)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/to_lower.json b/docs/reference/esql/functions/kibana/definition/to_lower.json index f1289f7b16dde..07bb057fe080d 100644 --- a/docs/reference/esql/functions/kibana/definition/to_lower.json +++ b/docs/reference/esql/functions/kibana/definition/to_lower.json @@ -26,10 +26,12 @@ } ], "variadic" : false, - "returnType" : "text" + "returnType" : "keyword" } ], "examples" : [ "ROW message = \"Some Text\"\n| EVAL message_lower = TO_LOWER(message)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/to_radians.json b/docs/reference/esql/functions/kibana/definition/to_radians.json index 773461074ed14..1b13ef22e23f0 100644 --- a/docs/reference/esql/functions/kibana/definition/to_radians.json +++ b/docs/reference/esql/functions/kibana/definition/to_radians.json @@ -55,5 +55,7 @@ ], "examples" : [ "ROW deg = [90.0, 180.0, 270.0]\n| EVAL rad = TO_RADIANS(deg)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/to_string.json b/docs/reference/esql/functions/kibana/definition/to_string.json index ea94171834908..1c86e81b31136 100644 --- a/docs/reference/esql/functions/kibana/definition/to_string.json +++ b/docs/reference/esql/functions/kibana/definition/to_string.json @@ -52,6 +52,18 @@ "variadic" : false, "returnType" : "keyword" }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Input value. The input can be a single- or multi-valued column or an expression." + } + ], + "variadic" : false, + "returnType" : "keyword" + }, { "params" : [ { @@ -176,5 +188,7 @@ "examples" : [ "ROW a=10\n| EVAL j = TO_STRING(a)", "ROW a=[10, 9, 8]\n| EVAL j = TO_STRING(a)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/to_timeduration.json b/docs/reference/esql/functions/kibana/definition/to_timeduration.json index 039de323044ed..eb7d4e0d5ccec 100644 --- a/docs/reference/esql/functions/kibana/definition/to_timeduration.json +++ b/docs/reference/esql/functions/kibana/definition/to_timeduration.json @@ -43,5 +43,7 @@ ], "examples" : [ "row x = \"2024-01-01\"::datetime | eval y = x + \"3 hours\"::time_duration, z = x - to_timeduration(\"3 hours\");" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/to_unsigned_long.json b/docs/reference/esql/functions/kibana/definition/to_unsigned_long.json index 5521241224d61..4a215b1ea97f3 100644 --- a/docs/reference/esql/functions/kibana/definition/to_unsigned_long.json +++ b/docs/reference/esql/functions/kibana/definition/to_unsigned_long.json @@ -103,5 +103,7 @@ ], "examples" : [ "ROW str1 = \"2147483648\", str2 = \"2147483648.2\", str3 = \"foo\"\n| EVAL long1 = TO_UNSIGNED_LONG(str1), long2 = TO_ULONG(str2), long3 = TO_UL(str3)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/to_upper.json b/docs/reference/esql/functions/kibana/definition/to_upper.json index ecfed0a29bcb9..caa9d563b08b1 100644 --- a/docs/reference/esql/functions/kibana/definition/to_upper.json +++ b/docs/reference/esql/functions/kibana/definition/to_upper.json @@ -26,10 +26,12 @@ } ], "variadic" : false, - "returnType" : "text" + "returnType" : "keyword" } ], "examples" : [ "ROW message = \"Some Text\"\n| EVAL message_upper = TO_UPPER(message)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/to_version.json b/docs/reference/esql/functions/kibana/definition/to_version.json index ff8b94dd63699..a77fc250c3a36 100644 --- a/docs/reference/esql/functions/kibana/definition/to_version.json +++ b/docs/reference/esql/functions/kibana/definition/to_version.json @@ -43,5 +43,7 @@ ], "examples" : [ "ROW v = TO_VERSION(\"1.2.3\")" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/top.json b/docs/reference/esql/functions/kibana/definition/top.json index c688bf5ea77c8..82bd80636152c 100644 --- a/docs/reference/esql/functions/kibana/definition/top.json +++ b/docs/reference/esql/functions/kibana/definition/top.json @@ -124,6 +124,30 @@ "variadic" : false, "returnType" : "ip" }, + { + "params" : [ + { + "name" : "field", + "type" : "keyword", + "optional" : false, + "description" : "The field to collect the top values for." + }, + { + "name" : "limit", + "type" : "integer", + "optional" : false, + "description" : "The maximum number of values to collect." + }, + { + "name" : "order", + "type" : "keyword", + "optional" : false, + "description" : "The order to calculate the top values. Either `asc` or `desc`." + } + ], + "variadic" : false, + "returnType" : "keyword" + }, { "params" : [ { @@ -147,9 +171,35 @@ ], "variadic" : false, "returnType" : "long" + }, + { + "params" : [ + { + "name" : "field", + "type" : "text", + "optional" : false, + "description" : "The field to collect the top values for." + }, + { + "name" : "limit", + "type" : "integer", + "optional" : false, + "description" : "The maximum number of values to collect." + }, + { + "name" : "order", + "type" : "keyword", + "optional" : false, + "description" : "The order to calculate the top values. Either `asc` or `desc`." + } + ], + "variadic" : false, + "returnType" : "keyword" } ], "examples" : [ "FROM employees\n| STATS top_salaries = TOP(salary, 3, \"desc\"), top_salary = MAX(salary)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/trim.json b/docs/reference/esql/functions/kibana/definition/trim.json index e19e09ed2320c..45805b3bfb054 100644 --- a/docs/reference/esql/functions/kibana/definition/trim.json +++ b/docs/reference/esql/functions/kibana/definition/trim.json @@ -26,10 +26,12 @@ } ], "variadic" : false, - "returnType" : "text" + "returnType" : "keyword" } ], "examples" : [ "ROW message = \" some text \", color = \" red \"\n| EVAL message = TRIM(message)\n| EVAL color = TRIM(color)" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/values.json b/docs/reference/esql/functions/kibana/definition/values.json index d9f37cd1ac83d..95ac402bb242a 100644 --- a/docs/reference/esql/functions/kibana/definition/values.json +++ b/docs/reference/esql/functions/kibana/definition/values.json @@ -28,6 +28,18 @@ "variadic" : false, "returnType" : "date" }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, { "params" : [ { @@ -98,7 +110,7 @@ } ], "variadic" : false, - "returnType" : "text" + "returnType" : "keyword" }, { "params" : [ @@ -115,5 +127,7 @@ ], "examples" : [ " FROM employees\n| EVAL first_letter = SUBSTRING(first_name, 0, 1)\n| STATS first_name=MV_SORT(VALUES(first_name)) BY first_letter\n| SORT first_letter" - ] + ], + "preview" : true, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/weighted_avg.json b/docs/reference/esql/functions/kibana/definition/weighted_avg.json index a04a5bff62bbb..a14af5345758e 100644 --- a/docs/reference/esql/functions/kibana/definition/weighted_avg.json +++ b/docs/reference/esql/functions/kibana/definition/weighted_avg.json @@ -169,5 +169,7 @@ ], "examples" : [ "FROM employees\n| STATS w_avg = WEIGHTED_AVG(salary, height) by languages\n| EVAL w_avg = ROUND(w_avg)\n| KEEP w_avg, languages\n| SORT languages" - ] + ], + "preview" : false, + "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/docs/bit_length.md b/docs/reference/esql/functions/kibana/docs/bit_length.md new file mode 100644 index 0000000000000..b1d8e24c4de76 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/bit_length.md @@ -0,0 +1,14 @@ + + +### BIT_LENGTH +Returns the bit length of a string. + +``` +FROM airports +| WHERE country == "India" +| KEEP city +| EVAL fn_length = LENGTH(city), fn_bit_length = BIT_LENGTH(city) +``` +Note: All strings are in UTF-8, so a single character can use multiple bytes. diff --git a/docs/reference/esql/functions/kibana/docs/byte_length.md b/docs/reference/esql/functions/kibana/docs/byte_length.md new file mode 100644 index 0000000000000..9cd4f87c9883b --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/byte_length.md @@ -0,0 +1,14 @@ + + +### BYTE_LENGTH +Returns the byte length of a string. + +``` +FROM airports +| WHERE country == "India" +| KEEP city +| EVAL fn_length = LENGTH(city), fn_byte_length = BYTE_LENGTH(city) +``` +Note: All strings are in UTF-8, so a single character can use multiple bytes. diff --git a/docs/reference/esql/functions/kibana/docs/categorize.md b/docs/reference/esql/functions/kibana/docs/categorize.md new file mode 100644 index 0000000000000..80c04b79084e9 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/categorize.md @@ -0,0 +1,11 @@ + + +### CATEGORIZE +Groups text messages into categories of similarly formatted text values. + +``` +FROM sample_data +| STATS count=COUNT() BY category=CATEGORIZE(message) +``` diff --git a/docs/reference/esql/functions/kibana/docs/date_format.md b/docs/reference/esql/functions/kibana/docs/date_format.md index 19aeb45e0d403..7c75f8eb566da 100644 --- a/docs/reference/esql/functions/kibana/docs/date_format.md +++ b/docs/reference/esql/functions/kibana/docs/date_format.md @@ -8,5 +8,5 @@ Returns a string representation of a date, in the provided format. ``` FROM employees | KEEP first_name, last_name, hire_date -| EVAL hired = DATE_FORMAT("YYYY-MM-dd", hire_date) +| EVAL hired = DATE_FORMAT("yyyy-MM-dd", hire_date) ``` diff --git a/docs/reference/esql/functions/kibana/docs/hash.md b/docs/reference/esql/functions/kibana/docs/hash.md new file mode 100644 index 0000000000000..9826e80ec5bec --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/hash.md @@ -0,0 +1,7 @@ + + +### HASH +Computes the hash of the input using various algorithms such as MD5, SHA, SHA-224, SHA-256, SHA-384, SHA-512. + diff --git a/docs/reference/esql/functions/kibana/docs/hypot.md b/docs/reference/esql/functions/kibana/docs/hypot.md new file mode 100644 index 0000000000000..f0cbea6b88e55 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/hypot.md @@ -0,0 +1,12 @@ + + +### HYPOT +Returns the hypotenuse of two numbers. The input can be any numeric values, the return value is always a double. +Hypotenuses of infinities are null. + +``` +ROW a = 3.0, b = 4.0 +| EVAL c = HYPOT(a, b) +``` diff --git a/docs/reference/esql/functions/kibana/docs/kql.md b/docs/reference/esql/functions/kibana/docs/kql.md new file mode 100644 index 0000000000000..0ba419c1cd032 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/kql.md @@ -0,0 +1,14 @@ + + +### KQL +Performs a KQL query. Returns true if the provided KQL query string matches the row. + +``` +FROM books +| WHERE KQL("author: Faulkner") +| KEEP book_no, author +| SORT book_no +| LIMIT 5; +``` diff --git a/docs/reference/esql/functions/kibana/docs/length.md b/docs/reference/esql/functions/kibana/docs/length.md index 19e3533e0ddfb..aed76ee14cedb 100644 --- a/docs/reference/esql/functions/kibana/docs/length.md +++ b/docs/reference/esql/functions/kibana/docs/length.md @@ -6,7 +6,9 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ Returns the character length of a string. ``` -FROM employees -| KEEP first_name, last_name -| EVAL fn_length = LENGTH(first_name) +FROM airports +| WHERE country == "India" +| KEEP city +| EVAL fn_length = LENGTH(city) ``` +Note: All strings are in UTF-8, so a single character can use multiple bytes. diff --git a/docs/reference/esql/functions/kibana/docs/like.md b/docs/reference/esql/functions/kibana/docs/like.md index 4c400bdc65479..ea2ac11b6f4b9 100644 --- a/docs/reference/esql/functions/kibana/docs/like.md +++ b/docs/reference/esql/functions/kibana/docs/like.md @@ -15,6 +15,6 @@ The following wildcard characters are supported: ``` FROM employees -| WHERE first_name LIKE "?b*" +| WHERE first_name LIKE """?b*""" | KEEP first_name, last_name ``` diff --git a/docs/reference/esql/functions/kibana/docs/match.md b/docs/reference/esql/functions/kibana/docs/match.md new file mode 100644 index 0000000000000..adf6de91c90f1 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/match.md @@ -0,0 +1,14 @@ + + +### MATCH +Performs a <> on the specified field. Returns true if the provided query matches the row. + +``` +FROM books +| WHERE MATCH(author, "Faulkner") +| KEEP book_no, author +| SORT book_no +| LIMIT 5; +``` diff --git a/docs/reference/esql/functions/kibana/docs/match_operator.md b/docs/reference/esql/functions/kibana/docs/match_operator.md new file mode 100644 index 0000000000000..b0b6196798087 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/match_operator.md @@ -0,0 +1,14 @@ + + +### MATCH_OPERATOR +Performs a <> on the specified field. Returns true if the provided query matches the row. + +``` +FROM books +| WHERE MATCH(author, "Faulkner") +| KEEP book_no, author +| SORT book_no +| LIMIT 5; +``` diff --git a/docs/reference/esql/functions/kibana/docs/mv_first.md b/docs/reference/esql/functions/kibana/docs/mv_first.md index 4faea6edd9162..c50ed7d764020 100644 --- a/docs/reference/esql/functions/kibana/docs/mv_first.md +++ b/docs/reference/esql/functions/kibana/docs/mv_first.md @@ -7,12 +7,6 @@ Converts a multivalued expression into a single valued column containing the first value. This is most useful when reading from a function that emits multivalued columns in a known order like <>. -The order that <> are read from -underlying storage is not guaranteed. It is *frequently* ascending, but don't -rely on that. If you need the minimum value use <> instead of -`MV_FIRST`. `MV_MIN` has optimizations for sorted values so there isn't a -performance benefit to `MV_FIRST`. - ``` ROW a="foo;bar;baz" | EVAL first_a = MV_FIRST(SPLIT(a, ";")) diff --git a/docs/reference/esql/functions/kibana/docs/mv_last.md b/docs/reference/esql/functions/kibana/docs/mv_last.md index a8c3bf25eb51b..eeefd929c1359 100644 --- a/docs/reference/esql/functions/kibana/docs/mv_last.md +++ b/docs/reference/esql/functions/kibana/docs/mv_last.md @@ -7,12 +7,6 @@ Converts a multivalue expression into a single valued column containing the last value. This is most useful when reading from a function that emits multivalued columns in a known order like <>. -The order that <> are read from -underlying storage is not guaranteed. It is *frequently* ascending, but don't -rely on that. If you need the maximum value use <> instead of -`MV_LAST`. `MV_MAX` has optimizations for sorted values so there isn't a -performance benefit to `MV_LAST`. - ``` ROW a="foo;bar;baz" | EVAL last_a = MV_LAST(SPLIT(a, ";")) diff --git a/docs/reference/esql/functions/kibana/docs/mv_slice.md b/docs/reference/esql/functions/kibana/docs/mv_slice.md index 3daf0de930a7f..bba7a219960ef 100644 --- a/docs/reference/esql/functions/kibana/docs/mv_slice.md +++ b/docs/reference/esql/functions/kibana/docs/mv_slice.md @@ -4,6 +4,8 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ ### MV_SLICE Returns a subset of the multivalued field using the start and end index values. +This is most useful when reading from a function that emits multivalued columns +in a known order like <> or <>. ``` row a = [1, 2, 2, 3] diff --git a/docs/reference/esql/functions/kibana/docs/qstr.md b/docs/reference/esql/functions/kibana/docs/qstr.md new file mode 100644 index 0000000000000..7df5a2fe08a9d --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/qstr.md @@ -0,0 +1,14 @@ + + +### QSTR +Performs a <>. Returns true if the provided query string matches the row. + +``` +FROM books +| WHERE QSTR("author: Faulkner") +| KEEP book_no, author +| SORT book_no +| LIMIT 5; +``` diff --git a/docs/reference/esql/functions/kibana/docs/reverse.md b/docs/reference/esql/functions/kibana/docs/reverse.md new file mode 100644 index 0000000000000..01ad271e45fa0 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/reverse.md @@ -0,0 +1,13 @@ + + +### REVERSE +Returns a new string representing the input string in reverse order. + +``` +ROW message = "Some Text" | EVAL message_reversed = REVERSE(message); +``` +Note: If Elasticsearch is running with a JDK version less than 20 then this will not properly reverse Grapheme Clusters. +Elastic Cloud and the JDK bundled with Elasticsearch all use newer JDKs. But if you've explicitly shifted to an older jdk +then you'll see things like "👍🏽😊" be reversed to "🏽👍😊" instead of the correct "😊👍🏽". diff --git a/docs/reference/esql/functions/kibana/docs/rlike.md b/docs/reference/esql/functions/kibana/docs/rlike.md index ed94553e7e44f..95b57799ffe29 100644 --- a/docs/reference/esql/functions/kibana/docs/rlike.md +++ b/docs/reference/esql/functions/kibana/docs/rlike.md @@ -10,6 +10,6 @@ expression. The right-hand side of the operator represents the pattern. ``` FROM employees -| WHERE first_name RLIKE ".leja.*" +| WHERE first_name RLIKE """.leja.*""" | KEEP first_name, last_name ``` diff --git a/docs/reference/esql/functions/kibana/docs/st_envelope.md b/docs/reference/esql/functions/kibana/docs/st_envelope.md new file mode 100644 index 0000000000000..5f4c3e4809a82 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/st_envelope.md @@ -0,0 +1,13 @@ + + +### ST_ENVELOPE +Determines the minimum bounding box of the supplied geometry. + +``` +FROM airport_city_boundaries +| WHERE abbrev == "CPH" +| EVAL envelope = ST_ENVELOPE(city_boundary) +| KEEP abbrev, airport, envelope +``` diff --git a/docs/reference/esql/functions/kibana/docs/st_extent_agg.md b/docs/reference/esql/functions/kibana/docs/st_extent_agg.md new file mode 100644 index 0000000000000..a2e307c5b2c55 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/st_extent_agg.md @@ -0,0 +1,12 @@ + + +### ST_EXTENT_AGG +Calculate the spatial extent over a field with geometry type. Returns a bounding box for all values of the field. + +``` +FROM airports +| WHERE country == "India" +| STATS extent = ST_EXTENT_AGG(location) +``` diff --git a/docs/reference/esql/functions/kibana/docs/st_xmax.md b/docs/reference/esql/functions/kibana/docs/st_xmax.md new file mode 100644 index 0000000000000..bbde89df76fd0 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/st_xmax.md @@ -0,0 +1,15 @@ + + +### ST_XMAX +Extracts the maximum value of the `x` coordinates from the supplied geometry. +If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the maximum `longitude` value. + +``` +FROM airport_city_boundaries +| WHERE abbrev == "CPH" +| EVAL envelope = ST_ENVELOPE(city_boundary) +| EVAL xmin = ST_XMIN(envelope), xmax = ST_XMAX(envelope), ymin = ST_YMIN(envelope), ymax = ST_YMAX(envelope) +| KEEP abbrev, airport, xmin, xmax, ymin, ymax +``` diff --git a/docs/reference/esql/functions/kibana/docs/st_xmin.md b/docs/reference/esql/functions/kibana/docs/st_xmin.md new file mode 100644 index 0000000000000..1a6cee7dcfd62 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/st_xmin.md @@ -0,0 +1,15 @@ + + +### ST_XMIN +Extracts the minimum value of the `x` coordinates from the supplied geometry. +If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the minimum `longitude` value. + +``` +FROM airport_city_boundaries +| WHERE abbrev == "CPH" +| EVAL envelope = ST_ENVELOPE(city_boundary) +| EVAL xmin = ST_XMIN(envelope), xmax = ST_XMAX(envelope), ymin = ST_YMIN(envelope), ymax = ST_YMAX(envelope) +| KEEP abbrev, airport, xmin, xmax, ymin, ymax +``` diff --git a/docs/reference/esql/functions/kibana/docs/st_ymax.md b/docs/reference/esql/functions/kibana/docs/st_ymax.md new file mode 100644 index 0000000000000..61c9b6c288ca5 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/st_ymax.md @@ -0,0 +1,15 @@ + + +### ST_YMAX +Extracts the maximum value of the `y` coordinates from the supplied geometry. +If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the maximum `latitude` value. + +``` +FROM airport_city_boundaries +| WHERE abbrev == "CPH" +| EVAL envelope = ST_ENVELOPE(city_boundary) +| EVAL xmin = ST_XMIN(envelope), xmax = ST_XMAX(envelope), ymin = ST_YMIN(envelope), ymax = ST_YMAX(envelope) +| KEEP abbrev, airport, xmin, xmax, ymin, ymax +``` diff --git a/docs/reference/esql/functions/kibana/docs/st_ymin.md b/docs/reference/esql/functions/kibana/docs/st_ymin.md new file mode 100644 index 0000000000000..f5817f10f20a5 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/st_ymin.md @@ -0,0 +1,15 @@ + + +### ST_YMIN +Extracts the minimum value of the `y` coordinates from the supplied geometry. +If the geometry is of type `geo_point` or `geo_shape` this is equivalent to extracting the minimum `latitude` value. + +``` +FROM airport_city_boundaries +| WHERE abbrev == "CPH" +| EVAL envelope = ST_ENVELOPE(city_boundary) +| EVAL xmin = ST_XMIN(envelope), xmax = ST_XMAX(envelope), ymin = ST_YMIN(envelope), ymax = ST_YMAX(envelope) +| KEEP abbrev, airport, xmin, xmax, ymin, ymax +``` diff --git a/docs/reference/esql/functions/kibana/docs/std_dev.md b/docs/reference/esql/functions/kibana/docs/std_dev.md new file mode 100644 index 0000000000000..a6afca7b8f6b3 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/std_dev.md @@ -0,0 +1,11 @@ + + +### STD_DEV +The standard deviation of a numeric field. + +``` +FROM employees +| STATS STD_DEV(height) +``` diff --git a/docs/reference/esql/functions/kibana/docs/term.md b/docs/reference/esql/functions/kibana/docs/term.md new file mode 100644 index 0000000000000..ffecd26d737f7 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/term.md @@ -0,0 +1,13 @@ + + +### TERM +Performs a Term query on the specified field. Returns true if the provided term matches the row. + +``` +FROM books +| WHERE TERM(author, "gabriel") +| KEEP book_no, title +| LIMIT 3; +``` diff --git a/docs/reference/esql/functions/kibana/inline_cast.json b/docs/reference/esql/functions/kibana/inline_cast.json index f1aa283c52e95..9f663c8d0d6a3 100644 --- a/docs/reference/esql/functions/kibana/inline_cast.json +++ b/docs/reference/esql/functions/kibana/inline_cast.json @@ -3,6 +3,7 @@ "boolean" : "to_boolean", "cartesian_point" : "to_cartesianpoint", "cartesian_shape" : "to_cartesianshape", + "date_nanos" : "to_date_nanos", "date_period" : "to_dateperiod", "datetime" : "to_datetime", "double" : "to_double", @@ -14,7 +15,6 @@ "keyword" : "to_string", "long" : "to_long", "string" : "to_string", - "text" : "to_string", "time_duration" : "to_timeduration", "unsigned_long" : "to_unsigned_long", "version" : "to_version" diff --git a/docs/reference/esql/functions/layout/bit_length.asciidoc b/docs/reference/esql/functions/layout/bit_length.asciidoc new file mode 100644 index 0000000000000..00a7206f3ceda --- /dev/null +++ b/docs/reference/esql/functions/layout/bit_length.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-bit_length]] +=== `BIT_LENGTH` + +*Syntax* + +[.text-center] +image::esql/functions/signature/bit_length.svg[Embedded,opts=inline] + +include::../parameters/bit_length.asciidoc[] +include::../description/bit_length.asciidoc[] +include::../types/bit_length.asciidoc[] +include::../examples/bit_length.asciidoc[] diff --git a/docs/reference/esql/functions/layout/byte_length.asciidoc b/docs/reference/esql/functions/layout/byte_length.asciidoc new file mode 100644 index 0000000000000..56dc341264e0f --- /dev/null +++ b/docs/reference/esql/functions/layout/byte_length.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-byte_length]] +=== `BYTE_LENGTH` + +*Syntax* + +[.text-center] +image::esql/functions/signature/byte_length.svg[Embedded,opts=inline] + +include::../parameters/byte_length.asciidoc[] +include::../description/byte_length.asciidoc[] +include::../types/byte_length.asciidoc[] +include::../examples/byte_length.asciidoc[] diff --git a/docs/reference/esql/functions/layout/categorize.asciidoc b/docs/reference/esql/functions/layout/categorize.asciidoc new file mode 100644 index 0000000000000..2ddc6bb31fae0 --- /dev/null +++ b/docs/reference/esql/functions/layout/categorize.asciidoc @@ -0,0 +1,17 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-categorize]] +=== `CATEGORIZE` + +preview::["Do not use `VALUES` on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] + +*Syntax* + +[.text-center] +image::esql/functions/signature/categorize.svg[Embedded,opts=inline] + +include::../parameters/categorize.asciidoc[] +include::../description/categorize.asciidoc[] +include::../types/categorize.asciidoc[] +include::../examples/categorize.asciidoc[] diff --git a/docs/reference/esql/functions/layout/hash.asciidoc b/docs/reference/esql/functions/layout/hash.asciidoc new file mode 100644 index 0000000000000..27c55ada6319b --- /dev/null +++ b/docs/reference/esql/functions/layout/hash.asciidoc @@ -0,0 +1,14 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-hash]] +=== `HASH` + +*Syntax* + +[.text-center] +image::esql/functions/signature/hash.svg[Embedded,opts=inline] + +include::../parameters/hash.asciidoc[] +include::../description/hash.asciidoc[] +include::../types/hash.asciidoc[] diff --git a/docs/reference/esql/functions/layout/hypot.asciidoc b/docs/reference/esql/functions/layout/hypot.asciidoc new file mode 100644 index 0000000000000..84376a9f15908 --- /dev/null +++ b/docs/reference/esql/functions/layout/hypot.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-hypot]] +=== `HYPOT` + +*Syntax* + +[.text-center] +image::esql/functions/signature/hypot.svg[Embedded,opts=inline] + +include::../parameters/hypot.asciidoc[] +include::../description/hypot.asciidoc[] +include::../types/hypot.asciidoc[] +include::../examples/hypot.asciidoc[] diff --git a/docs/reference/esql/functions/layout/kql.asciidoc b/docs/reference/esql/functions/layout/kql.asciidoc new file mode 100644 index 0000000000000..ea9e4871f5e53 --- /dev/null +++ b/docs/reference/esql/functions/layout/kql.asciidoc @@ -0,0 +1,17 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-kql]] +=== `KQL` + +preview::["Do not use `VALUES` on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] + +*Syntax* + +[.text-center] +image::esql/functions/signature/kql.svg[Embedded,opts=inline] + +include::../parameters/kql.asciidoc[] +include::../description/kql.asciidoc[] +include::../types/kql.asciidoc[] +include::../examples/kql.asciidoc[] diff --git a/docs/reference/esql/functions/layout/match.asciidoc b/docs/reference/esql/functions/layout/match.asciidoc new file mode 100644 index 0000000000000..ecd9f3901f894 --- /dev/null +++ b/docs/reference/esql/functions/layout/match.asciidoc @@ -0,0 +1,17 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-match]] +=== `MATCH` + +preview::["Do not use `VALUES` on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] + +*Syntax* + +[.text-center] +image::esql/functions/signature/match.svg[Embedded,opts=inline] + +include::../parameters/match.asciidoc[] +include::../description/match.asciidoc[] +include::../types/match.asciidoc[] +include::../examples/match.asciidoc[] diff --git a/docs/reference/esql/functions/layout/qstr.asciidoc b/docs/reference/esql/functions/layout/qstr.asciidoc new file mode 100644 index 0000000000000..b46c6498673b5 --- /dev/null +++ b/docs/reference/esql/functions/layout/qstr.asciidoc @@ -0,0 +1,17 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-qstr]] +=== `QSTR` + +preview::["Do not use `VALUES` on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] + +*Syntax* + +[.text-center] +image::esql/functions/signature/qstr.svg[Embedded,opts=inline] + +include::../parameters/qstr.asciidoc[] +include::../description/qstr.asciidoc[] +include::../types/qstr.asciidoc[] +include::../examples/qstr.asciidoc[] diff --git a/docs/reference/esql/functions/layout/reverse.asciidoc b/docs/reference/esql/functions/layout/reverse.asciidoc new file mode 100644 index 0000000000000..99c236d63492e --- /dev/null +++ b/docs/reference/esql/functions/layout/reverse.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-reverse]] +=== `REVERSE` + +*Syntax* + +[.text-center] +image::esql/functions/signature/reverse.svg[Embedded,opts=inline] + +include::../parameters/reverse.asciidoc[] +include::../description/reverse.asciidoc[] +include::../types/reverse.asciidoc[] +include::../examples/reverse.asciidoc[] diff --git a/docs/reference/esql/functions/layout/st_envelope.asciidoc b/docs/reference/esql/functions/layout/st_envelope.asciidoc new file mode 100644 index 0000000000000..a20d4275e0c9f --- /dev/null +++ b/docs/reference/esql/functions/layout/st_envelope.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-st_envelope]] +=== `ST_ENVELOPE` + +*Syntax* + +[.text-center] +image::esql/functions/signature/st_envelope.svg[Embedded,opts=inline] + +include::../parameters/st_envelope.asciidoc[] +include::../description/st_envelope.asciidoc[] +include::../types/st_envelope.asciidoc[] +include::../examples/st_envelope.asciidoc[] diff --git a/docs/reference/esql/functions/layout/st_extent_agg.asciidoc b/docs/reference/esql/functions/layout/st_extent_agg.asciidoc new file mode 100644 index 0000000000000..946bef661e70c --- /dev/null +++ b/docs/reference/esql/functions/layout/st_extent_agg.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-st_extent_agg]] +=== `ST_EXTENT_AGG` + +*Syntax* + +[.text-center] +image::esql/functions/signature/st_extent_agg.svg[Embedded,opts=inline] + +include::../parameters/st_extent_agg.asciidoc[] +include::../description/st_extent_agg.asciidoc[] +include::../types/st_extent_agg.asciidoc[] +include::../examples/st_extent_agg.asciidoc[] diff --git a/docs/reference/esql/functions/layout/st_xmax.asciidoc b/docs/reference/esql/functions/layout/st_xmax.asciidoc new file mode 100644 index 0000000000000..b0c5e7695521e --- /dev/null +++ b/docs/reference/esql/functions/layout/st_xmax.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-st_xmax]] +=== `ST_XMAX` + +*Syntax* + +[.text-center] +image::esql/functions/signature/st_xmax.svg[Embedded,opts=inline] + +include::../parameters/st_xmax.asciidoc[] +include::../description/st_xmax.asciidoc[] +include::../types/st_xmax.asciidoc[] +include::../examples/st_xmax.asciidoc[] diff --git a/docs/reference/esql/functions/layout/st_xmin.asciidoc b/docs/reference/esql/functions/layout/st_xmin.asciidoc new file mode 100644 index 0000000000000..55fbad88c4cf0 --- /dev/null +++ b/docs/reference/esql/functions/layout/st_xmin.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-st_xmin]] +=== `ST_XMIN` + +*Syntax* + +[.text-center] +image::esql/functions/signature/st_xmin.svg[Embedded,opts=inline] + +include::../parameters/st_xmin.asciidoc[] +include::../description/st_xmin.asciidoc[] +include::../types/st_xmin.asciidoc[] +include::../examples/st_xmin.asciidoc[] diff --git a/docs/reference/esql/functions/layout/st_ymax.asciidoc b/docs/reference/esql/functions/layout/st_ymax.asciidoc new file mode 100644 index 0000000000000..e1022de4ba664 --- /dev/null +++ b/docs/reference/esql/functions/layout/st_ymax.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-st_ymax]] +=== `ST_YMAX` + +*Syntax* + +[.text-center] +image::esql/functions/signature/st_ymax.svg[Embedded,opts=inline] + +include::../parameters/st_ymax.asciidoc[] +include::../description/st_ymax.asciidoc[] +include::../types/st_ymax.asciidoc[] +include::../examples/st_ymax.asciidoc[] diff --git a/docs/reference/esql/functions/layout/st_ymin.asciidoc b/docs/reference/esql/functions/layout/st_ymin.asciidoc new file mode 100644 index 0000000000000..65511e1925e27 --- /dev/null +++ b/docs/reference/esql/functions/layout/st_ymin.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-st_ymin]] +=== `ST_YMIN` + +*Syntax* + +[.text-center] +image::esql/functions/signature/st_ymin.svg[Embedded,opts=inline] + +include::../parameters/st_ymin.asciidoc[] +include::../description/st_ymin.asciidoc[] +include::../types/st_ymin.asciidoc[] +include::../examples/st_ymin.asciidoc[] diff --git a/docs/reference/esql/functions/layout/std_dev.asciidoc b/docs/reference/esql/functions/layout/std_dev.asciidoc new file mode 100644 index 0000000000000..a7a34b1331d17 --- /dev/null +++ b/docs/reference/esql/functions/layout/std_dev.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-std_dev]] +=== `STD_DEV` + +*Syntax* + +[.text-center] +image::esql/functions/signature/std_dev.svg[Embedded,opts=inline] + +include::../parameters/std_dev.asciidoc[] +include::../description/std_dev.asciidoc[] +include::../types/std_dev.asciidoc[] +include::../examples/std_dev.asciidoc[] diff --git a/docs/reference/esql/functions/layout/term.asciidoc b/docs/reference/esql/functions/layout/term.asciidoc new file mode 100644 index 0000000000000..2007535dc2c74 --- /dev/null +++ b/docs/reference/esql/functions/layout/term.asciidoc @@ -0,0 +1,17 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-term]] +=== `TERM` + +preview::["Do not use `VALUES` on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] + +*Syntax* + +[.text-center] +image::esql/functions/signature/term.svg[Embedded,opts=inline] + +include::../parameters/term.asciidoc[] +include::../description/term.asciidoc[] +include::../types/term.asciidoc[] +include::../examples/term.asciidoc[] diff --git a/docs/reference/esql/functions/like.asciidoc b/docs/reference/esql/functions/like.asciidoc index 2298617be5699..a569896bc3c1e 100644 --- a/docs/reference/esql/functions/like.asciidoc +++ b/docs/reference/esql/functions/like.asciidoc @@ -23,4 +23,20 @@ include::{esql-specs}/docs.csv-spec[tag=like] |=== include::{esql-specs}/docs.csv-spec[tag=like-result] |=== + +Matching the exact characters `*` and `.` will require escaping. +The escape character is backslash `\`. Since also backslash is a special character in string literals, +it will require further escaping. + +[source.merge.styled,esql] +---- +include::{esql-specs}/string.csv-spec[tag=likeEscapingSingleQuotes] +---- + +To reduce the overhead of escaping, we suggest using triple quotes strings `"""` + +[source.merge.styled,esql] +---- +include::{esql-specs}/string.csv-spec[tag=likeEscapingTripleQuotes] +---- // end::body[] diff --git a/docs/reference/esql/functions/math-functions.asciidoc b/docs/reference/esql/functions/math-functions.asciidoc index e311208795533..9fedfa57f50c5 100644 --- a/docs/reference/esql/functions/math-functions.asciidoc +++ b/docs/reference/esql/functions/math-functions.asciidoc @@ -20,6 +20,7 @@ * <> * <> * <> +* <> * <> * <> * <> @@ -46,6 +47,7 @@ include::layout/cosh.asciidoc[] include::layout/e.asciidoc[] include::layout/exp.asciidoc[] include::layout/floor.asciidoc[] +include::layout/hypot.asciidoc[] include::layout/log.asciidoc[] include::layout/log10.asciidoc[] include::layout/pi.asciidoc[] diff --git a/docs/reference/esql/functions/mv-functions.asciidoc b/docs/reference/esql/functions/mv-functions.asciidoc index 4093e44c16911..3da0249c9c0db 100644 --- a/docs/reference/esql/functions/mv-functions.asciidoc +++ b/docs/reference/esql/functions/mv-functions.asciidoc @@ -19,6 +19,7 @@ * <> * <> * <> +* <> * <> * <> * <> @@ -37,6 +38,7 @@ include::layout/mv_max.asciidoc[] include::layout/mv_median.asciidoc[] include::layout/mv_median_absolute_deviation.asciidoc[] include::layout/mv_min.asciidoc[] +include::layout/mv_percentile.asciidoc[] include::layout/mv_pseries_weighted_sum.asciidoc[] include::layout/mv_slice.asciidoc[] include::layout/mv_sort.asciidoc[] diff --git a/docs/reference/esql/functions/operators.asciidoc b/docs/reference/esql/functions/operators.asciidoc index ee344a52687c2..a1a2226335e9b 100644 --- a/docs/reference/esql/functions/operators.asciidoc +++ b/docs/reference/esql/functions/operators.asciidoc @@ -16,6 +16,7 @@ Boolean operators for comparing against one or multiple expressions. * <> * <> * <> +* experimental:[] <> // end::op_list[] include::binary.asciidoc[] @@ -26,3 +27,4 @@ include::cast.asciidoc[] include::in.asciidoc[] include::like.asciidoc[] include::rlike.asciidoc[] +include::search.asciidoc[] diff --git a/docs/reference/esql/functions/parameters/bit_length.asciidoc b/docs/reference/esql/functions/parameters/bit_length.asciidoc new file mode 100644 index 0000000000000..7bb8c080ce4a1 --- /dev/null +++ b/docs/reference/esql/functions/parameters/bit_length.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`string`:: +String expression. If `null`, the function returns `null`. diff --git a/docs/reference/esql/functions/parameters/byte_length.asciidoc b/docs/reference/esql/functions/parameters/byte_length.asciidoc new file mode 100644 index 0000000000000..7bb8c080ce4a1 --- /dev/null +++ b/docs/reference/esql/functions/parameters/byte_length.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`string`:: +String expression. If `null`, the function returns `null`. diff --git a/docs/reference/esql/functions/parameters/categorize.asciidoc b/docs/reference/esql/functions/parameters/categorize.asciidoc new file mode 100644 index 0000000000000..42629ebfc7d9f --- /dev/null +++ b/docs/reference/esql/functions/parameters/categorize.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`field`:: +Expression to categorize diff --git a/docs/reference/esql/functions/parameters/hash.asciidoc b/docs/reference/esql/functions/parameters/hash.asciidoc new file mode 100644 index 0000000000000..d47a82d4ab214 --- /dev/null +++ b/docs/reference/esql/functions/parameters/hash.asciidoc @@ -0,0 +1,9 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`algorithm`:: +Hash algorithm to use. + +`input`:: +Input to hash. diff --git a/docs/reference/esql/functions/parameters/hypot.asciidoc b/docs/reference/esql/functions/parameters/hypot.asciidoc new file mode 100644 index 0000000000000..9d6c7d50c7bec --- /dev/null +++ b/docs/reference/esql/functions/parameters/hypot.asciidoc @@ -0,0 +1,9 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`number1`:: +Numeric expression. If `null`, the function returns `null`. + +`number2`:: +Numeric expression. If `null`, the function returns `null`. diff --git a/docs/reference/esql/functions/parameters/kql.asciidoc b/docs/reference/esql/functions/parameters/kql.asciidoc new file mode 100644 index 0000000000000..6fb69323ff73c --- /dev/null +++ b/docs/reference/esql/functions/parameters/kql.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`query`:: +Query string in KQL query string format. diff --git a/docs/reference/esql/functions/parameters/match.asciidoc b/docs/reference/esql/functions/parameters/match.asciidoc new file mode 100644 index 0000000000000..46f6acad9e128 --- /dev/null +++ b/docs/reference/esql/functions/parameters/match.asciidoc @@ -0,0 +1,9 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`field`:: +Field that the query will target. + +`query`:: +Value to find in the provided field. diff --git a/docs/reference/esql/functions/parameters/qstr.asciidoc b/docs/reference/esql/functions/parameters/qstr.asciidoc new file mode 100644 index 0000000000000..e51096084f2f3 --- /dev/null +++ b/docs/reference/esql/functions/parameters/qstr.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`query`:: +Query string in Lucene query string format. diff --git a/docs/reference/esql/functions/parameters/reverse.asciidoc b/docs/reference/esql/functions/parameters/reverse.asciidoc new file mode 100644 index 0000000000000..d56d115662491 --- /dev/null +++ b/docs/reference/esql/functions/parameters/reverse.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`str`:: +String expression. If `null`, the function returns `null`. diff --git a/docs/reference/esql/functions/parameters/st_envelope.asciidoc b/docs/reference/esql/functions/parameters/st_envelope.asciidoc new file mode 100644 index 0000000000000..a31c6a85de367 --- /dev/null +++ b/docs/reference/esql/functions/parameters/st_envelope.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`geometry`:: +Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`. diff --git a/docs/reference/esql/functions/parameters/st_extent_agg.asciidoc b/docs/reference/esql/functions/parameters/st_extent_agg.asciidoc new file mode 100644 index 0000000000000..8903aa1a472a3 --- /dev/null +++ b/docs/reference/esql/functions/parameters/st_extent_agg.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`field`:: + diff --git a/docs/reference/esql/functions/parameters/st_xmax.asciidoc b/docs/reference/esql/functions/parameters/st_xmax.asciidoc new file mode 100644 index 0000000000000..788f3485af297 --- /dev/null +++ b/docs/reference/esql/functions/parameters/st_xmax.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`point`:: +Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`. diff --git a/docs/reference/esql/functions/parameters/st_xmin.asciidoc b/docs/reference/esql/functions/parameters/st_xmin.asciidoc new file mode 100644 index 0000000000000..788f3485af297 --- /dev/null +++ b/docs/reference/esql/functions/parameters/st_xmin.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`point`:: +Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`. diff --git a/docs/reference/esql/functions/parameters/st_ymax.asciidoc b/docs/reference/esql/functions/parameters/st_ymax.asciidoc new file mode 100644 index 0000000000000..788f3485af297 --- /dev/null +++ b/docs/reference/esql/functions/parameters/st_ymax.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`point`:: +Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`. diff --git a/docs/reference/esql/functions/parameters/st_ymin.asciidoc b/docs/reference/esql/functions/parameters/st_ymin.asciidoc new file mode 100644 index 0000000000000..788f3485af297 --- /dev/null +++ b/docs/reference/esql/functions/parameters/st_ymin.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`point`:: +Expression of type `geo_point`, `geo_shape`, `cartesian_point` or `cartesian_shape`. If `null`, the function returns `null`. diff --git a/docs/reference/esql/functions/parameters/std_dev.asciidoc b/docs/reference/esql/functions/parameters/std_dev.asciidoc new file mode 100644 index 0000000000000..91c56709d182a --- /dev/null +++ b/docs/reference/esql/functions/parameters/std_dev.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`number`:: + diff --git a/docs/reference/esql/functions/parameters/term.asciidoc b/docs/reference/esql/functions/parameters/term.asciidoc new file mode 100644 index 0000000000000..edba8625d04c5 --- /dev/null +++ b/docs/reference/esql/functions/parameters/term.asciidoc @@ -0,0 +1,9 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`field`:: +Field that the query will target. + +`query`:: +Term you wish to find in the provided field. diff --git a/docs/reference/esql/functions/rlike.asciidoc b/docs/reference/esql/functions/rlike.asciidoc index 031594ae403da..f6009b2c49528 100644 --- a/docs/reference/esql/functions/rlike.asciidoc +++ b/docs/reference/esql/functions/rlike.asciidoc @@ -18,4 +18,20 @@ include::{esql-specs}/docs.csv-spec[tag=rlike] |=== include::{esql-specs}/docs.csv-spec[tag=rlike-result] |=== + +Matching special characters (eg. `.`, `*`, `(`...) will require escaping. +The escape character is backslash `\`. Since also backslash is a special character in string literals, +it will require further escaping. + +[source.merge.styled,esql] +---- +include::{esql-specs}/string.csv-spec[tag=rlikeEscapingSingleQuotes] +---- + +To reduce the overhead of escaping, we suggest using triple quotes strings `"""` + +[source.merge.styled,esql] +---- +include::{esql-specs}/string.csv-spec[tag=rlikeEscapingTripleQuotes] +---- // end::body[] diff --git a/docs/reference/esql/functions/search-functions.asciidoc b/docs/reference/esql/functions/search-functions.asciidoc new file mode 100644 index 0000000000000..238813c382c8c --- /dev/null +++ b/docs/reference/esql/functions/search-functions.asciidoc @@ -0,0 +1,24 @@ +[[esql-search-functions]] +==== {esql} Full-text search functions + +++++ +Full-text Search functions +++++ + +Full text functions are used to search for text in fields. +<> is used to analyze the query before it is searched. + +Full text functions can be used to match <>. +A multivalued field that contains a value that matches a full text query is considered to match the query. + +See <> for information on the limitations of full text search. + +{esql} supports these full-text search functions: + +// tag::search_list[] +* experimental:[] <> +* experimental:[] <> +// end::search_list[] + +include::layout/match.asciidoc[] +include::layout/qstr.asciidoc[] diff --git a/docs/reference/esql/functions/search.asciidoc b/docs/reference/esql/functions/search.asciidoc new file mode 100644 index 0000000000000..ba399ead8adfc --- /dev/null +++ b/docs/reference/esql/functions/search.asciidoc @@ -0,0 +1,26 @@ +[discrete] +[[esql-search-operators]] +=== Search operators + +The only search operator is match (`:`). + +preview::["Do not use on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] + +The match operator performs a <> on the specified field. +Returns true if the provided query matches the row. + +The match operator is equivalent to the <>. + +[.text-center] +image::esql/functions/signature/match_operator.svg[Embedded,opts=inline] + +include::types/match.asciidoc[] + +[source.merge.styled,esql] +---- +include::{esql-specs}/match-operator.csv-spec[tag=match-with-field] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/match-operator.csv-spec[tag=match-with-field-result] +|=== diff --git a/docs/reference/esql/functions/signature/bit_length.svg b/docs/reference/esql/functions/signature/bit_length.svg new file mode 100644 index 0000000000000..904dbbe25c9c2 --- /dev/null +++ b/docs/reference/esql/functions/signature/bit_length.svg @@ -0,0 +1 @@ +BIT_LENGTH(string) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/byte_length.svg b/docs/reference/esql/functions/signature/byte_length.svg new file mode 100644 index 0000000000000..d88821e46e926 --- /dev/null +++ b/docs/reference/esql/functions/signature/byte_length.svg @@ -0,0 +1 @@ +BYTE_LENGTH(string) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/categorize.svg b/docs/reference/esql/functions/signature/categorize.svg new file mode 100644 index 0000000000000..c52fd1763eea1 --- /dev/null +++ b/docs/reference/esql/functions/signature/categorize.svg @@ -0,0 +1 @@ +CATEGORIZE(field) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/hash.svg b/docs/reference/esql/functions/signature/hash.svg new file mode 100644 index 0000000000000..f819e14c9d1a4 --- /dev/null +++ b/docs/reference/esql/functions/signature/hash.svg @@ -0,0 +1 @@ +HASH(algorithm,input) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/hypot.svg b/docs/reference/esql/functions/signature/hypot.svg new file mode 100644 index 0000000000000..b849ea42cfd9e --- /dev/null +++ b/docs/reference/esql/functions/signature/hypot.svg @@ -0,0 +1 @@ +HYPOT(number1,number2) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/kql.svg b/docs/reference/esql/functions/signature/kql.svg new file mode 100644 index 0000000000000..3f550f27ccdff --- /dev/null +++ b/docs/reference/esql/functions/signature/kql.svg @@ -0,0 +1 @@ +KQL(query) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/match.svg b/docs/reference/esql/functions/signature/match.svg new file mode 100644 index 0000000000000..14ddb87468e70 --- /dev/null +++ b/docs/reference/esql/functions/signature/match.svg @@ -0,0 +1 @@ +MATCH(field,query) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/match_operator.svg b/docs/reference/esql/functions/signature/match_operator.svg new file mode 100644 index 0000000000000..70cea841622eb --- /dev/null +++ b/docs/reference/esql/functions/signature/match_operator.svg @@ -0,0 +1 @@ +field:query \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/qstr.svg b/docs/reference/esql/functions/signature/qstr.svg new file mode 100644 index 0000000000000..fb6114822ae63 --- /dev/null +++ b/docs/reference/esql/functions/signature/qstr.svg @@ -0,0 +1 @@ +QSTR(query) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/reverse.svg b/docs/reference/esql/functions/signature/reverse.svg new file mode 100644 index 0000000000000..c23ce5583a8c0 --- /dev/null +++ b/docs/reference/esql/functions/signature/reverse.svg @@ -0,0 +1 @@ +REVERSE(str) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/st_envelope.svg b/docs/reference/esql/functions/signature/st_envelope.svg new file mode 100644 index 0000000000000..885a60e6fd86f --- /dev/null +++ b/docs/reference/esql/functions/signature/st_envelope.svg @@ -0,0 +1 @@ +ST_ENVELOPE(geometry) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/st_extent_agg.svg b/docs/reference/esql/functions/signature/st_extent_agg.svg new file mode 100644 index 0000000000000..bb19b68bfb08b --- /dev/null +++ b/docs/reference/esql/functions/signature/st_extent_agg.svg @@ -0,0 +1 @@ +ST_EXTENT_AGG(field) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/st_xmax.svg b/docs/reference/esql/functions/signature/st_xmax.svg new file mode 100644 index 0000000000000..348d5a7f72763 --- /dev/null +++ b/docs/reference/esql/functions/signature/st_xmax.svg @@ -0,0 +1 @@ +ST_XMAX(point) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/st_xmin.svg b/docs/reference/esql/functions/signature/st_xmin.svg new file mode 100644 index 0000000000000..13d479b0458be --- /dev/null +++ b/docs/reference/esql/functions/signature/st_xmin.svg @@ -0,0 +1 @@ +ST_XMIN(point) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/st_ymax.svg b/docs/reference/esql/functions/signature/st_ymax.svg new file mode 100644 index 0000000000000..e6ecb00185c84 --- /dev/null +++ b/docs/reference/esql/functions/signature/st_ymax.svg @@ -0,0 +1 @@ +ST_YMAX(point) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/st_ymin.svg b/docs/reference/esql/functions/signature/st_ymin.svg new file mode 100644 index 0000000000000..ae722f1edc3d4 --- /dev/null +++ b/docs/reference/esql/functions/signature/st_ymin.svg @@ -0,0 +1 @@ +ST_YMIN(point) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/std_dev.svg b/docs/reference/esql/functions/signature/std_dev.svg new file mode 100644 index 0000000000000..606d285154f59 --- /dev/null +++ b/docs/reference/esql/functions/signature/std_dev.svg @@ -0,0 +1 @@ +STD_DEV(number) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/term.svg b/docs/reference/esql/functions/signature/term.svg new file mode 100644 index 0000000000000..955dd7fa215ab --- /dev/null +++ b/docs/reference/esql/functions/signature/term.svg @@ -0,0 +1 @@ +TERM(field,query) \ No newline at end of file diff --git a/docs/reference/esql/functions/spatial-functions.asciidoc b/docs/reference/esql/functions/spatial-functions.asciidoc index 79acc2028d983..c6a8467b39996 100644 --- a/docs/reference/esql/functions/spatial-functions.asciidoc +++ b/docs/reference/esql/functions/spatial-functions.asciidoc @@ -8,19 +8,29 @@ {esql} supports these spatial functions: // tag::spatial_list[] -* experimental:[] <> -* experimental:[] <> -* experimental:[] <> -* experimental:[] <> -* experimental:[] <> -* experimental:[] <> -* experimental:[] <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* experimental:[] <> +* experimental:[] <> +* experimental:[] <> +* experimental:[] <> +* experimental:[] <> // end::spatial_list[] +include::layout/st_distance.asciidoc[] include::layout/st_intersects.asciidoc[] include::layout/st_disjoint.asciidoc[] include::layout/st_contains.asciidoc[] include::layout/st_within.asciidoc[] include::layout/st_x.asciidoc[] include::layout/st_y.asciidoc[] -include::layout/st_distance.asciidoc[] +include::layout/st_envelope.asciidoc[] +include::layout/st_xmax.asciidoc[] +include::layout/st_xmin.asciidoc[] +include::layout/st_ymax.asciidoc[] +include::layout/st_ymin.asciidoc[] diff --git a/docs/reference/esql/functions/string-functions.asciidoc b/docs/reference/esql/functions/string-functions.asciidoc index ed97769b900e7..da9580a55151a 100644 --- a/docs/reference/esql/functions/string-functions.asciidoc +++ b/docs/reference/esql/functions/string-functions.asciidoc @@ -8,15 +8,19 @@ {esql} supports these string functions: // tag::string_list[] +* <> +* <> * <> * <> * <> +* <> * <> * <> * <> * <> * <> * <> +* <> * <> * <> * <> @@ -29,15 +33,19 @@ * <> // end::string_list[] +include::layout/bit_length.asciidoc[] +include::layout/byte_length.asciidoc[] include::layout/concat.asciidoc[] include::layout/ends_with.asciidoc[] include::layout/from_base64.asciidoc[] +include::layout/hash.asciidoc[] include::layout/left.asciidoc[] include::layout/length.asciidoc[] include::layout/locate.asciidoc[] include::layout/ltrim.asciidoc[] include::layout/repeat.asciidoc[] include::layout/replace.asciidoc[] +include::layout/reverse.asciidoc[] include::layout/right.asciidoc[] include::layout/rtrim.asciidoc[] include::layout/space.asciidoc[] diff --git a/docs/reference/esql/functions/types/add.asciidoc b/docs/reference/esql/functions/types/add.asciidoc index 54d1aec463c1a..e47a0d81f27e7 100644 --- a/docs/reference/esql/functions/types/add.asciidoc +++ b/docs/reference/esql/functions/types/add.asciidoc @@ -7,7 +7,10 @@ lhs | rhs | result date | date_period | date date | time_duration | date +date_nanos | date_period | date_nanos +date_nanos | time_duration | date_nanos date_period | date | date +date_period | date_nanos | date_nanos date_period | date_period | date_period double | double | double double | integer | double @@ -19,6 +22,7 @@ long | double | double long | integer | long long | long | long time_duration | date | date +time_duration | date_nanos | date_nanos time_duration | time_duration | time_duration unsigned_long | unsigned_long | unsigned_long |=== diff --git a/docs/reference/esql/functions/types/bit_length.asciidoc b/docs/reference/esql/functions/types/bit_length.asciidoc new file mode 100644 index 0000000000000..db5a48c7c4390 --- /dev/null +++ b/docs/reference/esql/functions/types/bit_length.asciidoc @@ -0,0 +1,10 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +string | result +keyword | integer +text | integer +|=== diff --git a/docs/reference/esql/functions/types/bucket.asciidoc b/docs/reference/esql/functions/types/bucket.asciidoc index 172e84b6f7860..2e6985e6bc4ed 100644 --- a/docs/reference/esql/functions/types/bucket.asciidoc +++ b/docs/reference/esql/functions/types/bucket.asciidoc @@ -16,6 +16,17 @@ date | integer | text | date | date date | integer | text | keyword | date date | integer | text | text | date date | time_duration | | | date +date_nanos | date_period | | | date_nanos +date_nanos | integer | date | date | date_nanos +date_nanos | integer | date | keyword | date_nanos +date_nanos | integer | date | text | date_nanos +date_nanos | integer | keyword | date | date_nanos +date_nanos | integer | keyword | keyword | date_nanos +date_nanos | integer | keyword | text | date_nanos +date_nanos | integer | text | date | date_nanos +date_nanos | integer | text | keyword | date_nanos +date_nanos | integer | text | text | date_nanos +date_nanos | time_duration | | | date_nanos double | double | | | double double | integer | double | double | double double | integer | double | integer | double diff --git a/docs/reference/esql/functions/types/byte_length.asciidoc b/docs/reference/esql/functions/types/byte_length.asciidoc new file mode 100644 index 0000000000000..db5a48c7c4390 --- /dev/null +++ b/docs/reference/esql/functions/types/byte_length.asciidoc @@ -0,0 +1,10 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +string | result +keyword | integer +text | integer +|=== diff --git a/docs/reference/esql/functions/types/case.asciidoc b/docs/reference/esql/functions/types/case.asciidoc index e8aa3eaf5daae..9e6915a37fc14 100644 --- a/docs/reference/esql/functions/types/case.asciidoc +++ b/docs/reference/esql/functions/types/case.asciidoc @@ -13,6 +13,8 @@ boolean | cartesian_shape | cartesian_shape | cartesian_shape boolean | cartesian_shape | | cartesian_shape boolean | date | date | date boolean | date | | date +boolean | date_nanos | date_nanos | date_nanos +boolean | date_nanos | | date_nanos boolean | double | double | double boolean | double | | double boolean | geo_point | geo_point | geo_point @@ -24,11 +26,13 @@ boolean | integer | | integer boolean | ip | ip | ip boolean | ip | | ip boolean | keyword | keyword | keyword +boolean | keyword | text | keyword boolean | keyword | | keyword boolean | long | long | long boolean | long | | long -boolean | text | text | text -boolean | text | | text +boolean | text | keyword | keyword +boolean | text | text | keyword +boolean | text | | keyword boolean | unsigned_long | unsigned_long | unsigned_long boolean | unsigned_long | | unsigned_long boolean | version | version | version diff --git a/docs/reference/esql/functions/types/categorize.asciidoc b/docs/reference/esql/functions/types/categorize.asciidoc new file mode 100644 index 0000000000000..5b64971cbc482 --- /dev/null +++ b/docs/reference/esql/functions/types/categorize.asciidoc @@ -0,0 +1,10 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +field | result +keyword | keyword +text | keyword +|=== diff --git a/docs/reference/esql/functions/types/coalesce.asciidoc b/docs/reference/esql/functions/types/coalesce.asciidoc index 368a12db0dca4..b6479dc7ff86a 100644 --- a/docs/reference/esql/functions/types/coalesce.asciidoc +++ b/docs/reference/esql/functions/types/coalesce.asciidoc @@ -10,6 +10,7 @@ boolean | | boolean cartesian_point | cartesian_point | cartesian_point cartesian_shape | cartesian_shape | cartesian_shape date | date | date +date_nanos | date_nanos | date_nanos geo_point | geo_point | geo_point geo_shape | geo_shape | geo_shape integer | integer | integer @@ -19,7 +20,7 @@ keyword | keyword | keyword keyword | | keyword long | long | long long | | long -text | text | text -text | | text +text | text | keyword +text | | keyword version | version | version |=== diff --git a/docs/reference/esql/functions/types/count_distinct.asciidoc b/docs/reference/esql/functions/types/count_distinct.asciidoc index c365c8814573c..f5758a8914d20 100644 --- a/docs/reference/esql/functions/types/count_distinct.asciidoc +++ b/docs/reference/esql/functions/types/count_distinct.asciidoc @@ -13,6 +13,10 @@ date | integer | long date | long | long date | unsigned_long | long date | | long +date_nanos | integer | long +date_nanos | long | long +date_nanos | unsigned_long | long +date_nanos | | long double | integer | long double | long | long double | unsigned_long | long diff --git a/docs/reference/esql/functions/types/date_trunc.asciidoc b/docs/reference/esql/functions/types/date_trunc.asciidoc index aa7dee99c6c44..c610f9119e51c 100644 --- a/docs/reference/esql/functions/types/date_trunc.asciidoc +++ b/docs/reference/esql/functions/types/date_trunc.asciidoc @@ -6,5 +6,7 @@ |=== interval | date | result date_period | date | date +date_period | date_nanos | date_nanos time_duration | date | date +time_duration | date_nanos | date_nanos |=== diff --git a/docs/reference/esql/functions/types/equals.asciidoc b/docs/reference/esql/functions/types/equals.asciidoc index ad0e46ef4b8da..1bb8bf2122b35 100644 --- a/docs/reference/esql/functions/types/equals.asciidoc +++ b/docs/reference/esql/functions/types/equals.asciidoc @@ -9,6 +9,9 @@ boolean | boolean | boolean cartesian_point | cartesian_point | boolean cartesian_shape | cartesian_shape | boolean date | date | boolean +date | date_nanos | boolean +date_nanos | date | boolean +date_nanos | date_nanos | boolean double | double | boolean double | integer | boolean double | long | boolean diff --git a/docs/reference/esql/functions/types/greater_than.asciidoc b/docs/reference/esql/functions/types/greater_than.asciidoc index c506328126a94..39253ac445f42 100644 --- a/docs/reference/esql/functions/types/greater_than.asciidoc +++ b/docs/reference/esql/functions/types/greater_than.asciidoc @@ -6,6 +6,9 @@ |=== lhs | rhs | result date | date | boolean +date | date_nanos | boolean +date_nanos | date | boolean +date_nanos | date_nanos | boolean double | double | boolean double | integer | boolean double | long | boolean diff --git a/docs/reference/esql/functions/types/greater_than_or_equal.asciidoc b/docs/reference/esql/functions/types/greater_than_or_equal.asciidoc index c506328126a94..39253ac445f42 100644 --- a/docs/reference/esql/functions/types/greater_than_or_equal.asciidoc +++ b/docs/reference/esql/functions/types/greater_than_or_equal.asciidoc @@ -6,6 +6,9 @@ |=== lhs | rhs | result date | date | boolean +date | date_nanos | boolean +date_nanos | date | boolean +date_nanos | date_nanos | boolean double | double | boolean double | integer | boolean double | long | boolean diff --git a/docs/reference/esql/functions/types/greatest.asciidoc b/docs/reference/esql/functions/types/greatest.asciidoc index 537be55cd17ef..0bc11b569d426 100644 --- a/docs/reference/esql/functions/types/greatest.asciidoc +++ b/docs/reference/esql/functions/types/greatest.asciidoc @@ -7,6 +7,8 @@ first | rest | result boolean | boolean | boolean boolean | | boolean +date | date | date +date_nanos | date_nanos | date_nanos double | double | double integer | integer | integer integer | | integer @@ -15,7 +17,7 @@ keyword | keyword | keyword keyword | | keyword long | long | long long | | long -text | text | text -text | | text +text | text | keyword +text | | keyword version | version | version |=== diff --git a/docs/reference/esql/functions/types/hash.asciidoc b/docs/reference/esql/functions/types/hash.asciidoc new file mode 100644 index 0000000000000..786ba03b2aa60 --- /dev/null +++ b/docs/reference/esql/functions/types/hash.asciidoc @@ -0,0 +1,12 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +algorithm | input | result +keyword | keyword | keyword +keyword | text | keyword +text | keyword | keyword +text | text | keyword +|=== diff --git a/docs/reference/esql/functions/types/hypot.asciidoc b/docs/reference/esql/functions/types/hypot.asciidoc new file mode 100644 index 0000000000000..dd06ba96d7f34 --- /dev/null +++ b/docs/reference/esql/functions/types/hypot.asciidoc @@ -0,0 +1,24 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +number1 | number2 | result +double | double | double +double | integer | double +double | long | double +double | unsigned_long | double +integer | double | double +integer | integer | double +integer | long | double +integer | unsigned_long | double +long | double | double +long | integer | double +long | long | double +long | unsigned_long | double +unsigned_long | double | double +unsigned_long | integer | double +unsigned_long | long | double +unsigned_long | unsigned_long | double +|=== diff --git a/docs/reference/esql/functions/types/kql.asciidoc b/docs/reference/esql/functions/types/kql.asciidoc new file mode 100644 index 0000000000000..866a39e925665 --- /dev/null +++ b/docs/reference/esql/functions/types/kql.asciidoc @@ -0,0 +1,10 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +query | result +keyword | boolean +text | boolean +|=== diff --git a/docs/reference/esql/functions/types/least.asciidoc b/docs/reference/esql/functions/types/least.asciidoc index 537be55cd17ef..0bc11b569d426 100644 --- a/docs/reference/esql/functions/types/least.asciidoc +++ b/docs/reference/esql/functions/types/least.asciidoc @@ -7,6 +7,8 @@ first | rest | result boolean | boolean | boolean boolean | | boolean +date | date | date +date_nanos | date_nanos | date_nanos double | double | double integer | integer | integer integer | | integer @@ -15,7 +17,7 @@ keyword | keyword | keyword keyword | | keyword long | long | long long | | long -text | text | text -text | | text +text | text | keyword +text | | keyword version | version | version |=== diff --git a/docs/reference/esql/functions/types/less_than.asciidoc b/docs/reference/esql/functions/types/less_than.asciidoc index c506328126a94..39253ac445f42 100644 --- a/docs/reference/esql/functions/types/less_than.asciidoc +++ b/docs/reference/esql/functions/types/less_than.asciidoc @@ -6,6 +6,9 @@ |=== lhs | rhs | result date | date | boolean +date | date_nanos | boolean +date_nanos | date | boolean +date_nanos | date_nanos | boolean double | double | boolean double | integer | boolean double | long | boolean diff --git a/docs/reference/esql/functions/types/less_than_or_equal.asciidoc b/docs/reference/esql/functions/types/less_than_or_equal.asciidoc index c506328126a94..39253ac445f42 100644 --- a/docs/reference/esql/functions/types/less_than_or_equal.asciidoc +++ b/docs/reference/esql/functions/types/less_than_or_equal.asciidoc @@ -6,6 +6,9 @@ |=== lhs | rhs | result date | date | boolean +date | date_nanos | boolean +date_nanos | date | boolean +date_nanos | date_nanos | boolean double | double | boolean double | integer | boolean double | long | boolean diff --git a/docs/reference/esql/functions/types/ltrim.asciidoc b/docs/reference/esql/functions/types/ltrim.asciidoc index 41d60049d59b8..1ba0e98ec8f09 100644 --- a/docs/reference/esql/functions/types/ltrim.asciidoc +++ b/docs/reference/esql/functions/types/ltrim.asciidoc @@ -6,5 +6,5 @@ |=== string | result keyword | keyword -text | text +text | keyword |=== diff --git a/docs/reference/esql/functions/types/match.asciidoc b/docs/reference/esql/functions/types/match.asciidoc new file mode 100644 index 0000000000000..402277af44749 --- /dev/null +++ b/docs/reference/esql/functions/types/match.asciidoc @@ -0,0 +1,37 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +field | query | result +boolean | boolean | boolean +boolean | keyword | boolean +date | date | boolean +date | keyword | boolean +date_nanos | date_nanos | boolean +date_nanos | keyword | boolean +double | double | boolean +double | integer | boolean +double | keyword | boolean +double | long | boolean +integer | double | boolean +integer | integer | boolean +integer | keyword | boolean +integer | long | boolean +ip | ip | boolean +ip | keyword | boolean +keyword | keyword | boolean +long | double | boolean +long | integer | boolean +long | keyword | boolean +long | long | boolean +text | keyword | boolean +unsigned_long | double | boolean +unsigned_long | integer | boolean +unsigned_long | keyword | boolean +unsigned_long | long | boolean +unsigned_long | unsigned_long | boolean +version | keyword | boolean +version | version | boolean +|=== diff --git a/docs/reference/esql/functions/types/match_operator.asciidoc b/docs/reference/esql/functions/types/match_operator.asciidoc new file mode 100644 index 0000000000000..402277af44749 --- /dev/null +++ b/docs/reference/esql/functions/types/match_operator.asciidoc @@ -0,0 +1,37 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +field | query | result +boolean | boolean | boolean +boolean | keyword | boolean +date | date | boolean +date | keyword | boolean +date_nanos | date_nanos | boolean +date_nanos | keyword | boolean +double | double | boolean +double | integer | boolean +double | keyword | boolean +double | long | boolean +integer | double | boolean +integer | integer | boolean +integer | keyword | boolean +integer | long | boolean +ip | ip | boolean +ip | keyword | boolean +keyword | keyword | boolean +long | double | boolean +long | integer | boolean +long | keyword | boolean +long | long | boolean +text | keyword | boolean +unsigned_long | double | boolean +unsigned_long | integer | boolean +unsigned_long | keyword | boolean +unsigned_long | long | boolean +unsigned_long | unsigned_long | boolean +version | keyword | boolean +version | version | boolean +|=== diff --git a/docs/reference/esql/functions/types/max.asciidoc b/docs/reference/esql/functions/types/max.asciidoc index 35ce5811e0cd0..adf457dac31b8 100644 --- a/docs/reference/esql/functions/types/max.asciidoc +++ b/docs/reference/esql/functions/types/max.asciidoc @@ -7,11 +7,12 @@ field | result boolean | boolean date | date +date_nanos | date_nanos double | double integer | integer ip | ip keyword | keyword long | long -text | text +text | keyword version | version |=== diff --git a/docs/reference/esql/functions/types/min.asciidoc b/docs/reference/esql/functions/types/min.asciidoc index 35ce5811e0cd0..adf457dac31b8 100644 --- a/docs/reference/esql/functions/types/min.asciidoc +++ b/docs/reference/esql/functions/types/min.asciidoc @@ -7,11 +7,12 @@ field | result boolean | boolean date | date +date_nanos | date_nanos double | double integer | integer ip | ip keyword | keyword long | long -text | text +text | keyword version | version |=== diff --git a/docs/reference/esql/functions/types/mv_append.asciidoc b/docs/reference/esql/functions/types/mv_append.asciidoc index a1894e429ae82..05f9ff6b19f9e 100644 --- a/docs/reference/esql/functions/types/mv_append.asciidoc +++ b/docs/reference/esql/functions/types/mv_append.asciidoc @@ -16,6 +16,6 @@ integer | integer | integer ip | ip | ip keyword | keyword | keyword long | long | long -text | text | text +text | text | keyword version | version | version |=== diff --git a/docs/reference/esql/functions/types/mv_count.asciidoc b/docs/reference/esql/functions/types/mv_count.asciidoc index 260c531731f04..c58c4eda44396 100644 --- a/docs/reference/esql/functions/types/mv_count.asciidoc +++ b/docs/reference/esql/functions/types/mv_count.asciidoc @@ -9,6 +9,7 @@ boolean | integer cartesian_point | integer cartesian_shape | integer date | integer +date_nanos | integer double | integer geo_point | integer geo_shape | integer diff --git a/docs/reference/esql/functions/types/mv_dedupe.asciidoc b/docs/reference/esql/functions/types/mv_dedupe.asciidoc index 68e546451c8cb..1524ec86cd5ec 100644 --- a/docs/reference/esql/functions/types/mv_dedupe.asciidoc +++ b/docs/reference/esql/functions/types/mv_dedupe.asciidoc @@ -9,6 +9,7 @@ boolean | boolean cartesian_point | cartesian_point cartesian_shape | cartesian_shape date | date +date_nanos | date_nanos double | double geo_point | geo_point geo_shape | geo_shape @@ -16,6 +17,6 @@ integer | integer ip | ip keyword | keyword long | long -text | text +text | keyword version | version |=== diff --git a/docs/reference/esql/functions/types/mv_first.asciidoc b/docs/reference/esql/functions/types/mv_first.asciidoc index 35633544d99a0..e68af2f992b43 100644 --- a/docs/reference/esql/functions/types/mv_first.asciidoc +++ b/docs/reference/esql/functions/types/mv_first.asciidoc @@ -9,6 +9,7 @@ boolean | boolean cartesian_point | cartesian_point cartesian_shape | cartesian_shape date | date +date_nanos | date_nanos double | double geo_point | geo_point geo_shape | geo_shape @@ -16,7 +17,7 @@ integer | integer ip | ip keyword | keyword long | long -text | text +text | keyword unsigned_long | unsigned_long version | version |=== diff --git a/docs/reference/esql/functions/types/mv_last.asciidoc b/docs/reference/esql/functions/types/mv_last.asciidoc index 35633544d99a0..e68af2f992b43 100644 --- a/docs/reference/esql/functions/types/mv_last.asciidoc +++ b/docs/reference/esql/functions/types/mv_last.asciidoc @@ -9,6 +9,7 @@ boolean | boolean cartesian_point | cartesian_point cartesian_shape | cartesian_shape date | date +date_nanos | date_nanos double | double geo_point | geo_point geo_shape | geo_shape @@ -16,7 +17,7 @@ integer | integer ip | ip keyword | keyword long | long -text | text +text | keyword unsigned_long | unsigned_long version | version |=== diff --git a/docs/reference/esql/functions/types/mv_max.asciidoc b/docs/reference/esql/functions/types/mv_max.asciidoc index 8ea36aebbad37..ffba14489b97c 100644 --- a/docs/reference/esql/functions/types/mv_max.asciidoc +++ b/docs/reference/esql/functions/types/mv_max.asciidoc @@ -7,12 +7,13 @@ field | result boolean | boolean date | date +date_nanos | date_nanos double | double integer | integer ip | ip keyword | keyword long | long -text | text +text | keyword unsigned_long | unsigned_long version | version |=== diff --git a/docs/reference/esql/functions/types/mv_min.asciidoc b/docs/reference/esql/functions/types/mv_min.asciidoc index 8ea36aebbad37..ffba14489b97c 100644 --- a/docs/reference/esql/functions/types/mv_min.asciidoc +++ b/docs/reference/esql/functions/types/mv_min.asciidoc @@ -7,12 +7,13 @@ field | result boolean | boolean date | date +date_nanos | date_nanos double | double integer | integer ip | ip keyword | keyword long | long -text | text +text | keyword unsigned_long | unsigned_long version | version |=== diff --git a/docs/reference/esql/functions/types/mv_slice.asciidoc b/docs/reference/esql/functions/types/mv_slice.asciidoc index 0a9dc073370c7..75f45e333ee0c 100644 --- a/docs/reference/esql/functions/types/mv_slice.asciidoc +++ b/docs/reference/esql/functions/types/mv_slice.asciidoc @@ -9,6 +9,7 @@ boolean | integer | integer | boolean cartesian_point | integer | integer | cartesian_point cartesian_shape | integer | integer | cartesian_shape date | integer | integer | date +date_nanos | integer | integer | date_nanos double | integer | integer | double geo_point | integer | integer | geo_point geo_shape | integer | integer | geo_shape @@ -16,6 +17,6 @@ integer | integer | integer | integer ip | integer | integer | ip keyword | integer | integer | keyword long | integer | integer | long -text | integer | integer | text +text | integer | integer | keyword version | integer | integer | version |=== diff --git a/docs/reference/esql/functions/types/mv_sort.asciidoc b/docs/reference/esql/functions/types/mv_sort.asciidoc index 93965187482ac..83d3e45c7be02 100644 --- a/docs/reference/esql/functions/types/mv_sort.asciidoc +++ b/docs/reference/esql/functions/types/mv_sort.asciidoc @@ -7,11 +7,12 @@ field | order | result boolean | keyword | boolean date | keyword | date +date_nanos | keyword | date_nanos double | keyword | double integer | keyword | integer ip | keyword | ip keyword | keyword | keyword long | keyword | long -text | keyword | text +text | keyword | keyword version | keyword | version |=== diff --git a/docs/reference/esql/functions/types/not_equals.asciidoc b/docs/reference/esql/functions/types/not_equals.asciidoc index ad0e46ef4b8da..1bb8bf2122b35 100644 --- a/docs/reference/esql/functions/types/not_equals.asciidoc +++ b/docs/reference/esql/functions/types/not_equals.asciidoc @@ -9,6 +9,9 @@ boolean | boolean | boolean cartesian_point | cartesian_point | boolean cartesian_shape | cartesian_shape | boolean date | date | boolean +date | date_nanos | boolean +date_nanos | date | boolean +date_nanos | date_nanos | boolean double | double | boolean double | integer | boolean double | long | boolean diff --git a/docs/reference/esql/functions/types/qstr.asciidoc b/docs/reference/esql/functions/types/qstr.asciidoc new file mode 100644 index 0000000000000..866a39e925665 --- /dev/null +++ b/docs/reference/esql/functions/types/qstr.asciidoc @@ -0,0 +1,10 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +query | result +keyword | boolean +text | boolean +|=== diff --git a/docs/reference/esql/functions/types/reverse.asciidoc b/docs/reference/esql/functions/types/reverse.asciidoc new file mode 100644 index 0000000000000..9e5dc1c477316 --- /dev/null +++ b/docs/reference/esql/functions/types/reverse.asciidoc @@ -0,0 +1,10 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +str | result +keyword | keyword +text | keyword +|=== diff --git a/docs/reference/esql/functions/types/rtrim.asciidoc b/docs/reference/esql/functions/types/rtrim.asciidoc index 41d60049d59b8..1ba0e98ec8f09 100644 --- a/docs/reference/esql/functions/types/rtrim.asciidoc +++ b/docs/reference/esql/functions/types/rtrim.asciidoc @@ -6,5 +6,5 @@ |=== string | result keyword | keyword -text | text +text | keyword |=== diff --git a/docs/reference/esql/functions/types/st_envelope.asciidoc b/docs/reference/esql/functions/types/st_envelope.asciidoc new file mode 100644 index 0000000000000..43355394c6015 --- /dev/null +++ b/docs/reference/esql/functions/types/st_envelope.asciidoc @@ -0,0 +1,12 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +geometry | result +cartesian_point | cartesian_shape +cartesian_shape | cartesian_shape +geo_point | geo_shape +geo_shape | geo_shape +|=== diff --git a/docs/reference/esql/functions/types/st_extent_agg.asciidoc b/docs/reference/esql/functions/types/st_extent_agg.asciidoc new file mode 100644 index 0000000000000..c836aa1896f07 --- /dev/null +++ b/docs/reference/esql/functions/types/st_extent_agg.asciidoc @@ -0,0 +1,12 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +field | result +cartesian_point | cartesian_shape +cartesian_shape | cartesian_shape +geo_point | geo_shape +geo_shape | geo_shape +|=== diff --git a/docs/reference/esql/functions/types/st_xmax.asciidoc b/docs/reference/esql/functions/types/st_xmax.asciidoc new file mode 100644 index 0000000000000..418c5cafae6f3 --- /dev/null +++ b/docs/reference/esql/functions/types/st_xmax.asciidoc @@ -0,0 +1,12 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +point | result +cartesian_point | double +cartesian_shape | double +geo_point | double +geo_shape | double +|=== diff --git a/docs/reference/esql/functions/types/st_xmin.asciidoc b/docs/reference/esql/functions/types/st_xmin.asciidoc new file mode 100644 index 0000000000000..418c5cafae6f3 --- /dev/null +++ b/docs/reference/esql/functions/types/st_xmin.asciidoc @@ -0,0 +1,12 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +point | result +cartesian_point | double +cartesian_shape | double +geo_point | double +geo_shape | double +|=== diff --git a/docs/reference/esql/functions/types/st_ymax.asciidoc b/docs/reference/esql/functions/types/st_ymax.asciidoc new file mode 100644 index 0000000000000..418c5cafae6f3 --- /dev/null +++ b/docs/reference/esql/functions/types/st_ymax.asciidoc @@ -0,0 +1,12 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +point | result +cartesian_point | double +cartesian_shape | double +geo_point | double +geo_shape | double +|=== diff --git a/docs/reference/esql/functions/types/st_ymin.asciidoc b/docs/reference/esql/functions/types/st_ymin.asciidoc new file mode 100644 index 0000000000000..418c5cafae6f3 --- /dev/null +++ b/docs/reference/esql/functions/types/st_ymin.asciidoc @@ -0,0 +1,12 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +point | result +cartesian_point | double +cartesian_shape | double +geo_point | double +geo_shape | double +|=== diff --git a/docs/reference/esql/functions/types/std_dev.asciidoc b/docs/reference/esql/functions/types/std_dev.asciidoc new file mode 100644 index 0000000000000..273dae4af76c2 --- /dev/null +++ b/docs/reference/esql/functions/types/std_dev.asciidoc @@ -0,0 +1,11 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +number | result +double | double +integer | double +long | double +|=== diff --git a/docs/reference/esql/functions/types/sub.asciidoc b/docs/reference/esql/functions/types/sub.asciidoc index c3ded301ebe68..dca56026071ee 100644 --- a/docs/reference/esql/functions/types/sub.asciidoc +++ b/docs/reference/esql/functions/types/sub.asciidoc @@ -7,6 +7,9 @@ lhs | rhs | result date | date_period | date date | time_duration | date +date_nanos | date_period | date_nanos +date_nanos | time_duration | date_nanos +date_period | date_nanos | date_nanos date_period | date_period | date_period double | double | double double | integer | double @@ -17,6 +20,7 @@ integer | long | long long | double | double long | integer | long long | long | long +time_duration | date_nanos | date_nanos time_duration | time_duration | time_duration unsigned_long | unsigned_long | unsigned_long |=== diff --git a/docs/reference/esql/functions/types/term.asciidoc b/docs/reference/esql/functions/types/term.asciidoc new file mode 100644 index 0000000000000..7523b29c62b1d --- /dev/null +++ b/docs/reference/esql/functions/types/term.asciidoc @@ -0,0 +1,12 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +field | query | result +keyword | keyword | boolean +keyword | text | boolean +text | keyword | boolean +text | text | boolean +|=== diff --git a/docs/reference/esql/functions/types/to_datetime.asciidoc b/docs/reference/esql/functions/types/to_datetime.asciidoc index 80c986efca794..118ed6c09c11c 100644 --- a/docs/reference/esql/functions/types/to_datetime.asciidoc +++ b/docs/reference/esql/functions/types/to_datetime.asciidoc @@ -6,6 +6,7 @@ |=== field | result date | date +date_nanos | date double | date integer | date keyword | date diff --git a/docs/reference/esql/functions/types/to_long.asciidoc b/docs/reference/esql/functions/types/to_long.asciidoc index a07990cb1cfbf..1009543c1bbde 100644 --- a/docs/reference/esql/functions/types/to_long.asciidoc +++ b/docs/reference/esql/functions/types/to_long.asciidoc @@ -9,6 +9,7 @@ boolean | long counter_integer | long counter_long | long date | long +date_nanos | long double | long integer | long keyword | long diff --git a/docs/reference/esql/functions/types/to_lower.asciidoc b/docs/reference/esql/functions/types/to_lower.asciidoc index 974066d225bca..9e5dc1c477316 100644 --- a/docs/reference/esql/functions/types/to_lower.asciidoc +++ b/docs/reference/esql/functions/types/to_lower.asciidoc @@ -6,5 +6,5 @@ |=== str | result keyword | keyword -text | text +text | keyword |=== diff --git a/docs/reference/esql/functions/types/to_string.asciidoc b/docs/reference/esql/functions/types/to_string.asciidoc index 26a5b31a2a589..9d4188214b3d8 100644 --- a/docs/reference/esql/functions/types/to_string.asciidoc +++ b/docs/reference/esql/functions/types/to_string.asciidoc @@ -9,6 +9,7 @@ boolean | keyword cartesian_point | keyword cartesian_shape | keyword date | keyword +date_nanos | keyword double | keyword geo_point | keyword geo_shape | keyword diff --git a/docs/reference/esql/functions/types/to_upper.asciidoc b/docs/reference/esql/functions/types/to_upper.asciidoc index 974066d225bca..9e5dc1c477316 100644 --- a/docs/reference/esql/functions/types/to_upper.asciidoc +++ b/docs/reference/esql/functions/types/to_upper.asciidoc @@ -6,5 +6,5 @@ |=== str | result keyword | keyword -text | text +text | keyword |=== diff --git a/docs/reference/esql/functions/types/top.asciidoc b/docs/reference/esql/functions/types/top.asciidoc index 0eb329c10b9ed..699bc7b10ce84 100644 --- a/docs/reference/esql/functions/types/top.asciidoc +++ b/docs/reference/esql/functions/types/top.asciidoc @@ -10,5 +10,7 @@ date | integer | keyword | date double | integer | keyword | double integer | integer | keyword | integer ip | integer | keyword | ip +keyword | integer | keyword | keyword long | integer | keyword | long +text | integer | keyword | keyword |=== diff --git a/docs/reference/esql/functions/types/trim.asciidoc b/docs/reference/esql/functions/types/trim.asciidoc index 41d60049d59b8..1ba0e98ec8f09 100644 --- a/docs/reference/esql/functions/types/trim.asciidoc +++ b/docs/reference/esql/functions/types/trim.asciidoc @@ -6,5 +6,5 @@ |=== string | result keyword | keyword -text | text +text | keyword |=== diff --git a/docs/reference/esql/functions/types/values.asciidoc b/docs/reference/esql/functions/types/values.asciidoc index 35ce5811e0cd0..adf457dac31b8 100644 --- a/docs/reference/esql/functions/types/values.asciidoc +++ b/docs/reference/esql/functions/types/values.asciidoc @@ -7,11 +7,12 @@ field | result boolean | boolean date | date +date_nanos | date_nanos double | double integer | integer ip | ip keyword | keyword long | long -text | text +text | keyword version | version |=== diff --git a/docs/reference/esql/implicit-casting.asciidoc b/docs/reference/esql/implicit-casting.asciidoc index f0c0aa3d82063..b24be0b645472 100644 --- a/docs/reference/esql/implicit-casting.asciidoc +++ b/docs/reference/esql/implicit-casting.asciidoc @@ -5,7 +5,7 @@ Implicit casting ++++ -Often users will input `datetime`, `ip`, `version`, or geospatial objects as simple strings in their queries for use in predicates, functions, or expressions. {esql} provides <> to explicitly convert these strings into the desired data types. +Often users will input `date`, `date_period`, `time_duration`, `ip` or `version` as simple strings in their queries for use in predicates, functions, or expressions. {esql} provides <> to explicitly convert these strings into the desired data types. Without implicit casting users must explicitly code these `to_X` functions in their queries, when string literals don't match the target data types they are assigned or compared to. Here is an example of using `to_datetime` to explicitly perform a data type conversion. @@ -18,7 +18,10 @@ FROM employees | LIMIT 1 ---- -Implicit casting improves usability, by automatically converting string literals to the target data type. This is most useful when the target data type is `datetime`, `ip`, `version` or a geo spatial. It is natural to specify these as a string in queries. +[discrete] +[[esql-implicit-casting-example]] +==== Implicit casting example +Implicit casting automatically converts string literals to the target data type. This allows users to specify string values for types like `date`, `date_period`, `time_duration`, `ip` and `version` in their queries. The first query can be coded without calling the `to_datetime` function, as follows: @@ -31,23 +34,36 @@ FROM employees | LIMIT 1 ---- -[float] -=== Implicit casting support +[discrete] +[[esql-implicit-casting-supported-operations]] +==== Operations that support implicit casting The following table details which {esql} operations support implicit casting for different data types. [%header.monospaced.styled,format=dsv,separator=|] |=== -||ScalarFunction|BinaryComparison|ArithmeticOperation|InListPredicate|AggregateFunction -|DATETIME|Y|Y|Y|Y|N -|DOUBLE|Y|N|N|N|N -|LONG|Y|N|N|N|N -|INTEGER|Y|N|N|N|N -|IP|Y|Y|Y|Y|N -|VERSION|Y|Y|Y|Y|N -|GEO_POINT|Y|N|N|N|N -|GEO_SHAPE|Y|N|N|N|N -|CARTESIAN_POINT|Y|N|N|N|N -|CARTESIAN_SHAPE|Y|N|N|N|N -|BOOLEAN|Y|Y|Y|Y|N +|ScalarFunctions|Operators|<>|<> +DATE|Y|Y|Y|N +DATE_PERIOD/TIME_DURATION|Y|N|Y|N +IP|Y|Y|Y|N +VERSION|Y|Y|Y|N +BOOLEAN|Y|Y|Y|N |=== + +ScalarFunctions includes: + +* <> + +* <> + +* <> + + +Operators includes: + +* <> + +* <> + +* <> + diff --git a/docs/reference/esql/multivalued-fields.asciidoc b/docs/reference/esql/multivalued-fields.asciidoc index 8ff645bba863e..562ea2a2e6b4a 100644 --- a/docs/reference/esql/multivalued-fields.asciidoc +++ b/docs/reference/esql/multivalued-fields.asciidoc @@ -26,6 +26,7 @@ Multivalued fields come back as a JSON array: [source,console-result] ---- { + "took": 28, "columns": [ { "name": "a", "type": "long"}, { "name": "b", "type": "long"} @@ -36,6 +37,8 @@ Multivalued fields come back as a JSON array: ] } ---- +// TESTRESPONSE[s/"took": 28/"took": "$body.took"/] + The relative order of values in a multivalued field is undefined. They'll frequently be in ascending order but don't rely on that. @@ -74,6 +77,7 @@ And {esql} sees that removal: [source,console-result] ---- { + "took": 28, "columns": [ { "name": "a", "type": "long"}, { "name": "b", "type": "keyword"} @@ -84,6 +88,8 @@ And {esql} sees that removal: ] } ---- +// TESTRESPONSE[s/"took": 28/"took": "$body.took"/] + But other types, like `long` don't remove duplicates. @@ -115,6 +121,7 @@ And {esql} also sees that: [source,console-result] ---- { + "took": 28, "columns": [ { "name": "a", "type": "long"}, { "name": "b", "type": "long"} @@ -125,6 +132,8 @@ And {esql} also sees that: ] } ---- +// TESTRESPONSE[s/"took": 28/"took": "$body.took"/] + This is all at the storage layer. If you store duplicate `long`s and then convert them to strings the duplicates will stay: @@ -155,6 +164,7 @@ POST /_query [source,console-result] ---- { + "took": 28, "columns": [ { "name": "a", "type": "long"}, { "name": "b", "type": "keyword"} @@ -165,6 +175,38 @@ POST /_query ] } ---- +// TESTRESPONSE[s/"took": 28/"took": "$body.took"/] + +[discrete] +[[esql-multivalued-nulls]] +==== `null` in a list + +`null` values in a list are not preserved at the storage layer: + +[source,console,id=esql-multivalued-fields-multivalued-nulls] +---- +POST /mv/_doc?refresh +{ "a": [2, null, 1] } + +POST /_query +{ + "query": "FROM mv | LIMIT 1" +} +---- + +[source,console-result] +---- +{ + "took": 28, + "columns": [ + { "name": "a", "type": "long"}, + ], + "values": [ + [[1, 2]], + ] +} +---- +// TESTRESPONSE[s/"took": 28/"took": "$body.took"/] [discrete] [[esql-multivalued-fields-functions]] @@ -198,6 +240,7 @@ POST /_query [source,console-result] ---- { + "took": 28, "columns": [ { "name": "a", "type": "long"}, { "name": "b", "type": "long"}, @@ -210,6 +253,7 @@ POST /_query ] } ---- +// TESTRESPONSE[s/"took": 28/"took": "$body.took"/] Work around this limitation by converting the field to single value with one of: @@ -233,6 +277,7 @@ POST /_query [source,console-result] ---- { + "took": 28, "columns": [ { "name": "a", "type": "long"}, { "name": "b", "type": "long"}, @@ -245,4 +290,4 @@ POST /_query ] } ---- - +// TESTRESPONSE[s/"took": 28/"took": "$body.took"/] diff --git a/docs/reference/esql/processing-commands/inlinestats.asciidoc b/docs/reference/esql/processing-commands/inlinestats.disabled similarity index 100% rename from docs/reference/esql/processing-commands/inlinestats.asciidoc rename to docs/reference/esql/processing-commands/inlinestats.disabled diff --git a/docs/reference/esql/processing-commands/lookup.asciidoc b/docs/reference/esql/processing-commands/lookup.disabled similarity index 100% rename from docs/reference/esql/processing-commands/lookup.asciidoc rename to docs/reference/esql/processing-commands/lookup.disabled diff --git a/docs/reference/esql/processing-commands/stats.asciidoc b/docs/reference/esql/processing-commands/stats.asciidoc index 0c479c1f62b76..3ed296fb6db24 100644 --- a/docs/reference/esql/processing-commands/stats.asciidoc +++ b/docs/reference/esql/processing-commands/stats.asciidoc @@ -1,16 +1,18 @@ [discrete] [[esql-stats-by]] -=== `STATS ... BY` +=== `STATS` -The `STATS ... BY` processing command groups rows according to a common value +The `STATS` processing command groups rows according to a common value and calculates one or more aggregated values over the grouped rows. **Syntax** [source,esql] ---- -STATS [column1 =] expression1[, ..., [columnN =] expressionN] -[BY grouping_expression1[, ..., grouping_expressionN]] +STATS [column1 =] expression1 [WHERE boolean_expression1][, + ..., + [columnN =] expressionN [WHERE boolean_expressionN]] + [BY grouping_expression1[, ..., grouping_expressionN]] ---- *Parameters* @@ -28,14 +30,18 @@ An expression that computes an aggregated value. An expression that outputs the values to group by. If its name coincides with one of the computed columns, that column will be ignored. +`boolean_expressionX`:: +The condition that must be met for a row to be included in the evaluation of `expressionX`. + NOTE: Individual `null` values are skipped when computing aggregations. *Description* -The `STATS ... BY` processing command groups rows according to a common value -and calculate one or more aggregated values over the grouped rows. If `BY` is -omitted, the output table contains exactly one row with the aggregations applied -over the entire dataset. +The `STATS` processing command groups rows according to a common value +and calculates one or more aggregated values over the grouped rows. For the +calculation of each aggregated value, the rows in a group can be filtered with +`WHERE`. If `BY` is omitted, the output table contains exactly one row with +the aggregations applied over the entire dataset. The following <> are supported: @@ -90,6 +96,29 @@ include::{esql-specs}/stats.csv-spec[tag=statsCalcMultipleValues] include::{esql-specs}/stats.csv-spec[tag=statsCalcMultipleValues-result] |=== +To filter the rows that go into an aggregation, use the `WHERE` clause: + +[source.merge.styled,esql] +---- +include::{esql-specs}/stats.csv-spec[tag=aggFiltering] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/stats.csv-spec[tag=aggFiltering-result] +|=== + +The aggregations can be mixed, with and without a filter and grouping is +optional as well: + +[source.merge.styled,esql] +---- +include::{esql-specs}/stats.csv-spec[tag=aggFilteringNoGroup] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/stats.csv-spec[tag=aggFilteringNoGroup-result] +|=== + [[esql-stats-mv-group]] If the grouping key is multivalued then the input row is in all groups: @@ -109,7 +138,7 @@ It's also possible to group by multiple values: include::{esql-specs}/stats.csv-spec[tag=statsGroupByMultipleValues] ---- -If the all grouping keys are multivalued then the input row is in all groups: +If all the grouping keys are multivalued then the input row is in all groups: [source.merge.styled,esql] ---- @@ -121,7 +150,7 @@ include::{esql-specs}/stats.csv-spec[tag=multi-mv-group-result] |=== Both the aggregating functions and the grouping expressions accept other -functions. This is useful for using `STATS...BY` on multivalue columns. +functions. This is useful for using `STATS` on multivalue columns. For example, to calculate the average salary change, you can use `MV_AVG` to first average the multiple values per employee, and use the result with the `AVG` function: diff --git a/docs/reference/esql/time-spans.asciidoc b/docs/reference/esql/time-spans.asciidoc new file mode 100644 index 0000000000000..d2aa0c4fa252e --- /dev/null +++ b/docs/reference/esql/time-spans.asciidoc @@ -0,0 +1,111 @@ +[[esql-time-spans]] +=== {esql} time spans + +++++ +Time spans +++++ + +Time spans represent intervals between two datetime values. There are currently two supported types of time spans: + +* `DATE_PERIOD` specifies intervals in years, quarters, months, weeks and days +* `TIME_DURATION` specifies intervals in hours, minutes, seconds and milliseconds + +A time span requires two elements: an integer value and a temporal unit. + +Time spans work with grouping functions such as <>, scalar functions such as <> and arithmetic operators such as <> and <>. Convert strings to time spans using <>, <>, or the cast operators `::DATE_PERIOD`, `::TIME_DURATION`. + +[discrete] +[[esql-time-spans-examples]] +==== Examples of using time spans in {esql} + + +With `BUCKET`: +[source.merge.styled,esql] +---- +include::{esql-specs}/bucket.csv-spec[tag=docsBucketWeeklyHistogramWithSpan] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/bucket.csv-spec[tag=docsBucketWeeklyHistogramWithSpan-result] +|=== + + +With `DATE_TRUNC`: +[source.merge.styled,esql] +---- +include::{esql-specs}/date.csv-spec[tag=docsDateTrunc] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/date.csv-spec[tag=docsDateTrunc-result] +|=== + + +With `+` and/or `-`: +[source.merge.styled,esql] +---- +include::{esql-specs}/date.csv-spec[tag=docsNowWhere] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/date.csv-spec[tag=docsNowWhere-result] +|=== + + +When a time span is provided as a named parameter in string format, `TO_DATEPERIOD`, `::DATE_PERIOD`, `TO_TIMEDURATION` or `::TIME_DURATION` can be used to convert to its corresponding time span value for arithmetic operations like `+` and/or `-`. +[source, esql] +---- +POST /_query +{ + "query": """ + FROM employees + | EVAL x = hire_date + ?timespan::DATE_PERIOD, y = hire_date - TO_DATEPERIOD(?timespan) + """, + "params": [{"timespan" : "1 day"}] +} +---- + +When a time span is provided as a named parameter in string format, it can be automatically converted to its corresponding time span value in grouping functions and scalar functions, like `BUCKET` and `DATE_TRUNC`. +[source, esql] +---- +POST /_query +{ + "query": """ + FROM employees + | WHERE hire_date >= "1985-01-01T00:00:00Z" AND hire_date < "1986-01-01T00:00:00Z" + | STATS hires_per_week = COUNT(*) BY week = BUCKET(hire_date, ?timespan) + | SORT week + """, + "params": [{"timespan" : "1 week"}] +} +---- + +[source, esql] +---- +POST /_query +{ + "query": """ + FROM employees + | KEEP first_name, last_name, hire_date + | EVAL year_hired = DATE_TRUNC(?timespan, hire_date) + """, + "params": [{"timespan" : "1 year"}] +} +---- + +[discrete] +[[esql-time-spans-table]] +==== Supported temporal units +[%header.monospaced.styled,format=dsv,separator=|] +|=== +Temporal Units|Valid Abbreviations +year|y, yr, years +quarter|q, quarters +month|mo, months +week|w, weeks +day|d, days +hour|h, hours +minute|min, minutes +second|s, sec, seconds +millisecond|ms, milliseconds +|=== diff --git a/docs/reference/features/apis/features-apis.asciidoc b/docs/reference/features/apis/features-apis.asciidoc index fe06471cff0df..2582446340f15 100644 --- a/docs/reference/features/apis/features-apis.asciidoc +++ b/docs/reference/features/apis/features-apis.asciidoc @@ -1,6 +1,12 @@ [[features-apis]] == Features APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-features[Features APIs]. +-- + You can use the following APIs to introspect and manage Features provided by Elasticsearch and Elasticsearch plugins. diff --git a/docs/reference/features/apis/get-features-api.asciidoc b/docs/reference/features/apis/get-features-api.asciidoc index 676ec3c41da24..62986d7728ca7 100644 --- a/docs/reference/features/apis/get-features-api.asciidoc +++ b/docs/reference/features/apis/get-features-api.asciidoc @@ -4,6 +4,12 @@ Get features ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-features[Features APIs]. +-- + Gets a list of features which can be included in snapshots using the <> when creating a snapshot. diff --git a/docs/reference/features/apis/reset-features-api.asciidoc b/docs/reference/features/apis/reset-features-api.asciidoc index 36ff12cf0fc33..e2d3f249304bf 100644 --- a/docs/reference/features/apis/reset-features-api.asciidoc +++ b/docs/reference/features/apis/reset-features-api.asciidoc @@ -6,6 +6,12 @@ experimental::[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-features[Features APIs]. +-- + Clears all of the state information stored in system indices by {es} features, including the security and machine learning indices. WARNING: Intended for development and testing use only. Do not reset features on a production cluster. diff --git a/docs/reference/fleet/fleet-multi-search.asciidoc b/docs/reference/fleet/fleet-multi-search.asciidoc index 3ee6b67b06ba4..5673e1abbbd1d 100644 --- a/docs/reference/fleet/fleet-multi-search.asciidoc +++ b/docs/reference/fleet/fleet-multi-search.asciidoc @@ -5,6 +5,12 @@ Fleet multi search ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-fleet[{fleet} APIs]. +-- + Executes several <> with a single API request. The API follows the same structure as the <> API. However, diff --git a/docs/reference/fleet/fleet-search.asciidoc b/docs/reference/fleet/fleet-search.asciidoc index 961846385969a..81ad0c9a7aa95 100644 --- a/docs/reference/fleet/fleet-search.asciidoc +++ b/docs/reference/fleet/fleet-search.asciidoc @@ -5,6 +5,12 @@ Fleet search ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-fleet[{fleet} APIs]. +-- + The purpose of the fleet search api is to provide a search api where the search will only be executed after provided checkpoint has been processed and is visible for searches inside of Elasticsearch. diff --git a/docs/reference/fleet/index.asciidoc b/docs/reference/fleet/index.asciidoc index b22609aff32e7..8cd3cd59f21b0 100644 --- a/docs/reference/fleet/index.asciidoc +++ b/docs/reference/fleet/index.asciidoc @@ -2,6 +2,12 @@ [[fleet-apis]] == Fleet APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-fleet[{fleet} APIs]. +-- + TIP: For the {kib} {fleet} APIs, see the {fleet-guide}/fleet-api-docs.html[Fleet API Documentation]. diff --git a/docs/reference/geospatial-analysis.asciidoc b/docs/reference/geospatial-analysis.asciidoc index 6760040e14bc7..678e0ee17aec2 100644 --- a/docs/reference/geospatial-analysis.asciidoc +++ b/docs/reference/geospatial-analysis.asciidoc @@ -38,11 +38,11 @@ Data is often messy and incomplete. <> lets you clean, <> has support for <> functions, enabling efficient index searching for documents that intersect with, are within, are contained by, or are disjoint from a query geometry. In addition, the `ST_DISTANCE` function calculates the distance between two points. -* experimental:[] <> -* experimental:[] <> -* experimental:[] <> -* experimental:[] <> -* experimental:[] <> +* <> +* <> +* <> +* <> +* <> [discrete] [[geospatial-aggregate]] diff --git a/docs/reference/graph/explore.asciidoc b/docs/reference/graph/explore.asciidoc index 34ac367125ade..60e9edb3b0f38 100644 --- a/docs/reference/graph/explore.asciidoc +++ b/docs/reference/graph/explore.asciidoc @@ -2,6 +2,12 @@ [[graph-explore-api]] == Graph explore API +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-graph[Graph explore APIs]. +-- + The graph explore API enables you to extract and summarize information about the documents and terms in an {es} data stream or index. diff --git a/docs/reference/health/health.asciidoc b/docs/reference/health/health.asciidoc index 34714e80e1b18..606804a83bbda 100644 --- a/docs/reference/health/health.asciidoc +++ b/docs/reference/health/health.asciidoc @@ -4,6 +4,12 @@ Health ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-health_report[Cluster health APIs]. +-- + An API that reports the health status of an {es} cluster. [[health-api-request]] diff --git a/docs/reference/high-availability.asciidoc b/docs/reference/high-availability.asciidoc index 2f34b6bc1bb21..37e2a38aa0f2c 100644 --- a/docs/reference/high-availability.asciidoc +++ b/docs/reference/high-availability.asciidoc @@ -3,28 +3,28 @@ [partintro] -- -Your data is important to you. Keeping it safe and available is important -to {es}. Sometimes your cluster may experience hardware failure or a power -loss. To help you plan for this, {es} offers a number of features -to achieve high availability despite failures. +Your data is important to you. Keeping it safe and available is important to Elastic. Sometimes your cluster may experience hardware failure or a power loss. To help you plan for this, {es} offers a number of features to achieve high availability despite failures. Depending on your deployment type, you might need to provision servers in different zones or configure external repositories to meet your organization's availability needs. -* With proper planning, a cluster can be - <> to many of the - things that commonly go wrong, from the loss of a single node or network - connection right up to a zone-wide outage such as power loss. +* *<>* ++ +Distributed systems like Elasticsearch are designed to keep working even if some of their components have failed. An Elasticsearch cluster can continue operating normally if some of its nodes are unavailable or disconnected, as long as there are enough well-connected nodes to take over the unavailable node's responsibilities. ++ +If you're designing a smaller cluster, you might focus on making your cluster resilient to single-node failures. Designers of larger clusters must also consider cases where multiple nodes fail at the same time. +// need to improve connections to ECE, EC hosted, ECK pod/zone docs in the child topics -* You can use <> to replicate data to a remote _follower_ - cluster which may be in a different data centre or even on a different - continent from the leader cluster. The follower cluster acts as a hot - standby, ready for you to fail over in the event of a disaster so severe that - the leader cluster fails. The follower cluster can also act as a geo-replica - to serve searches from nearby clients. +* *<>* ++ +To effectively distribute read and write operations across nodes, the nodes in a cluster need good, reliable connections to each other. To provide better connections, you typically co-locate the nodes in the same data center or nearby data centers. ++ +Co-locating nodes in a single location exposes you to the risk of a single outage taking your entire cluster offline. To maintain high availability, you can prepare a second cluster that can take over in case of disaster by implementing {ccr} (CCR). ++ +CCR provides a way to automatically synchronize indices from a leader cluster to a follower cluster. This cluster could be in a different data center or even a different content from the leader cluster. If the primary cluster fails, the secondary cluster can take over. ++ +TIP: You can also use CCR to create secondary clusters to serve read requests in geo-proximity to your users. -* The last line of defence against data loss is to take - <> of your cluster so that you can - restore a completely fresh copy of it elsewhere if needed. +* *<>* ++ +Take snapshots of your cluster that can be restored in case of failure. -- include::high-availability/cluster-design.asciidoc[] - -include::ccr/index.asciidoc[] diff --git a/docs/reference/how-to.asciidoc b/docs/reference/how-to.asciidoc index ec514eb05be29..eeac2fe5c2f50 100644 --- a/docs/reference/how-to.asciidoc +++ b/docs/reference/how-to.asciidoc @@ -1,23 +1,21 @@ [[how-to]] -= How to += Optimizations -[partintro] --- -Elasticsearch ships with defaults which are intended to give a good out of -the box experience. Full text search, highlighting, aggregations, and indexing -should all just work without the user having to change anything. +Elasticsearch's default settings provide a good out-of-box experience for basic operations like full text search, highlighting, aggregations, and indexing. -Once you better understand how you want to use Elasticsearch, however, -there are a number of optimizations you can make to improve performance -for your use case. +However, there are a number of optimizations you can make to improve performance for your use case. -This section provides guidance about which changes should and shouldn't be -made. --- +This section provides recommendations for various use cases. -include::how-to/general.asciidoc[] +* <> +* <> +* <> +* <> +* <> +* <> +* <> -include::how-to/recipes.asciidoc[] +include::how-to/general.asciidoc[] include::how-to/indexing-speed.asciidoc[] diff --git a/docs/reference/how-to/knn-search.asciidoc b/docs/reference/how-to/knn-search.asciidoc index 194d122cef159..60c32cabdb5c1 100644 --- a/docs/reference/how-to/knn-search.asciidoc +++ b/docs/reference/how-to/knn-search.asciidoc @@ -16,10 +16,11 @@ structures. So these same recommendations also help with indexing speed. The default <> is `float`. But this can be automatically quantized during index time through <>. Quantization will reduce the -required memory by 4x, but it will also reduce the precision of the vectors and -increase disk usage for the field (by up to 25%). Increased disk usage is a +required memory by 4x, 8x, or as much as 32x, but it will also reduce the precision of the vectors and +increase disk usage for the field (by up to 25%, 12.5%, or 3.125%, respectively). Increased disk usage is a result of {es} storing both the quantized and the unquantized vectors. -For example, when quantizing 40GB of floating point vectors an extra 10GB of data will be stored for the quantized vectors. The total disk usage amounts to 50GB, but the memory usage for fast search will be reduced to 10GB. +For example, when int8 quantizing 40GB of floating point vectors an extra 10GB of data will be stored for the quantized vectors. +The total disk usage amounts to 50GB, but the memory usage for fast search will be reduced to 10GB. For `float` vectors with `dim` greater than or equal to `384`, using a <> index is highly recommended. @@ -45,6 +46,12 @@ results contains the full document `_source`. When the documents contain high-dimensional `dense_vector` fields, the `_source` can be quite large and expensive to load. This could significantly slow down the speed of kNN search. +NOTE: <>, <>, +and <> operations generally +require the `_source` field. Disabling `_source` for a field might result in +unexpected behavior for these operations. For example, reindex might not actually +contain the `dense_vector` field in the new index. + You can disable storing `dense_vector` fields in the `_source` through the <> mapping parameter. This prevents loading and returning large vectors during search, and also cuts down on the index size. @@ -53,8 +60,7 @@ since it relies on separate data structures to perform the search. Before using the <> parameter, make sure to review the downsides of omitting fields from `_source`. -Another option is to use <> if all -your index fields support it. +Another option is to use <>. [discrete] === Ensure data nodes have enough memory @@ -63,12 +69,21 @@ your index fields support it. kNN search. HNSW is a graph-based algorithm which only works efficiently when most vector data is held in memory. You should ensure that data nodes have at least enough RAM to hold the vector data and index structures. To check the -size of the vector data, you can use the <> API. As a -loose rule of thumb, and assuming the default HNSW options, the bytes used will -be `num_vectors * 4 * (num_dimensions + 12)`. When using the `byte` <> -the space required will be closer to `num_vectors * (num_dimensions + 12)`. Note that -the required RAM is for the filesystem cache, which is separate from the Java -heap. +size of the vector data, you can use the <> API. + +Here are estimates for different element types and quantization levels: + +* `element_type: float`: `num_vectors * num_dimensions * 4` +* `element_type: float` with `quantization: int8`: `num_vectors * (num_dimensions + 4)` +* `element_type: float` with `quantization: int4`: `num_vectors * (num_dimensions/2 + 4)` +* `element_type: float` with `quantization: bbq`: `num_vectors * (num_dimensions/8 + 12)` +* `element_type: byte`: `num_vectors * num_dimensions` +* `element_type: bit`: `num_vectors * (num_dimensions/8)` + +If utilizing HNSW, the graph must also be in memory, to estimate the required bytes use `num_vectors * 4 * HNSW.m`. The +default value for `HNSW.m` is 16, so by default `num_vectors * 4 * 16`. + +Note that the required RAM is for the filesystem cache, which is separate from the Java heap. The data nodes should also leave a buffer for other ways that RAM is needed. For example your index might also include text fields and numerics, which also @@ -80,13 +95,20 @@ and https://elasticsearch-benchmarks.elastic.co/#tracks/dense_vector[here] some of datasets and configurations that we use for our nightly benchmarks. [discrete] +[[dense-vector-preloading]] include::search-speed.asciidoc[tag=warm-fs-cache] The following file extensions are used for the approximate kNN search: +Each extension is broken down by the quantization types. + +* `vex` for the HNSW graph +* `vec` for all non-quantized vector values. This includes all element types: `float`, `byte`, and `bit`. +* `veq` for quantized vectors indexed with <>: `int4` or `int8` +* `veb` for binary vectors indexed with <>: `bbq` +* `vem`, `vemf`, `vemq`, and `vemb` for metadata, usually small and not a concern for preloading -* `vec` and `veq` for vector values -* `vex` for HNSW graph -* `vem`, `vemf`, and `vemq` for metadata +Generally, if you are using a quantized index, you should only preload the relevant quantized values and the HNSW graph. +Preloading the raw vectors is not necessary and might be counterproductive. [discrete] === Reduce the number of index segments @@ -102,14 +124,14 @@ merges smaller segments into larger ones through a background explicit steps to reduce the number of index segments. [discrete] -==== Force merge to one segment - -The <> operation forces an index merge. If you -force merge to one segment, the kNN search only need to check a single, -all-inclusive HNSW graph. Force merging `dense_vector` fields is an expensive -operation that can take significant time to complete. - -include::{es-ref-dir}/indices/forcemerge.asciidoc[tag=force-merge-read-only-warn] +==== Increase maximum segment size + +{es} provides many tunable settings for controlling the merge process. One +important setting is `index.merge.policy.max_merged_segment`. This controls +the maximum size of the segments that are created during the merge process. +By increasing the value, you can reduce the number of segments in the index. +The default value is `5GB`, but that might be too small for larger dimensional vectors. +Consider increasing this value to `10GB` or `20GB` can help reduce the number of segments. [discrete] ==== Create large segments during bulk indexing diff --git a/docs/reference/how-to/recipes.asciidoc b/docs/reference/how-to/recipes.asciidoc index b46f624aef51d..de23404be6164 100644 --- a/docs/reference/how-to/recipes.asciidoc +++ b/docs/reference/how-to/recipes.asciidoc @@ -1,7 +1,7 @@ [[recipes]] -== Recipes +== Search relevance optimizations -This section includes a few recipes to help with common problems: +This section includes a few recipes to help with common search relevance issues: * <> * <> diff --git a/docs/reference/how-to/recipes/scoring.asciidoc b/docs/reference/how-to/recipes/scoring.asciidoc index 5c5a8977d34d4..a578826e31fac 100644 --- a/docs/reference/how-to/recipes/scoring.asciidoc +++ b/docs/reference/how-to/recipes/scoring.asciidoc @@ -88,8 +88,9 @@ pages independently of the query. There are two main queries that allow combining static score contributions with textual relevance, eg. as computed with BM25: - - <> - - <> + +* <> +* <> For instance imagine that you have a `pagerank` field that you wish to combine with the BM25 score so that the final score is equal to diff --git a/docs/reference/how-to/search-speed.asciidoc b/docs/reference/how-to/search-speed.asciidoc index 0ef55d7808873..2db03a19b3532 100644 --- a/docs/reference/how-to/search-speed.asciidoc +++ b/docs/reference/how-to/search-speed.asciidoc @@ -567,3 +567,8 @@ also possible to update the client-side logic in order to route queries to the relevant indices based on filters. However `constant_keyword` makes it transparently and allows to decouple search requests from the index topology in exchange of very little overhead. + +[discrete] +=== Default search timeout + +By default, search requests don't time out. You can set a timeout using the <> setting. diff --git a/docs/reference/ilm/actions/ilm-forcemerge.asciidoc b/docs/reference/ilm/actions/ilm-forcemerge.asciidoc index ef74e462d4bf2..24c3c08c24668 100644 --- a/docs/reference/ilm/actions/ilm-forcemerge.asciidoc +++ b/docs/reference/ilm/actions/ilm-forcemerge.asciidoc @@ -49,7 +49,7 @@ Number of segments to merge to. To fully merge the index, set to `1`. `index_codec`:: (Optional, string) Codec used to compress the document store. The only accepted value is -`best_compression`, which uses {wikipedia}/DEFLATE[DEFLATE] for a higher +`best_compression`, which uses {wikipedia}/Zstd[ZSTD] for a higher compression ratio but slower stored fields performance. To use the default LZ4 codec, omit this argument. + diff --git a/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc b/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc index 4ba4782174bef..73a77bef09bde 100644 --- a/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc +++ b/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc @@ -19,7 +19,7 @@ index>> prefixed with `partial-` to the frozen tier. In other phases, the action In the frozen tier, the action will ignore the setting <>, if it was present in the original index, -to account for the difference in the number of nodes between the frozen and the other tiers. +to account for the difference in the number of nodes between the frozen and the other tiers. To set <> for searchable snapshots, set the `total_shards_per_node` option in the frozen phase's `searchable_snapshot` action within the ILM policy. WARNING: Don't include the `searchable_snapshot` action in both the hot and cold @@ -74,6 +74,9 @@ will be performed on the hot nodes. If using a `searchable_snapshot` action in t force merge will be performed on whatever tier the index is *prior* to the `cold` phase (either `hot` or `warm`). +`total_shards_per_node`:: +The maximum number of shards (replicas and primaries) that will be allocated to a single node for the searchable snapshot index. Defaults to unbounded. + [[ilm-searchable-snapshot-ex]] ==== Examples //// diff --git a/docs/reference/ilm/apis/delete-lifecycle.asciidoc b/docs/reference/ilm/apis/delete-lifecycle.asciidoc index fc9a35e4ef570..64a421f091eff 100644 --- a/docs/reference/ilm/apis/delete-lifecycle.asciidoc +++ b/docs/reference/ilm/apis/delete-lifecycle.asciidoc @@ -5,6 +5,12 @@ Delete policy ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ilm[{ilm-cap} APIs]. +-- + Deletes an index <> policy. [[ilm-delete-lifecycle-request]] diff --git a/docs/reference/ilm/apis/explain.asciidoc b/docs/reference/ilm/apis/explain.asciidoc index a1ddde8c9f2d9..4cc2667186442 100644 --- a/docs/reference/ilm/apis/explain.asciidoc +++ b/docs/reference/ilm/apis/explain.asciidoc @@ -5,6 +5,12 @@ Explain lifecycle ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ilm[{ilm-cap} APIs]. +-- + Retrieves the current <> status for one or more indices. For data streams, the API retrieves the current lifecycle status for the stream's backing indices. @@ -303,6 +309,12 @@ the case. "index_uuid": "H7lF9n36Rzqa-KfKcnGQMg", "index": "test-000057" }, + "previous_step_info": { <5> + "type": "cluster_block_exception", + "reason": "index [test-000057/H7lF9n36Rzqa-KfKcnGQMg] blocked by: [FORBIDDEN/5/index read-only (api)", + "index_uuid": "H7lF9n36Rzqa-KfKcnGQMg", + "index": "test-000057" + }, "phase_execution": { "policy": "my_lifecycle3", "phase_definition": { @@ -329,3 +341,4 @@ is true, {ilm-init} will retry the failed step automatically. <3> Shows the number of attempted automatic retries to execute the failed step. <4> What went wrong +<5> Contains a copy of the `step_info` field (when it exists) of the last attempted or executed step for diagnostic purposes, since the `step_info` is overwritten during each new attempt. diff --git a/docs/reference/ilm/apis/get-lifecycle.asciidoc b/docs/reference/ilm/apis/get-lifecycle.asciidoc index b4e07389a9fb7..b02d129ebe734 100644 --- a/docs/reference/ilm/apis/get-lifecycle.asciidoc +++ b/docs/reference/ilm/apis/get-lifecycle.asciidoc @@ -5,6 +5,12 @@ Get policy ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ilm[{ilm-cap} APIs]. +-- + Retrieves a <> policy. [[ilm-get-lifecycle-request]] diff --git a/docs/reference/ilm/apis/get-status.asciidoc b/docs/reference/ilm/apis/get-status.asciidoc index f2ab8d65ec9a1..648080f26b79b 100644 --- a/docs/reference/ilm/apis/get-status.asciidoc +++ b/docs/reference/ilm/apis/get-status.asciidoc @@ -7,6 +7,12 @@ Get {ilm} status ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ilm[{ilm-cap} APIs]. +-- + Retrieves the current <> ({ilm-init}) status. You can start or stop {ilm-init} with the <> and diff --git a/docs/reference/ilm/apis/ilm-api.asciidoc b/docs/reference/ilm/apis/ilm-api.asciidoc index 149ba2a6b4491..9cc594be1bea7 100644 --- a/docs/reference/ilm/apis/ilm-api.asciidoc +++ b/docs/reference/ilm/apis/ilm-api.asciidoc @@ -1,6 +1,12 @@ [[index-lifecycle-management-api]] == {ilm-cap} APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ilm[{ilm-cap} APIs]. +-- + You use the following APIs to set up policies to automatically manage the index lifecycle. For more information about {ilm} ({ilm-init}), see <>. diff --git a/docs/reference/ilm/apis/migrate-to-data-tiers.asciidoc b/docs/reference/ilm/apis/migrate-to-data-tiers.asciidoc index 8ba57120a8a65..76810170daa19 100644 --- a/docs/reference/ilm/apis/migrate-to-data-tiers.asciidoc +++ b/docs/reference/ilm/apis/migrate-to-data-tiers.asciidoc @@ -5,6 +5,12 @@ Migrate indices, ILM policies, and legacy, composable and component templates to data tiers routing ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ilm[{ilm-cap} APIs]. +-- + Switches the indices, ILM policies, and legacy, composable and component templates from using custom node attributes and <> to using <>, and optionally deletes one legacy index template. diff --git a/docs/reference/ilm/apis/move-to-step.asciidoc b/docs/reference/ilm/apis/move-to-step.asciidoc index f3441fa997cff..a7a8cacf551db 100644 --- a/docs/reference/ilm/apis/move-to-step.asciidoc +++ b/docs/reference/ilm/apis/move-to-step.asciidoc @@ -5,6 +5,12 @@ Move to step ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ilm[{ilm-cap} APIs]. +-- + Triggers execution of a specific step in the <> policy. [[ilm-move-to-step-request]] diff --git a/docs/reference/ilm/apis/put-lifecycle.asciidoc b/docs/reference/ilm/apis/put-lifecycle.asciidoc index 390f6b1bb4d15..3aa691ab06da9 100644 --- a/docs/reference/ilm/apis/put-lifecycle.asciidoc +++ b/docs/reference/ilm/apis/put-lifecycle.asciidoc @@ -5,6 +5,12 @@ Create or update lifecycle policy ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ilm[{ilm-cap} APIs]. +-- + Creates or updates <> policy. See <> for definitions of policy components. diff --git a/docs/reference/ilm/apis/remove-policy-from-index.asciidoc b/docs/reference/ilm/apis/remove-policy-from-index.asciidoc index 107cab4d5aa19..5b5842f28619c 100644 --- a/docs/reference/ilm/apis/remove-policy-from-index.asciidoc +++ b/docs/reference/ilm/apis/remove-policy-from-index.asciidoc @@ -5,6 +5,12 @@ Remove policy ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ilm[{ilm-cap} APIs]. +-- + Removes assigned <> policies from an index or a data stream's backing indices. diff --git a/docs/reference/ilm/apis/retry-policy.asciidoc b/docs/reference/ilm/apis/retry-policy.asciidoc index 8f01f15e0c3ad..a41d064b73400 100644 --- a/docs/reference/ilm/apis/retry-policy.asciidoc +++ b/docs/reference/ilm/apis/retry-policy.asciidoc @@ -5,6 +5,12 @@ Retry policy ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ilm[{ilm-cap} APIs]. +-- + Retry executing the <> policy for an index that is in the ERROR step. [[ilm-retry-policy-request]] diff --git a/docs/reference/ilm/apis/start.asciidoc b/docs/reference/ilm/apis/start.asciidoc index c38b3d9ca8831..ce9b64455be05 100644 --- a/docs/reference/ilm/apis/start.asciidoc +++ b/docs/reference/ilm/apis/start.asciidoc @@ -7,6 +7,12 @@ Start {ilm} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ilm[{ilm-cap} APIs]. +-- + Start the <> ({ilm-init}) plugin. [[ilm-start-request]] diff --git a/docs/reference/ilm/apis/stop.asciidoc b/docs/reference/ilm/apis/stop.asciidoc index a6100d794c2d3..50d8aaf4e1fd6 100644 --- a/docs/reference/ilm/apis/stop.asciidoc +++ b/docs/reference/ilm/apis/stop.asciidoc @@ -7,6 +7,12 @@ Stop {ilm} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ilm[{ilm-cap} APIs]. +-- + Stop the <> ({ilm-init}) plugin. [[ilm-stop-request]] diff --git a/docs/reference/ilm/error-handling.asciidoc b/docs/reference/ilm/error-handling.asciidoc index f810afc6c2b5f..911dc8b9cce40 100644 --- a/docs/reference/ilm/error-handling.asciidoc +++ b/docs/reference/ilm/error-handling.asciidoc @@ -8,6 +8,10 @@ When this happens, {ilm-init} moves the index to an `ERROR` step. If {ilm-init} cannot resolve the error automatically, execution is halted until you resolve the underlying issues with the policy, index, or cluster. +See https://www.youtube.com/watch?v=VCIqkji3IwY[this video] +for a walkthrough of troubleshooting current {ilm-init} health issues, and https://www.youtube.com/watch?v=onrnnwjYWSQ[this video] +for a walkthrough of troubleshooting historical {ilm-init} issues. + For example, you might have a `shrink-index` policy that shrinks an index to four shards once it is at least five days old: @@ -183,6 +187,8 @@ The rollover action then manages setting and updating the alias to Do not explicitly configure this same alias in the aliases section of an index template. +See this https://www.youtube.com/watch?v=Ww5POq4zZtY[resolving `duplicate alias` video] for an example troubleshooting walkthrough. + [discrete] ==== index.lifecycle.rollover_alias [x] does not point to index [y] @@ -191,6 +197,8 @@ Either the index is using the wrong alias or the alias does not exist. Check the `index.lifecycle.rollover_alias` <>. To see what aliases are configured, use <>. +See this https://www.youtube.com/watch?v=NKSe67x7aw8[resolving `not point to index` video] for an example troubleshooting walkthrough. + [discrete] ==== Setting [index.lifecycle.rollover_alias] for index [y] is empty or not defined @@ -198,6 +206,8 @@ The `index.lifecycle.rollover_alias` setting must be configured for the rollover Update the index settings to set `index.lifecycle.rollover_alias`. +See this https://www.youtube.com/watch?v=LRpMC2GS_FQ[resolving `empty or not defined` video] for an example troubleshooting walkthrough. + [discrete] ==== Alias [x] has more than one write index [y,z] @@ -205,6 +215,8 @@ Only one index can be designated as the write index for a particular alias. Use the <> API to set `is_write_index:false` for all but one index. +See this https://www.youtube.com/watch?v=jCUvZCT5Hm4[resolving `more than one write index` video] for an example troubleshooting walkthrough. + [discrete] ==== index name [x] does not match pattern ^.*-\d+ @@ -214,6 +226,8 @@ For example, `my-index` does not match the pattern requirement. Append a numeric value to the index name, for example `my-index-000001`. +See this https://www.youtube.com/watch?v=9sp1zF6iL00[resolving `does not match pattern` video] for an example troubleshooting walkthrough. + [discrete] ==== CircuitBreakingException: [x] data too large, data for [y] @@ -227,8 +241,7 @@ For more information, see <>. This indicates that the cluster is running out of disk space. This can happen when you don't have {ilm} set up to roll over from hot to warm nodes. - -Consider adding nodes, upgrading your hardware, or deleting unneeded indices. +For more information, see <>. [discrete] ==== security_exception: action [] is unauthorized for user [] with roles [], this action is granted by the index privileges [manage_follow_index,manage,all] diff --git a/docs/reference/ilm/example-index-lifecycle-policy.asciidoc b/docs/reference/ilm/example-index-lifecycle-policy.asciidoc index 6ec261fabc448..0b3c17fb2caae 100644 --- a/docs/reference/ilm/example-index-lifecycle-policy.asciidoc +++ b/docs/reference/ilm/example-index-lifecycle-policy.asciidoc @@ -24,7 +24,7 @@ and retention requirements. You want to send log files to an {es} cluster so you can visualize and analyze the data. This data has the following retention requirements: -* When the write index reaches 50GB or is 30 days old, roll over to a new index. +* When the primary shard size of the write index reaches 50GB or the index is 30 days old, roll over to a new index. * After rollover, keep indices in the hot data tier for 30 days. * 30 days after rollover: ** Move indices to the warm data tier. @@ -84,7 +84,7 @@ To save the `logs@lifecycle` policy as a new policy in {kib}: . On the **Edit policy logs** page, toggle **Save as new policy**, and then provide a new name for the policy, for example, `logs-custom`. The `logs@lifecycle` policy uses the recommended rollover defaults: Start writing to a new -index when the current write index reaches 50GB or becomes 30 days old. +index when the primary shard size of the current write index reaches 50GB or the index becomes 30 days old. To view or change the rollover settings, click **Advanced settings** for the hot phase. Then disable **Use recommended defaults** to display the rollover diff --git a/docs/reference/images/esql/esql-dashboard-panel.png b/docs/reference/images/esql/esql-dashboard-panel.png index d621d1170edcf..61b44f7c9f857 100644 Binary files a/docs/reference/images/esql/esql-dashboard-panel.png and b/docs/reference/images/esql/esql-dashboard-panel.png differ diff --git a/docs/reference/images/esql/esql-discover-query-history.png b/docs/reference/images/esql/esql-discover-query-history.png index da31e4a6acce4..eb064684af700 100644 Binary files a/docs/reference/images/esql/esql-discover-query-history.png and b/docs/reference/images/esql/esql-discover-query-history.png differ diff --git a/docs/reference/images/esql/esql-discover-query-starred.png b/docs/reference/images/esql/esql-discover-query-starred.png new file mode 100644 index 0000000000000..525aa9acbea28 Binary files /dev/null and b/docs/reference/images/esql/esql-discover-query-starred.png differ diff --git a/docs/reference/images/esql/esql-discover-show-recent-query.png b/docs/reference/images/esql/esql-discover-show-recent-query.png deleted file mode 100644 index 13c8df9965ea3..0000000000000 Binary files a/docs/reference/images/esql/esql-discover-show-recent-query.png and /dev/null differ diff --git a/docs/reference/images/esql/esql-kibana-auto-complete.png b/docs/reference/images/esql/esql-kibana-auto-complete.png index d50d6b133442f..155df2447dd6c 100644 Binary files a/docs/reference/images/esql/esql-kibana-auto-complete.png and b/docs/reference/images/esql/esql-kibana-auto-complete.png differ diff --git a/docs/reference/images/esql/esql-kibana-bar-chart.png b/docs/reference/images/esql/esql-kibana-bar-chart.png index a760d3d69920e..b74b33710d908 100644 Binary files a/docs/reference/images/esql/esql-kibana-bar-chart.png and b/docs/reference/images/esql/esql-kibana-bar-chart.png differ diff --git a/docs/reference/images/esql/esql-kibana-create-rule.png b/docs/reference/images/esql/esql-kibana-create-rule.png index c9fb14b0d2ee9..a763c6f366df0 100644 Binary files a/docs/reference/images/esql/esql-kibana-create-rule.png and b/docs/reference/images/esql/esql-kibana-create-rule.png differ diff --git a/docs/reference/images/esql/esql-kibana-edit-on-dashboard.png b/docs/reference/images/esql/esql-kibana-edit-on-dashboard.png index 14f6be81af7df..348b77150ed87 100644 Binary files a/docs/reference/images/esql/esql-kibana-edit-on-dashboard.png and b/docs/reference/images/esql/esql-kibana-edit-on-dashboard.png differ diff --git a/docs/reference/images/esql/esql-kibana-enrich-autocomplete.png b/docs/reference/images/esql/esql-kibana-enrich-autocomplete.png index 95a997ca2ac30..f2a0779be348a 100644 Binary files a/docs/reference/images/esql/esql-kibana-enrich-autocomplete.png and b/docs/reference/images/esql/esql-kibana-enrich-autocomplete.png differ diff --git a/docs/reference/images/esql/esql-kibana-in-line-editor.png b/docs/reference/images/esql/esql-kibana-in-line-editor.png index 7b7a11e532226..85631896e833f 100644 Binary files a/docs/reference/images/esql/esql-kibana-in-line-editor.png and b/docs/reference/images/esql/esql-kibana-in-line-editor.png differ diff --git a/docs/reference/images/ilm/tutorial-ilm-hotphaserollover-default.png b/docs/reference/images/ilm/tutorial-ilm-hotphaserollover-default.png index 14ff66e410835..d7f314cedb261 100644 Binary files a/docs/reference/images/ilm/tutorial-ilm-hotphaserollover-default.png and b/docs/reference/images/ilm/tutorial-ilm-hotphaserollover-default.png differ diff --git a/docs/reference/images/index-mgmt/management-data-stream-fields.png b/docs/reference/images/index-mgmt/management-data-stream-fields.png new file mode 100644 index 0000000000000..605d49b80ab1f Binary files /dev/null and b/docs/reference/images/index-mgmt/management-data-stream-fields.png differ diff --git a/docs/reference/images/index-mgmt/management-data-stream.png b/docs/reference/images/index-mgmt/management-data-stream.png deleted file mode 100644 index 01534fdec2a23..0000000000000 Binary files a/docs/reference/images/index-mgmt/management-data-stream.png and /dev/null differ diff --git a/docs/reference/images/index-mgmt/management-index-templates.png b/docs/reference/images/index-mgmt/management-index-templates.png index 9188aa85e68cd..1ed004e85e71d 100644 Binary files a/docs/reference/images/index-mgmt/management-index-templates.png and b/docs/reference/images/index-mgmt/management-index-templates.png differ diff --git a/docs/reference/images/search/full-text-search-overview.svg b/docs/reference/images/search/full-text-search-overview.svg new file mode 100644 index 0000000000000..e7a1c5ba14cfa --- /dev/null +++ b/docs/reference/images/search/full-text-search-overview.svg @@ -0,0 +1,81 @@ + + + + + + + + + + + + + + + Full-text search with Elasticsearch + + + + + Source documents + + + + Analysis pipeline + Transforms text to normalized terms + + + + Inverted index + Search-optimized data structure + + + + Search query + + + + Relevance scoring + Similarity algorithm scores documents + + + + Search results + Most relevant first + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/reference/images/search/rag-schema.svg b/docs/reference/images/search/rag-schema.svg new file mode 100644 index 0000000000000..f26edac6c0077 --- /dev/null +++ b/docs/reference/images/search/rag-schema.svg @@ -0,0 +1,73 @@ + + + + + + + + + + + + + + + Define how the model should + parse and render information + + Custom instructions + + + + + + Full-text, semantic + or hybrid search + + Search strategy + + + + + + + 1 + + + 2 + + + 3 + + + 4 + + + + + + User query + + + + Elasticsearch + + Retrieves relevant + documents + + + + Language model + Processes context & + + generates answer + + + + + + + + + Response + \ No newline at end of file diff --git a/docs/reference/images/search/rag-venn-diagram.svg b/docs/reference/images/search/rag-venn-diagram.svg new file mode 100644 index 0000000000000..9906aaefaba0c --- /dev/null +++ b/docs/reference/images/search/rag-venn-diagram.svg @@ -0,0 +1,19 @@ + + + + + + + + + Information + retrieval + + + Generative + AI + + + + RAG + \ No newline at end of file diff --git a/docs/reference/images/semantic-options.svg b/docs/reference/images/semantic-options.svg new file mode 100644 index 0000000000000..3bedf5307357e --- /dev/null +++ b/docs/reference/images/semantic-options.svg @@ -0,0 +1,62 @@ + + + + Elasticsearch semantic search workflows + + + + + + semantic_text + (Recommended) + + + + Inference API + + + + Model Deployment + + + Complexity: Low + Complexity: Medium + Complexity: High + + + + + + Create Inference Endpoint + + + Define Index Mapping + + + + Create Inference Endpoint + + + Configure Model Settings + + + Define Index Mapping + + + Setup Ingest Pipeline + + + + Select NLP Model + + + Deploy with Eland Client + + + Define Index Mapping + + + Setup Ingest Pipeline + + + diff --git a/docs/reference/index-modules.asciidoc b/docs/reference/index-modules.asciidoc index 7232de12c8c50..73e2db6e45e34 100644 --- a/docs/reference/index-modules.asciidoc +++ b/docs/reference/index-modules.asciidoc @@ -76,14 +76,16 @@ breaking change]. The +default+ value compresses stored data with LZ4 compression, but this can be set to +best_compression+ - which uses {wikipedia}/DEFLATE[DEFLATE] for a higher - compression ratio, at the expense of slower stored fields performance. + which uses {wikipedia}/Zstd[ZSTD] for a higher + compression ratio, at the expense of slower stored fields read performance. If you are updating the compression type, the new one will be applied after segments are merged. Segment merging can be forced using <>. Experiments with indexing log datasets - have shown that `best_compression` gives up to ~18% lower storage usage in - the most ideal scenario compared to `default` while only minimally affecting - indexing throughput (~2%). + have shown that `best_compression` gives up to ~28% lower storage usage and + similar indexing throughput (sometimes a bit slower or faster depending on other used options) compared + to `default` while affecting get by id latencies between ~10% and ~33%. The higher get + by id latencies is not a concern for many use cases like logging or metrics, since + these don't really rely on get by id functionality (Get APIs or searching by _id). [[index-mode-setting]] `index.mode`:: + @@ -111,16 +113,15 @@ Index mode supports the following values: `standard`::: Standard indexing with default settings. -`time_series`::: Index mode optimized for storage of metrics documented in <>. +`time_series`::: _(data streams only)_ Index mode optimized for storage of metrics. For more information, see <>. -`logsdb`::: Index mode optimized for storage of logs. It applies default sort settings on the `hostname` and `timestamp` fields and uses <>. <> on different fields is still allowed. -preview:[] +`logsdb`::: _(data streams only)_ Index mode optimized for <>. [[routing-partition-size]] `index.routing_partition_size`:: The number of shards a custom <> value can go to. Defaults to 1 and can only be set at index creation time. This value must be less - than the `index.number_of_shards` unless the `index.number_of_shards` value is also 1. + than the `index.number_of_routing_shards` unless the `index.number_of_routing_shards` value is also 1. See <> for more details about how this setting is used. [[ccr-index-soft-deletes]] diff --git a/docs/reference/index-modules/slowlog.asciidoc b/docs/reference/index-modules/slowlog.asciidoc index c29296b59ad4a..e848668c1a66d 100644 --- a/docs/reference/index-modules/slowlog.asciidoc +++ b/docs/reference/index-modules/slowlog.asciidoc @@ -1,15 +1,118 @@ [[index-modules-slowlog]] -== Slow Log +== Slow log + +The slow log records database searching and indexing events that have execution durations above specified thresholds. You can use these logs to investigate analyze or troubleshoot your cluster's historical search and indexing performance. + +Slow logs report task duration at the shard level for searches, and at the index level +for indexing, but might not encompass the full task execution time observed on the client. For example, slow logs don't surface HTTP network delays or the impact of <>. + +Events that meet the specified threshold are emitted into <> under the `fileset.name` of `slowlog`. These logs can be viewed in the following locations: + +* If <> is enabled, from +{kibana-ref}/xpack-monitoring.html[Stack Monitoring]. Slow log events have a `logger` value of `index.search.slowlog` or `index.indexing.slowlog`. + +* From local {es} service logs directory. Slow log files have a suffix of `_index_search_slowlog.json` or `_index_indexing_slowlog.json`. + +[discrete] +[[slow-log-format]] +=== Slow log format + +The following is an example of a search event in the slow log: + +TIP: If a call was initiated with an `X-Opaque-ID` header, then the ID is automatically included in Search slow logs in the **elasticsearch.slowlog.id** field. See <> for details and best practices. + +[source,js] +--------------------------- +{ + "@timestamp": "2024-12-21T12:42:37.255Z", + "auth.type": "REALM", + "ecs.version": "1.2.0", + "elasticsearch.cluster.name": "distribution_run", + "elasticsearch.cluster.uuid": "Ui23kfF1SHKJwu_hI1iPPQ", + "elasticsearch.node.id": "JK-jn-XpQ3OsDUsq5ZtfGg", + "elasticsearch.node.name": "node-0", + "elasticsearch.slowlog.id": "tomcat-123", + "elasticsearch.slowlog.message": "[index6][0]", + "elasticsearch.slowlog.search_type": "QUERY_THEN_FETCH", + "elasticsearch.slowlog.source": "{\"query\":{\"match_all\":{\"boost\":1.0}}}", + "elasticsearch.slowlog.stats": "[]", + "elasticsearch.slowlog.took": "747.3micros", + "elasticsearch.slowlog.took_millis": 0, + "elasticsearch.slowlog.total_hits": "1 hits", + "elasticsearch.slowlog.total_shards": 1, + "event.dataset": "elasticsearch.index_search_slowlog", + "fileset.name" : "slowlog", + "log.level": "WARN", + "log.logger": "index.search.slowlog.query", + "process.thread.name": "elasticsearch[runTask-0][search][T#5]", + "service.name": "ES_ECS", + "user.name": "elastic", + "user.realm": "reserved" +} + +--------------------------- +// NOTCONSOLE + + +The following is an example of an indexing event in the slow log: + +[source,js] +--------------------------- +{ + "@timestamp" : "2024-12-11T22:34:22.613Z", + "auth.type": "REALM", + "ecs.version": "1.2.0", + "elasticsearch.cluster.name" : "41bd111609d849fc9bf9d25b5df9ce96", + "elasticsearch.cluster.uuid" : "BZTn4I9URXSK26imlia0QA", + "elasticsearch.index.id" : "3VfGR7wRRRKmMCEn7Ii58g", + "elasticsearch.index.name": "my-index-000001", + "elasticsearch.node.id" : "GGiBgg21S3eqPDHzQiCMvQ", + "elasticsearch.node.name" : "instance-0000000001", + "elasticsearch.slowlog.id" : "RCHbt5MBT0oSsCOu54AJ", + "elasticsearch.slowlog.source": "{\"key\":\"value\"}" + "elasticsearch.slowlog.took" : "0.01ms", + "event.dataset": "elasticsearch.index_indexing_slowlog", + "fileset.name" : "slowlog", + "log.level" : "TRACE", + "log.logger" : "index.indexing.slowlog.index", + "service.name" : "ES_ECS", + "user.name": "elastic", + "user.realm": "reserved" +} + +--------------------------- +// NOTCONSOLE + +[discrete] +[[enable-slow-log]] +=== Enable slow logging + +You can enable slow logging at two levels: + +* For all indices under the <>. This method requires a node restart. +* At the index level, using the <> + +By default, all thresholds are set to `-1`, which results in no events being logged. + +Slow log thresholds can be enabled for the four logging levels: `trace`, `debug`, `info`, and `warn`. You can mimic setting log level thresholds by disabling more verbose levels. + +To view the current slow log settings, use the <>: + +[source,console] +-------------------------------------------------- +GET _all/_settings?expand_wildcards=all&filter_path=*.settings.index.*.slowlog +-------------------------------------------------- [discrete] [[search-slow-log]] -=== Search Slow Log +==== Enable slow logging for search events + +Search slow logs emit per shard. They must be enabled separately for the shard's link:https://www.elastic.co/blog/understanding-query-then-fetch-vs-dfs-query-then-fetch[query and fetch search phases]. -Shard level slow search log allows to log slow search (query and fetch -phases) into a dedicated log file. +You can use the `index.search.slowlog.include.user` setting to append `user.*` and `auth.type` fields to slow log entries. These fields contain information about the user who triggered the request. -Thresholds can be set for both the query phase of the execution, and -fetch phase, here is a sample: +The following snippet adjusts all available search slow log settings across all indices using the +<>: [source,yaml] -------------------------------------------------- @@ -22,10 +125,11 @@ index.search.slowlog.threshold.fetch.warn: 1s index.search.slowlog.threshold.fetch.info: 800ms index.search.slowlog.threshold.fetch.debug: 500ms index.search.slowlog.threshold.fetch.trace: 200ms + +index.search.slowlog.include.user: true -------------------------------------------------- -All of the above settings are _dynamic_ and can be set for each index using the -<> API. For example: +The following snippet adjusts the same settings for a single index using the <>: [source,console] -------------------------------------------------- @@ -38,138 +142,109 @@ PUT /my-index-000001/_settings "index.search.slowlog.threshold.fetch.warn": "1s", "index.search.slowlog.threshold.fetch.info": "800ms", "index.search.slowlog.threshold.fetch.debug": "500ms", - "index.search.slowlog.threshold.fetch.trace": "200ms" + "index.search.slowlog.threshold.fetch.trace": "200ms", + "index.search.slowlog.include.user": true } -------------------------------------------------- // TEST[setup:my_index] -By default thresholds are disabled (set to `-1`). -The logging is done on the shard level scope, meaning the execution of a -search request within a specific shard. It does not encompass the whole -search request, which can be broadcast to several shards in order to -execute. Some of the benefits of shard level logging is the association -of the actual execution on the specific machine, compared with request -level. +[discrete] +[[index-slow-log]] +==== Enable slow logging for indexing events +Indexing slow logs emit per index document. -The search slow log file is configured in the `log4j2.properties` file. +You can use the `index.indexing.slowlog.include.user` setting to append `user.*` and `auth.type` fields to slow log entries. These fields contain information about the user who triggered the request. -[discrete] -==== Identifying search slow log origin +The following snippet adjusts all available indexing slow log settings across all indices using the +<>: -It is often useful to identify what triggered a slow running query. -To include information about the user that triggered a slow search, -use the `index.search.slowlog.include.user` setting. +[source,yaml] +-------------------------------------------------- +index.indexing.slowlog.threshold.index.warn: 10s +index.indexing.slowlog.threshold.index.info: 5s +index.indexing.slowlog.threshold.index.debug: 2s +index.indexing.slowlog.threshold.index.trace: 500ms + +index.indexing.slowlog.source: 1000 +index.indexing.slowlog.reformat: true + +index.indexing.slowlog.include.user: true +-------------------------------------------------- + + +The following snippet adjusts the same settings for a single index using the <>: [source,console] -------------------------------------------------- PUT /my-index-000001/_settings { - "index.search.slowlog.include.user": true + "index.indexing.slowlog.threshold.index.warn": "10s", + "index.indexing.slowlog.threshold.index.info": "5s", + "index.indexing.slowlog.threshold.index.debug": "2s", + "index.indexing.slowlog.threshold.index.trace": "500ms", + "index.indexing.slowlog.source": "1000", + "index.indexing.slowlog.reformat": true, + "index.indexing.slowlog.include.user": true } -------------------------------------------------- // TEST[setup:my_index] -This will result in user information being included in the slow log. +[discrete] +===== Logging the `_source` field -[source,js] ---------------------------- -{ - "@timestamp": "2024-02-21T12:42:37.255Z", - "log.level": "WARN", - "auth.type": "REALM", - "elasticsearch.slowlog.id": "tomcat-123", - "elasticsearch.slowlog.message": "[index6][0]", - "elasticsearch.slowlog.search_type": "QUERY_THEN_FETCH", - "elasticsearch.slowlog.source": "{\"query\":{\"match_all\":{\"boost\":1.0}}}", - "elasticsearch.slowlog.stats": "[]", - "elasticsearch.slowlog.took": "747.3micros", - "elasticsearch.slowlog.took_millis": 0, - "elasticsearch.slowlog.total_hits": "1 hits", - "elasticsearch.slowlog.total_shards": 1, - "user.name": "elastic", - "user.realm": "reserved", - "ecs.version": "1.2.0", - "service.name": "ES_ECS", - "event.dataset": "elasticsearch.index_search_slowlog", - "process.thread.name": "elasticsearch[runTask-0][search][T#5]", - "log.logger": "index.search.slowlog.query", - "elasticsearch.cluster.uuid": "Ui23kfF1SHKJwu_hI1iPPQ", - "elasticsearch.node.id": "JK-jn-XpQ3OsDUsq5ZtfGg", - "elasticsearch.node.name": "node-0", - "elasticsearch.cluster.name": "distribution_run" -} +By default, {es} logs the first 1000 characters of the `_source` in the slow log. You can adjust how `_source` is logged using the `index.indexing.slowlog.source` setting. Set `index.indexing.slowlog.source` to `false` or `0` to skip logging the source entirely. Set `index.indexing.slowlog.source` to `true` to log the entire source regardless of size. ---------------------------- -// NOTCONSOLE +The original `_source` is reformatted by default to make sure that it fits on a single log line. If preserving the original document format is important, then you can turn off reformatting by setting `index.indexing.slowlog.reformat` to `false`. This causes source to be logged with the original formatting intact, potentially spanning multiple log lines. -If a call was initiated with an `X-Opaque-ID` header, then the ID is included -in Search Slow logs in the **elasticsearch.slowlog.id** field. See -<> for details and best practices. +[discrete] +[[slow-log-fields]] [discrete] -[[index-slow-log]] -=== Index Slow log +[[troubleshoot-slow-log]] +=== Best practices for slow logging -The indexing slow log, similar in functionality to the search slow -log. The log file name ends with `_index_indexing_slowlog.json`. Log and -the thresholds are configured in the same way as the search slowlog. -Index slowlog sample: +Logging slow requests can be resource intensive to your {es} cluster depending on the qualifying traffic's volume. For example, emitted logs might increase the index disk usage of your <> cluster. To reduce the impact of slow logs, consider the following: -[source,yaml] --------------------------------------------------- -index.indexing.slowlog.threshold.index.warn: 10s -index.indexing.slowlog.threshold.index.info: 5s -index.indexing.slowlog.threshold.index.debug: 2s -index.indexing.slowlog.threshold.index.trace: 500ms -index.indexing.slowlog.source: 1000 --------------------------------------------------- +* Enable slow logs against specific indices rather than across all indices. +* Set high thresholds to reduce the number of logged events. +* Enable slow logs only when troubleshooting. -All of the above settings are _dynamic_ and can be set for each index using the -<> API. For example: +If you aren't sure how to start investigating traffic issues, consider enabling the `warn` threshold with a high `30s` threshold at the index level using the <>: +* Enable for search requests: ++ [source,console] -------------------------------------------------- -PUT /my-index-000001/_settings +PUT /*/_settings { - "index.indexing.slowlog.threshold.index.warn": "10s", - "index.indexing.slowlog.threshold.index.info": "5s", - "index.indexing.slowlog.threshold.index.debug": "2s", - "index.indexing.slowlog.threshold.index.trace": "500ms", - "index.indexing.slowlog.source": "1000" + "index.search.slowlog.include.user": true, + "index.search.slowlog.threshold.fetch.warn": "30s", + "index.search.slowlog.threshold.query.warn": "30s" } -------------------------------------------------- // TEST[setup:my_index] -To include information about the user that triggered a slow indexing event, -use the `index.indexing.slowlog.include.user` setting. - +* Enable for indexing requests: ++ [source,console] -------------------------------------------------- -PUT /my-index-000001/_settings +PUT /*/_settings { - "index.indexing.slowlog.include.user": true + "index.indexing.slowlog.include.user": true, + "index.indexing.slowlog.threshold.index.warn": "30s" } -------------------------------------------------- // TEST[setup:my_index] -By default Elasticsearch will log the first 1000 characters of the _source in -the slowlog. You can change that with `index.indexing.slowlog.source`. Setting -it to `false` or `0` will skip logging the source entirely, while setting it to -`true` will log the entire source regardless of size. The original `_source` is -reformatted by default to make sure that it fits on a single log line. If preserving -the original document format is important, you can turn off reformatting by setting -`index.indexing.slowlog.reformat` to `false`, which will cause the source to be -logged "as is" and can potentially span multiple log lines. +Slow log thresholds being met does not guarantee cluster performance issues. In the event that symptoms are noticed, slow logs can provide helpful data to diagnose upstream traffic patterns or sources to resolve client-side issues. For example, you can use data included in `X-Opaque-ID`, the `_source` request body, or `user.*` fields to identify the source of your issue. This is similar to troubleshooting <>. + +If you're experiencing search performance issues, then you might also consider investigating searches flagged for their query durations using the <>. You can then use the profiled query to investigate optimization options using the link:{kibana-ref}/xpack-profiler.html[query profiler]. This type of investigation should usually take place in a non-production environment. -The index slow log file is configured in the `log4j2.properties` file. +Slow logging checks each event against the reporting threshold when the event is complete. This means that it can't report if events trigger <>. If suspect circuit breaker errors, then you should also consider enabling <>, which logs events before they are executed. [discrete] -=== Slow log levels +=== Learn more -You can mimic the search or indexing slow log level by setting appropriate -threshold making "more verbose" loggers to be switched off. -If for instance we want to simulate `index.indexing.slowlog.level: INFO` -then all we need to do is to set -`index.indexing.slowlog.threshold.index.debug` and `index.indexing.slowlog.threshold.index.trace` to `-1`. +To learn about other ways to optimize your search and indexing requests, refer to <> and <>. \ No newline at end of file diff --git a/docs/reference/index-modules/store.asciidoc b/docs/reference/index-modules/store.asciidoc index 9b30ba9dbde35..aba0850c76437 100644 --- a/docs/reference/index-modules/store.asciidoc +++ b/docs/reference/index-modules/store.asciidoc @@ -143,8 +143,8 @@ terms dictionaries, postings lists and points, which are the most important parts of the index for search and aggregations. For vector search, you use <>, -you might want to set the setting to vector search files: `["vec", "vex", "vem"]` -("vec" is used for vector values, "vex" – for HNSW graph, "vem" – for metadata). +you might want to set the setting to vector search files. See <> for a detailed +list of the files. Note that this setting can be dangerous on indices that are larger than the size of the main memory of the host, as it would cause the filesystem cache to be diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc index 2057519719177..f5834a725a2d7 100644 --- a/docs/reference/index.asciidoc +++ b/docs/reference/index.asciidoc @@ -6,10 +6,10 @@ include::links.asciidoc[] include::landing-page.asciidoc[] -include::intro.asciidoc[] - include::release-notes/highlights.asciidoc[] +include::intro.asciidoc[] + include::quickstart/index.asciidoc[] include::setup.asciidoc[] @@ -32,12 +32,16 @@ include::alias.asciidoc[] include::search/search-your-data/search-your-data.asciidoc[] +include::reranking/index.asciidoc[] + include::query-dsl.asciidoc[] include::aggregations.asciidoc[] include::geospatial-analysis.asciidoc[] +include::connector/docs/index.asciidoc[] + include::eql/eql.asciidoc[] include::esql/index.asciidoc[] @@ -62,14 +66,18 @@ include::security/index.asciidoc[] include::watcher/index.asciidoc[] +include::ccr/index.asciidoc[leveloffset=-1] + +include::data-store-architecture.asciidoc[] + +include::rest-api/index.asciidoc[] + include::commands/index.asciidoc[] include::how-to.asciidoc[] include::troubleshooting.asciidoc[] -include::rest-api/index.asciidoc[] - include::migration/index.asciidoc[] include::release-notes.asciidoc[] diff --git a/docs/reference/indices.asciidoc b/docs/reference/indices.asciidoc index eaef54a1effb1..ca7de396147a8 100644 --- a/docs/reference/indices.asciidoc +++ b/docs/reference/indices.asciidoc @@ -1,6 +1,12 @@ [[indices]] == Index APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Index APIs are used to manage individual indices, index settings, aliases, mappings, and index templates. diff --git a/docs/reference/indices/add-alias.asciidoc b/docs/reference/indices/add-alias.asciidoc index e14af6a64a2ec..13b49f5a47dd1 100644 --- a/docs/reference/indices/add-alias.asciidoc +++ b/docs/reference/indices/add-alias.asciidoc @@ -4,6 +4,12 @@ Create or update alias ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Adds a data stream or index to an <>. [source,console] diff --git a/docs/reference/indices/alias-exists.asciidoc b/docs/reference/indices/alias-exists.asciidoc index f820a95028a0f..f654604516676 100644 --- a/docs/reference/indices/alias-exists.asciidoc +++ b/docs/reference/indices/alias-exists.asciidoc @@ -4,6 +4,12 @@ Alias exists ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Checks if an <> exists. [source,console] diff --git a/docs/reference/indices/aliases.asciidoc b/docs/reference/indices/aliases.asciidoc index 1df9e0a4883b8..243f436f55b5a 100644 --- a/docs/reference/indices/aliases.asciidoc +++ b/docs/reference/indices/aliases.asciidoc @@ -4,6 +4,12 @@ Aliases ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Performs one or more <> actions in a single atomic operation. [source,console] diff --git a/docs/reference/indices/analyze.asciidoc b/docs/reference/indices/analyze.asciidoc index eb4d877b463cd..a211a44730a64 100644 --- a/docs/reference/indices/analyze.asciidoc +++ b/docs/reference/indices/analyze.asciidoc @@ -4,6 +4,12 @@ Analyze ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Performs <> on a text string and returns the resulting tokens. diff --git a/docs/reference/indices/apis/reload-analyzers.asciidoc b/docs/reference/indices/apis/reload-analyzers.asciidoc index ca5f540564f8e..a27a3bb859cf1 100644 --- a/docs/reference/indices/apis/reload-analyzers.asciidoc +++ b/docs/reference/indices/apis/reload-analyzers.asciidoc @@ -2,6 +2,12 @@ [[indices-reload-analyzers]] == Reload search analyzers API +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/operation/operation-indices-reload-search-analyzers[Reload search analyzers]. +-- + Reloads an index's <> and their resources. For data streams, the API reloads search analyzers and resources for the stream's backing indices. diff --git a/docs/reference/indices/apis/unfreeze.asciidoc b/docs/reference/indices/apis/unfreeze.asciidoc index 450efc03c492a..5d04d44db7443 100644 --- a/docs/reference/indices/apis/unfreeze.asciidoc +++ b/docs/reference/indices/apis/unfreeze.asciidoc @@ -17,6 +17,12 @@ You can use this API to unfreeze indices that were frozen in 7.x. Frozen indices are not related to the frozen data tier. ==== +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Unfreezes an index. [[unfreeze-index-api-request]] diff --git a/docs/reference/indices/clearcache.asciidoc b/docs/reference/indices/clearcache.asciidoc index a3150ec6f72ee..f981a6d69dd63 100644 --- a/docs/reference/indices/clearcache.asciidoc +++ b/docs/reference/indices/clearcache.asciidoc @@ -4,6 +4,12 @@ Clear cache ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Clears the caches of one or more indices. For data streams, the API clears the caches of the stream's backing indices. diff --git a/docs/reference/indices/clone-index.asciidoc b/docs/reference/indices/clone-index.asciidoc index c8e5d2e200f2e..734ff33e48215 100644 --- a/docs/reference/indices/clone-index.asciidoc +++ b/docs/reference/indices/clone-index.asciidoc @@ -4,6 +4,12 @@ Clone index ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Clones an existing index. [source,console] diff --git a/docs/reference/indices/close.asciidoc b/docs/reference/indices/close.asciidoc index a4bf1742fdea8..56d318aad2f7a 100644 --- a/docs/reference/indices/close.asciidoc +++ b/docs/reference/indices/close.asciidoc @@ -4,6 +4,12 @@ Close index ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Closes an index. [source,console] diff --git a/docs/reference/indices/create-data-stream.asciidoc b/docs/reference/indices/create-data-stream.asciidoc index e97e9973f1063..dd2d471117949 100644 --- a/docs/reference/indices/create-data-stream.asciidoc +++ b/docs/reference/indices/create-data-stream.asciidoc @@ -5,6 +5,12 @@ Create data stream ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs]. +-- + Creates a new <>. //// diff --git a/docs/reference/indices/create-index.asciidoc b/docs/reference/indices/create-index.asciidoc index 2e66f3d6030cb..a210b05b2bcf0 100644 --- a/docs/reference/indices/create-index.asciidoc +++ b/docs/reference/indices/create-index.asciidoc @@ -4,6 +4,12 @@ Create index ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Creates a new index. [source,console] diff --git a/docs/reference/indices/dangling-index-delete.asciidoc b/docs/reference/indices/dangling-index-delete.asciidoc index 6af35031e9e61..b1fd6790972c4 100644 --- a/docs/reference/indices/dangling-index-delete.asciidoc +++ b/docs/reference/indices/dangling-index-delete.asciidoc @@ -4,6 +4,12 @@ Delete dangling index ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Deletes a dangling index. [[dangling-index-delete-api-request]] diff --git a/docs/reference/indices/dangling-index-import.asciidoc b/docs/reference/indices/dangling-index-import.asciidoc index 44cde56de8c95..e266ecd2c9c70 100644 --- a/docs/reference/indices/dangling-index-import.asciidoc +++ b/docs/reference/indices/dangling-index-import.asciidoc @@ -4,6 +4,12 @@ Import dangling index ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Imports a dangling index. [[dangling-index-import-api-request]] diff --git a/docs/reference/indices/dangling-indices-list.asciidoc b/docs/reference/indices/dangling-indices-list.asciidoc index b7774843e36d9..2b7dcdc497ebf 100644 --- a/docs/reference/indices/dangling-indices-list.asciidoc +++ b/docs/reference/indices/dangling-indices-list.asciidoc @@ -4,6 +4,12 @@ List dangling indices ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Lists dangling indices. [[dangling-indices-list-api-request]] diff --git a/docs/reference/indices/data-stream-stats.asciidoc b/docs/reference/indices/data-stream-stats.asciidoc index 3ed285abc035a..cce145b4dadcc 100644 --- a/docs/reference/indices/data-stream-stats.asciidoc +++ b/docs/reference/indices/data-stream-stats.asciidoc @@ -5,6 +5,12 @@ Data stream stats ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs]. +-- + Retrieves statistics for one or more <>. //// diff --git a/docs/reference/indices/delete-alias.asciidoc b/docs/reference/indices/delete-alias.asciidoc index 748862df06100..c204c89579352 100644 --- a/docs/reference/indices/delete-alias.asciidoc +++ b/docs/reference/indices/delete-alias.asciidoc @@ -4,6 +4,12 @@ Delete alias ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Removes a data stream or index from an <>. [source,console] diff --git a/docs/reference/indices/delete-component-template.asciidoc b/docs/reference/indices/delete-component-template.asciidoc index 065a4adb90023..27c1e00a5d57a 100644 --- a/docs/reference/indices/delete-component-template.asciidoc +++ b/docs/reference/indices/delete-component-template.asciidoc @@ -4,6 +4,12 @@ Delete component template ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Deletes an existing component template. //// diff --git a/docs/reference/indices/delete-data-stream.asciidoc b/docs/reference/indices/delete-data-stream.asciidoc index 38e7a00d451d8..1cb43a615ede8 100644 --- a/docs/reference/indices/delete-data-stream.asciidoc +++ b/docs/reference/indices/delete-data-stream.asciidoc @@ -5,6 +5,12 @@ Delete data stream ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs]. +-- + Deletes one or more <> and their backing indices. See <>. diff --git a/docs/reference/indices/delete-index-template-v1.asciidoc b/docs/reference/indices/delete-index-template-v1.asciidoc index 98b1e2fb255f1..e035c13a7bceb 100644 --- a/docs/reference/indices/delete-index-template-v1.asciidoc +++ b/docs/reference/indices/delete-index-template-v1.asciidoc @@ -9,6 +9,12 @@ templates>>, which are deprecated and will be replaced by the composable templates introduced in {es} 7.8. For information about composable templates, see <>. +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Deletes a legacy index template. //// diff --git a/docs/reference/indices/delete-index-template.asciidoc b/docs/reference/indices/delete-index-template.asciidoc index b828e4a536b71..23713f62fa031 100644 --- a/docs/reference/indices/delete-index-template.asciidoc +++ b/docs/reference/indices/delete-index-template.asciidoc @@ -4,6 +4,12 @@ Delete index template ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Deletes an <>. //// diff --git a/docs/reference/indices/delete-index.asciidoc b/docs/reference/indices/delete-index.asciidoc index d5d168154e44a..81ee7b502d377 100644 --- a/docs/reference/indices/delete-index.asciidoc +++ b/docs/reference/indices/delete-index.asciidoc @@ -4,6 +4,12 @@ Delete index ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Deletes one or more indices. [source,console] diff --git a/docs/reference/indices/diskusage.asciidoc b/docs/reference/indices/diskusage.asciidoc index 3510ba346e5a7..6db122ab7d836 100644 --- a/docs/reference/indices/diskusage.asciidoc +++ b/docs/reference/indices/diskusage.asciidoc @@ -6,6 +6,12 @@ experimental[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Analyzes the disk usage of each field of an index or data stream. This API might not support indices created in previous {es} versions. The result of a small index can be inaccurate as some parts of an index diff --git a/docs/reference/indices/downsample-data-stream.asciidoc b/docs/reference/indices/downsample-data-stream.asciidoc index 5ace4e03dfb66..a99d7b246ba43 100644 --- a/docs/reference/indices/downsample-data-stream.asciidoc +++ b/docs/reference/indices/downsample-data-stream.asciidoc @@ -5,6 +5,12 @@ Downsample ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs]. +-- + Aggregates a time series (TSDS) index and stores pre-computed statistical summaries (`min`, `max`, `sum`, `value_count` and `avg`) for each metric field grouped by a configured time interval. For example, diff --git a/docs/reference/indices/field-usage-stats.asciidoc b/docs/reference/indices/field-usage-stats.asciidoc index a4856092834e5..cbb65f8c2effb 100644 --- a/docs/reference/indices/field-usage-stats.asciidoc +++ b/docs/reference/indices/field-usage-stats.asciidoc @@ -6,6 +6,12 @@ experimental[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Returns field usage information for each shard and field of an index. Field usage statistics are automatically captured when diff --git a/docs/reference/indices/flush.asciidoc b/docs/reference/indices/flush.asciidoc index 61c44f157da95..a458e6906b62f 100644 --- a/docs/reference/indices/flush.asciidoc +++ b/docs/reference/indices/flush.asciidoc @@ -4,6 +4,12 @@ Flush ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Flushes one or more data streams or indices. [source,console] diff --git a/docs/reference/indices/forcemerge.asciidoc b/docs/reference/indices/forcemerge.asciidoc index 6eacaac5e7b2a..4bac16f1aa659 100644 --- a/docs/reference/indices/forcemerge.asciidoc +++ b/docs/reference/indices/forcemerge.asciidoc @@ -4,6 +4,12 @@ Force merge ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Forces a <> on the shards of one or more indices. For data streams, the API forces a merge on the shards of the stream's backing indices. diff --git a/docs/reference/indices/get-alias.asciidoc b/docs/reference/indices/get-alias.asciidoc index 743aaf7aee174..154aea0ac4890 100644 --- a/docs/reference/indices/get-alias.asciidoc +++ b/docs/reference/indices/get-alias.asciidoc @@ -4,6 +4,12 @@ Get alias ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Retrieves information for one or more <>. [source,console] diff --git a/docs/reference/indices/get-component-template.asciidoc b/docs/reference/indices/get-component-template.asciidoc index f35192ca448db..e7ef16de1b749 100644 --- a/docs/reference/indices/get-component-template.asciidoc +++ b/docs/reference/indices/get-component-template.asciidoc @@ -4,6 +4,12 @@ Get component template ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Retrieves information about one or more component templates. ////////////////////////// diff --git a/docs/reference/indices/get-data-stream.asciidoc b/docs/reference/indices/get-data-stream.asciidoc index 6bf150897acab..ccab53f020e5f 100644 --- a/docs/reference/indices/get-data-stream.asciidoc +++ b/docs/reference/indices/get-data-stream.asciidoc @@ -5,6 +5,12 @@ Get data stream ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs]. +-- + Retrieves information about one or more <>. See <>. diff --git a/docs/reference/indices/get-field-mapping.asciidoc b/docs/reference/indices/get-field-mapping.asciidoc index ac5895872fbb5..4b25e0e925541 100644 --- a/docs/reference/indices/get-field-mapping.asciidoc +++ b/docs/reference/indices/get-field-mapping.asciidoc @@ -4,6 +4,12 @@ Get field mapping ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Retrieves <> for one or more fields. For data streams, the API retrieves field mappings for the stream's backing indices. diff --git a/docs/reference/indices/get-index-template-v1.asciidoc b/docs/reference/indices/get-index-template-v1.asciidoc index 602ca2fe454ad..5072a194319ef 100644 --- a/docs/reference/indices/get-index-template-v1.asciidoc +++ b/docs/reference/indices/get-index-template-v1.asciidoc @@ -8,6 +8,12 @@ IMPORTANT: This documentation is about legacy index templates, which are deprecated and will be replaced by the composable templates introduced in {es} 7.8. For information about composable templates, see <>. +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Retrieves information about one or more index templates. //// diff --git a/docs/reference/indices/get-index-template.asciidoc b/docs/reference/indices/get-index-template.asciidoc index 2cde5adc8ae23..9d27bf5b2b2a4 100644 --- a/docs/reference/indices/get-index-template.asciidoc +++ b/docs/reference/indices/get-index-template.asciidoc @@ -4,6 +4,12 @@ Get index template ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Returns information about one or more index templates. //// diff --git a/docs/reference/indices/get-index.asciidoc b/docs/reference/indices/get-index.asciidoc index 2551d25801d70..4e026f4ff07a3 100644 --- a/docs/reference/indices/get-index.asciidoc +++ b/docs/reference/indices/get-index.asciidoc @@ -4,6 +4,12 @@ Get index ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Returns information about one or more indices. For data streams, the API returns information about the stream's backing indices. diff --git a/docs/reference/indices/get-mapping.asciidoc b/docs/reference/indices/get-mapping.asciidoc index 16dc8c66d0715..543bda6cb3242 100644 --- a/docs/reference/indices/get-mapping.asciidoc +++ b/docs/reference/indices/get-mapping.asciidoc @@ -4,6 +4,12 @@ Get mapping ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Retrieves <> for one or more indices. For data streams, the API retrieves mappings for the stream's backing indices. diff --git a/docs/reference/indices/get-settings.asciidoc b/docs/reference/indices/get-settings.asciidoc index b6cb6d2926387..d48514b9a3493 100644 --- a/docs/reference/indices/get-settings.asciidoc +++ b/docs/reference/indices/get-settings.asciidoc @@ -4,6 +4,12 @@ Get index settings ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Returns setting information for one or more indices. For data streams, the API returns setting information for the stream's backing indices. diff --git a/docs/reference/indices/index-mgmt.asciidoc b/docs/reference/indices/index-mgmt.asciidoc index 7a78f9452b85e..73643dbfd4b3b 100644 --- a/docs/reference/indices/index-mgmt.asciidoc +++ b/docs/reference/indices/index-mgmt.asciidoc @@ -67,7 +67,7 @@ This value is the time period for which your data is guaranteed to be stored. Da Elasticsearch at a later time. [role="screenshot"] -image::images/index-mgmt/management-data-stream.png[Data stream details] +image::images/index-mgmt/management-data-stream-fields.png[Data stream details] * To view more information about a data stream, such as its generation or its current index lifecycle policy, click the stream's name. From this view, you can navigate to *Discover* to diff --git a/docs/reference/indices/index-template-exists-v1.asciidoc b/docs/reference/indices/index-template-exists-v1.asciidoc index 2358f0b1a376e..f7b3a4416715d 100644 --- a/docs/reference/indices/index-template-exists-v1.asciidoc +++ b/docs/reference/indices/index-template-exists-v1.asciidoc @@ -9,6 +9,12 @@ templates>>, which are deprecated and will be replaced by the composable templates introduced in {es} 7.8. For information about composable templates, see <>. +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Checks if an <> exists. diff --git a/docs/reference/indices/index-templates.asciidoc b/docs/reference/indices/index-templates.asciidoc index 66911716ffee2..874263505fa4b 100644 --- a/docs/reference/indices/index-templates.asciidoc +++ b/docs/reference/indices/index-templates.asciidoc @@ -60,7 +60,7 @@ applying the templates, do one or more of the following: - Use a non-overlapping index pattern. -- Assign templates with an overlapping pattern a `priority` higher than `200`. +- Assign templates with an overlapping pattern a `priority` higher than `500`. For example, if you don't use {fleet} or {agent} and want to create a template for the `logs-*` index pattern, assign your template a priority of `500`. This ensures your template is applied instead of the built-in template for @@ -134,8 +134,7 @@ PUT _index_template/template_1 "type": "keyword" }, "created_at": { - "type": "date", - "format": "EEE MMM dd HH:mm:ss Z yyyy" + "type": "date" } } }, diff --git a/docs/reference/indices/indices-exists.asciidoc b/docs/reference/indices/indices-exists.asciidoc index d699d36add03b..11da491adf8e5 100644 --- a/docs/reference/indices/indices-exists.asciidoc +++ b/docs/reference/indices/indices-exists.asciidoc @@ -4,6 +4,12 @@ Exists ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Checks if a data stream, index, or alias exists. [source,console] diff --git a/docs/reference/indices/migrate-to-data-stream.asciidoc b/docs/reference/indices/migrate-to-data-stream.asciidoc index 48302761c7dc9..745e3e28683c4 100644 --- a/docs/reference/indices/migrate-to-data-stream.asciidoc +++ b/docs/reference/indices/migrate-to-data-stream.asciidoc @@ -5,6 +5,12 @@ Migrate to data stream ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs]. +-- + Converts an <> to a <>. //// diff --git a/docs/reference/indices/open-close.asciidoc b/docs/reference/indices/open-close.asciidoc index a077c4d19fd56..fe00e5a968d96 100644 --- a/docs/reference/indices/open-close.asciidoc +++ b/docs/reference/indices/open-close.asciidoc @@ -4,6 +4,12 @@ Open index ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Opens a closed index. For data streams, the API opens any closed backing indices. diff --git a/docs/reference/indices/put-component-template.asciidoc b/docs/reference/indices/put-component-template.asciidoc index 0a0e36b63e6cd..413548364d310 100644 --- a/docs/reference/indices/put-component-template.asciidoc +++ b/docs/reference/indices/put-component-template.asciidoc @@ -4,6 +4,12 @@ Create or update component template ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Creates or updates a component template. Component templates are building blocks for constructing <> that specify index <>, <>, and @@ -26,8 +32,7 @@ PUT _component_template/template_1 "type": "keyword" }, "created_at": { - "type": "date", - "format": "EEE MMM dd HH:mm:ss Z yyyy" + "type": "date" } } } @@ -129,6 +134,8 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [[put-component-template-api-request-body]] ==== {api-request-body-title} +// tag::template[] + `template`:: (Required, object) This is the template to be applied, may optionally include a `mappings`, @@ -136,7 +143,7 @@ This is the template to be applied, may optionally include a `mappings`, + .Properties of `template` [%collapsible%open] -==== +===== `aliases`:: (Optional, object of objects) Aliases to add. + @@ -147,7 +154,7 @@ include::{es-ref-dir}/indices/create-index.asciidoc[tag=aliases-props] include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=mappings] include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=settings] -==== +===== `version`:: (Optional, integer) @@ -174,6 +181,7 @@ This map is not automatically generated by {es}. Marks this component template as deprecated. When a deprecated component template is referenced when creating or updating a non-deprecated index template, {es} will emit a deprecation warning. +end::template[] [[put-component-template-api-example]] ==== {api-examples-title} diff --git a/docs/reference/indices/put-index-template-v1.asciidoc b/docs/reference/indices/put-index-template-v1.asciidoc index 86a8a54edd97f..4e109551b7a93 100644 --- a/docs/reference/indices/put-index-template-v1.asciidoc +++ b/docs/reference/indices/put-index-template-v1.asciidoc @@ -4,10 +4,16 @@ Create or update index template (legacy) ++++ -IMPORTANT: This documentation is about legacy index templates, -which are deprecated and will be replaced by the composable templates introduced in {es} 7.8. +IMPORTANT: This documentation is about legacy index templates, +which are deprecated and will be replaced by the composable templates introduced in {es} 7.8. For information about composable templates, see <>. +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Creates or updates an index template. [source,console] @@ -27,8 +33,7 @@ PUT _template/template_1 "type": "keyword" }, "created_at": { - "type": "date", - "format": "EEE MMM dd HH:mm:ss Z yyyy" + "type": "date" } } } @@ -67,8 +72,8 @@ that you can automatically apply when creating new indices. {es} applies templates to new indices based on an index pattern that matches the index name. -NOTE: Composable templates always take precedence over legacy templates. -If no composable template matches a new index, +NOTE: Composable templates always take precedence over legacy templates. +If no composable template matches a new index, matching legacy templates are applied according to their order. Index templates are only applied during index creation. @@ -263,4 +268,4 @@ The API returns the following response: "version" : 123 } } --------------------------------------------------- \ No newline at end of file +-------------------------------------------------- diff --git a/docs/reference/indices/put-index-template.asciidoc b/docs/reference/indices/put-index-template.asciidoc index 772bd51afdce8..fac928c4a61b0 100644 --- a/docs/reference/indices/put-index-template.asciidoc +++ b/docs/reference/indices/put-index-template.asciidoc @@ -4,6 +4,12 @@ Create or update index template ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Creates or updates an index template. Index templates define <>, <>, and <> that can be applied automatically to new indices. @@ -85,6 +91,8 @@ include::{docdir}/rest-api/common-parms.asciidoc[tag=master-timeout] [[put-index-template-api-request-body]] ==== {api-request-body-title} +// tag::request-body[] + `composed_of`:: (Optional, array of strings) An ordered list of component template names. Component templates are merged in the order @@ -102,7 +110,7 @@ See <>. + .Properties of `data_stream` [%collapsible%open] -==== +===== `allow_custom_routing`:: (Optional, Boolean) If `true`, the data stream supports <>. Defaults to `false`. @@ -113,11 +121,11 @@ See <>. `index_mode`:: (Optional, string) Type of data stream to create. Valid values are `null` -(regular data stream) and `time_series` (<>). +(standard data stream), `time_series` (<>) and `logsdb` +(<>). + -If `time_series`, each backing index has an `index.mode` index setting of -`time_series`. -==== +The template's `index_mode` sets the `index.mode` of the backing index. +===== `index_patterns`:: (Required, array of strings) @@ -146,7 +154,7 @@ Template to be applied. It may optionally include an `aliases`, `mappings`, or + .Properties of `template` [%collapsible%open] -==== +===== `aliases`:: (Optional, object of objects) Aliases to add. + @@ -161,7 +169,7 @@ include::{es-ref-dir}/indices/create-index.asciidoc[tag=aliases-props] include::{docdir}/rest-api/common-parms.asciidoc[tag=mappings] include::{docdir}/rest-api/common-parms.asciidoc[tag=settings] -==== +===== `version`:: (Optional, integer) @@ -174,6 +182,7 @@ Marks this index template as deprecated. When creating or updating a non-deprecated index template that uses deprecated components, {es} will emit a deprecation warning. // end::index-template-api-body[] +// end::request-body[] [[put-index-template-api-example]] ==== {api-examples-title} diff --git a/docs/reference/indices/put-mapping.asciidoc b/docs/reference/indices/put-mapping.asciidoc index dc6dbff1df42c..479bdff22a80c 100644 --- a/docs/reference/indices/put-mapping.asciidoc +++ b/docs/reference/indices/put-mapping.asciidoc @@ -4,6 +4,12 @@ Update mapping ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Adds new fields to an existing data stream or index. You can also use this API to change the search settings of existing fields. diff --git a/docs/reference/indices/recovery.asciidoc b/docs/reference/indices/recovery.asciidoc index 06b4d9d92e49f..032ffc5c5a082 100644 --- a/docs/reference/indices/recovery.asciidoc +++ b/docs/reference/indices/recovery.asciidoc @@ -4,6 +4,12 @@ Index recovery ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Returns information about ongoing and completed shard recoveries for one or more indices. For data streams, the API returns information for the stream's backing indices. diff --git a/docs/reference/indices/refresh.asciidoc b/docs/reference/indices/refresh.asciidoc index bd8e821ff56b9..6cc78b22323c1 100644 --- a/docs/reference/indices/refresh.asciidoc +++ b/docs/reference/indices/refresh.asciidoc @@ -4,6 +4,12 @@ Refresh ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + A refresh makes recent operations performed on one or more indices available for search. For data streams, the API runs the refresh operation on the stream's backing indices. For more information about the refresh operation, see diff --git a/docs/reference/indices/resolve-cluster.asciidoc b/docs/reference/indices/resolve-cluster.asciidoc index 48e6bfac4af10..e623cb983e86b 100644 --- a/docs/reference/indices/resolve-cluster.asciidoc +++ b/docs/reference/indices/resolve-cluster.asciidoc @@ -4,6 +4,12 @@ Resolve cluster ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Resolves the specified index expressions to return information about each cluster, including the local cluster, if included. diff --git a/docs/reference/indices/resolve.asciidoc b/docs/reference/indices/resolve.asciidoc index 856546b037fea..f0b91695b0aa1 100644 --- a/docs/reference/indices/resolve.asciidoc +++ b/docs/reference/indices/resolve.asciidoc @@ -4,6 +4,12 @@ Resolve index ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Resolves the specified name(s) and/or index patterns for indices, aliases, and data streams. Multiple patterns and remote clusters are supported. diff --git a/docs/reference/indices/rollover-index.asciidoc b/docs/reference/indices/rollover-index.asciidoc index 2a47d28e5358d..cc93204469b80 100644 --- a/docs/reference/indices/rollover-index.asciidoc +++ b/docs/reference/indices/rollover-index.asciidoc @@ -4,6 +4,12 @@ Rollover ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Creates a new index for a <> or <>. [source,console] diff --git a/docs/reference/indices/segments.asciidoc b/docs/reference/indices/segments.asciidoc index 54f0b6c0cd6de..b2b7b244321f4 100644 --- a/docs/reference/indices/segments.asciidoc +++ b/docs/reference/indices/segments.asciidoc @@ -4,6 +4,12 @@ Index segments ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Returns low-level information about the https://lucene.apache.org/core/[Lucene] segments in index shards. For data streams, the API returns information about the stream's backing indices. diff --git a/docs/reference/indices/shard-stores.asciidoc b/docs/reference/indices/shard-stores.asciidoc index 1b001a3175b8c..f9a6fb0905f15 100644 --- a/docs/reference/indices/shard-stores.asciidoc +++ b/docs/reference/indices/shard-stores.asciidoc @@ -4,6 +4,12 @@ Index shard stores ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Retrieves store information about replica shards in one or more indices. For data streams, the API retrieves store @@ -193,6 +199,7 @@ The API returns the following response: // TESTRESPONSE[s/"roles": \[[^]]*\]/"roles": $body.$_path/] // TESTRESPONSE[s/"8.10.0"/\$node_version/] // TESTRESPONSE[s/"min_index_version": 7000099/"min_index_version": $body.$_path/] +// TESTRESPONSE[s/"min_index_version": 7000099/"min_index_version": $body.$_path/] // TESTRESPONSE[s/"max_index_version": 8100099/"max_index_version": $body.$_path/] diff --git a/docs/reference/indices/shrink-index.asciidoc b/docs/reference/indices/shrink-index.asciidoc index 244733282d46e..931731fceb49d 100644 --- a/docs/reference/indices/shrink-index.asciidoc +++ b/docs/reference/indices/shrink-index.asciidoc @@ -4,6 +4,12 @@ Shrink index ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Shrinks an existing index into a new index with fewer primary shards. diff --git a/docs/reference/indices/simulate-index.asciidoc b/docs/reference/indices/simulate-index.asciidoc index 5e5709a2d82fc..414b54aa5f414 100644 --- a/docs/reference/indices/simulate-index.asciidoc +++ b/docs/reference/indices/simulate-index.asciidoc @@ -4,6 +4,12 @@ Simulate index ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Returns the index configuration that would be applied to the specified index from an existing <>. diff --git a/docs/reference/indices/simulate-template.asciidoc b/docs/reference/indices/simulate-template.asciidoc index c7397ace97886..298756318f6fe 100644 --- a/docs/reference/indices/simulate-template.asciidoc +++ b/docs/reference/indices/simulate-template.asciidoc @@ -4,6 +4,12 @@ Simulate template ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Returns the index configuration that would be applied by a particular <>. @@ -26,8 +32,7 @@ PUT _index_template/template_1 "type": "keyword" }, "created_at": { - "type": "date", - "format": "EEE MMM dd HH:mm:ss Z yyyy" + "type": "date" } } }, @@ -74,19 +79,19 @@ privilege>> to use this API. ``:: (Optional, string) Name of the index template to simulate. -To test a template configuration before you add it to the cluster, +To test a template configuration before you add it to the cluster, omit this parameter and specify the template configuration in the request body. [[simulate-template-api-query-params]] ==== {api-query-parms-title} //// `cause`:: -(Optional, string) The reason for using the specified template for the simulation. +(Optional, string) The reason for using the specified template for the simulation. //// `create`:: -(Optional, Boolean) If `true`, the template passed in the body is -only used if no existing templates match the same index patterns. +(Optional, Boolean) If `true`, the template passed in the body is +only used if no existing templates match the same index patterns. If `false`, the simulation uses the template with the highest priority. Note that the template is not permanently added or updated in either case; it is only used for the simulation. @@ -114,7 +119,7 @@ include::{es-ref-dir}/indices/put-index-template.asciidoc[tag=index-template-api .Properties of `overlapping` [%collapsible%open] ==== -`index_patterns`:: +`index_patterns`:: (array) Index patterns that the superseded template applies to. `name`:: @@ -123,7 +128,7 @@ include::{es-ref-dir}/indices/put-index-template.asciidoc[tag=index-template-api `template`:: (object) -The settings, mappings, and aliases that would be applied to matching indices. +The settings, mappings, and aliases that would be applied to matching indices. + .Properties of `template` [%collapsible%open] @@ -227,8 +232,8 @@ The response shows the index settings, mappings, and aliases applied by the `fin [[simulate-template-config-ex]] ===== Simulating an arbitrary template configuration -To see what settings will be applied by a template before you add it to the cluster, -you can pass a template configuration in the request body. +To see what settings will be applied by a template before you add it to the cluster, +you can pass a template configuration in the request body. The specified template is used for the simulation if it has a higher priority than existing templates. [source,console] @@ -266,8 +271,8 @@ The response shows any overlapping templates with a lower priority. } }, "mappings" : { - "properties" : { - "@timestamp" : { + "properties" : { + "@timestamp" : { "type" : "date" } } diff --git a/docs/reference/indices/split-index.asciidoc b/docs/reference/indices/split-index.asciidoc index 0c93b572639db..4bda778839735 100644 --- a/docs/reference/indices/split-index.asciidoc +++ b/docs/reference/indices/split-index.asciidoc @@ -4,6 +4,12 @@ Split index ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Splits an existing index into a new index with more primary shards. [source,console] diff --git a/docs/reference/indices/stats.asciidoc b/docs/reference/indices/stats.asciidoc index 088d65c37ec6e..1370935a7cde7 100644 --- a/docs/reference/indices/stats.asciidoc +++ b/docs/reference/indices/stats.asciidoc @@ -4,6 +4,12 @@ Index stats ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Returns statistics for one or more indices. For data streams, the API retrieves statistics for the stream's backing indices. diff --git a/docs/reference/indices/update-settings.asciidoc b/docs/reference/indices/update-settings.asciidoc index 3b29946d5ed7d..dd92a922ec655 100644 --- a/docs/reference/indices/update-settings.asciidoc +++ b/docs/reference/indices/update-settings.asciidoc @@ -4,6 +4,12 @@ Update index settings ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. +-- + Changes a <> in real time. For data streams, index setting changes are applied to all backing indices by diff --git a/docs/reference/inference/delete-inference.asciidoc b/docs/reference/inference/delete-inference.asciidoc index 4df72ba672092..a9dfeabcdf259 100644 --- a/docs/reference/inference/delete-inference.asciidoc +++ b/docs/reference/inference/delete-inference.asciidoc @@ -2,16 +2,17 @@ [[delete-inference-api]] === Delete {infer} API -experimental[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs]. +-- Deletes an {infer} endpoint. -IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in -{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI or -Hugging Face. For built-in models and models uploaded through Eland, the {infer} -APIs offer an alternative way to use and manage trained models. However, if you -do not plan to use the {infer} APIs to use these models or if you want to use -non-NLP models, use the <>. +IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face. +For built-in models and models uploaded through Eland, the {infer} APIs offer an alternative way to use and manage trained models. +However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the <>. [discrete] @@ -49,13 +50,12 @@ The type of {infer} task that the model performs. `dry_run`:: (Optional, Boolean) -When `true`, checks the {infer} processors that reference the endpoint and -returns them in a list, but does not delete the endpoint. Defaults to `false`. +When `true`, checks the `semantic_text` fields and {infer} processors that reference the endpoint and returns them in a list, but does not delete the endpoint. +Defaults to `false`. `force`:: (Optional, Boolean) -Deletes the endpoint regardless if it's used in an {infer} pipeline or in a -`semantic_text` field. +Deletes the endpoint regardless if it's used in a `semantic_text` field or in an {infer} pipeline. [discrete] diff --git a/docs/reference/inference/get-inference.asciidoc b/docs/reference/inference/get-inference.asciidoc index c3fe841603bcc..74220514eeb98 100644 --- a/docs/reference/inference/get-inference.asciidoc +++ b/docs/reference/inference/get-inference.asciidoc @@ -2,16 +2,17 @@ [[get-inference-api]] === Get {infer} API -experimental[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs]. +-- Retrieves {infer} endpoint information. -IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in -{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI or -Hugging Face. For built-in models and models uploaded through Eland, the {infer} -APIs offer an alternative way to use and manage trained models. However, if you -do not plan to use the {infer} APIs to use these models or if you want to use -non-NLP models, use the <>. +IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face. +For built-in models and models uploaded through Eland, the {infer} APIs offer an alternative way to use and manage trained models. +However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the <>. [discrete] @@ -83,12 +84,17 @@ The API returns the following response: { "inference_id": "my-elser-model", "task_type": "sparse_embedding", - "service": "elser", + "service": "elasticsearch", "service_settings": { "num_allocations": 1, - "num_threads": 1 + "num_threads": 1, + "model_id": ".elser_model_2" }, - "task_settings": {} + "chunking_settings": { + "strategy": "sentence", + "max_chunk_size": 250, + "sentence_overlap": 1 + } } ------------------------------------------------------------ // NOTCONSOLE diff --git a/docs/reference/inference/images/inference-landscape.jpg b/docs/reference/inference/images/inference-landscape.jpg new file mode 100644 index 0000000000000..d66d67763cab5 Binary files /dev/null and b/docs/reference/inference/images/inference-landscape.jpg differ diff --git a/docs/reference/inference/images/inference-landscape.png b/docs/reference/inference/images/inference-landscape.png deleted file mode 100644 index a35d1370fd09b..0000000000000 Binary files a/docs/reference/inference/images/inference-landscape.png and /dev/null differ diff --git a/docs/reference/inference/inference-apis.asciidoc b/docs/reference/inference/inference-apis.asciidoc index 8fdf8aecc2ae5..ca273afc478ea 100644 --- a/docs/reference/inference/inference-apis.asciidoc +++ b/docs/reference/inference/inference-apis.asciidoc @@ -2,8 +2,6 @@ [[inference-apis]] == {infer-cap} APIs -experimental[] - IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio or Hugging Face. For built-in models and models uploaded @@ -12,6 +10,12 @@ trained models. However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the <>. +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs]. +-- + The {infer} APIs enable you to create {infer} endpoints and use {ml} models of different providers - such as Amazon Bedrock, Anthropic, Azure AI Studio, Cohere, Google AI, Mistral, OpenAI, or HuggingFace - as a service. Use @@ -21,10 +25,12 @@ the following APIs to manage {infer} models and perform {infer}: * <> * <> * <> +* <> +* <> [[inference-landscape]] .A representation of the Elastic inference landscape -image::images/inference-landscape.png[A representation of the Elastic inference landscape,align="center"] +image::images/inference-landscape.jpg[A representation of the Elastic inference landscape,align="center"] An {infer} endpoint enables you to use the corresponding {ml} model without manual deployment and apply it to your data at ingestion time through @@ -35,10 +41,103 @@ Elastic –, then create an {infer} endpoint by the <>. Now use <> to perform <> on your data. +[discrete] +[[adaptive-allocations]] +=== Adaptive allocations + +Adaptive allocations allow inference services to dynamically adjust the number of model allocations based on the current load. + +When adaptive allocations are enabled: + +* The number of allocations scales up automatically when the load increases. +- Allocations scale down to a minimum of 0 when the load decreases, saving resources. + +For more information about adaptive allocations and resources, refer to the {ml-docs}/ml-nlp-auto-scale.html[trained model autoscaling] documentation. + +[discrete] +[[default-enpoints]] +=== Default {infer} endpoints + +Your {es} deployment contains preconfigured {infer} endpoints which makes them easier to use when defining `semantic_text` fields or using {infer} processors. +The following list contains the default {infer} endpoints listed by `inference_id`: + +* `.elser-2-elasticsearch`: uses the {ml-docs}/ml-nlp-elser.html[ELSER] built-in trained model for `sparse_embedding` tasks (recommended for English language texts) +* `.multilingual-e5-small-elasticsearch`: uses the {ml-docs}/ml-nlp-e5.html[E5] built-in trained model for `text_embedding` tasks (recommended for non-English language texts) + +Use the `inference_id` of the endpoint in a <> field definition or when creating an <>. +The API call will automatically download and deploy the model which might take a couple of minutes. +Default {infer} enpoints have {ml-docs}/ml-nlp-auto-scale.html#nlp-model-adaptive-allocations[adaptive allocations] enabled. +For these models, the minimum number of allocations is `0`. +If there is no {infer} activity that uses the endpoint, the number of allocations will scale down to `0` automatically after 15 minutes. + + +[discrete] +[[infer-chunking-config]] +=== Configuring chunking + +{infer-cap} endpoints have a limit on the amount of text they can process at once, determined by the model's input capacity. +Chunking is the process of splitting the input text into pieces that remain within these limits. +It occurs when ingesting documents into <>. +Chunking also helps produce sections that are digestible for humans. +Returning a long document in search results is less useful than providing the most relevant chunk of text. + +Each chunk will include the text subpassage and the corresponding embedding generated from it. + +By default, documents are split into sentences and grouped in sections up to 250 words with 1 sentence overlap so that each chunk shares a sentence with the previous chunk. +Overlapping ensures continuity and prevents vital contextual information in the input text from being lost by a hard break. + +{es} uses the https://unicode-org.github.io/icu-docs/[ICU4J] library to detect word and sentence boundaries for chunking. +https://unicode-org.github.io/icu/userguide/boundaryanalysis/#word-boundary[Word boundaries] are identified by following a series of rules, not just the presence of a whitespace character. +For written languages that do use whitespace such as Chinese or Japanese dictionary lookups are used to detect word boundaries. + + +[discrete] +==== Chunking strategies + +Two strategies are available for chunking: `sentence` and `word`. + +The `sentence` strategy splits the input text at sentence boundaries. +Each chunk contains one or more complete sentences ensuring that the integrity of sentence-level context is preserved, except when a sentence causes a chunk to exceed a word count of `max_chunk_size`, in which case it will be split across chunks. +The `sentence_overlap` option defines the number of sentences from the previous chunk to include in the current chunk which is either `0` or `1`. + +The `word` strategy splits the input text on individual words up to the `max_chunk_size` limit. +The `overlap` option is the number of words from the previous chunk to include in the current chunk. + +The default chunking strategy is `sentence`. + +NOTE: The default chunking strategy for {infer} endpoints created before 8.16 is `word`. + + +[discrete] +==== Example of configuring the chunking behavior + +The following example creates an {infer} endpoint with the `elasticsearch` service that deploys the ELSER model by default and configures the chunking behavior. + +[source,console] +------------------------------------------------------------ +PUT _inference/sparse_embedding/small_chunk_size +{ + "service": "elasticsearch", + "service_settings": { + "num_allocations": 1, + "num_threads": 1 + }, + "chunking_settings": { + "strategy": "sentence", + "max_chunk_size": 100, + "sentence_overlap": 0 + } +} +------------------------------------------------------------ +// TEST[skip:TBD] + + include::delete-inference.asciidoc[] include::get-inference.asciidoc[] include::post-inference.asciidoc[] include::put-inference.asciidoc[] +include::stream-inference.asciidoc[] +include::update-inference.asciidoc[] include::service-alibabacloud-ai-search.asciidoc[] include::service-amazon-bedrock.asciidoc[] include::service-anthropic.asciidoc[] @@ -50,5 +149,7 @@ include::service-elser.asciidoc[] include::service-google-ai-studio.asciidoc[] include::service-google-vertex-ai.asciidoc[] include::service-hugging-face.asciidoc[] +include::service-jinaai.asciidoc[] include::service-mistral.asciidoc[] include::service-openai.asciidoc[] +include::service-watsonx-ai.asciidoc[] diff --git a/docs/reference/inference/inference-shared.asciidoc b/docs/reference/inference/inference-shared.asciidoc index 2eafa3434e89e..da497c6581e5d 100644 --- a/docs/reference/inference/inference-shared.asciidoc +++ b/docs/reference/inference/inference-shared.asciidoc @@ -31,4 +31,36 @@ end::task-settings[] tag::task-type[] The type of the {infer} task that the model will perform. -end::task-type[] \ No newline at end of file +end::task-type[] + +tag::chunking-settings[] +Chunking configuration object. +Refer to <> to learn more about chunking. +end::chunking-settings[] + +tag::chunking-settings-max-chunking-size[] +Specifies the maximum size of a chunk in words. +Defaults to `250`. +This value cannot be higher than `300` or lower than `20` (for `sentence` strategy) or `10` (for `word` strategy). +end::chunking-settings-max-chunking-size[] + +tag::chunking-settings-overlap[] +Only for `word` chunking strategy. +Specifies the number of overlapping words for chunks. +Defaults to `100`. +This value cannot be higher than the half of `max_chunking_size`. +end::chunking-settings-overlap[] + +tag::chunking-settings-sentence-overlap[] +Only for `sentence` chunking strategy. +Specifies the numnber of overlapping sentences for chunks. +It can be either `1` or `0`. +Defaults to `1`. +end::chunking-settings-sentence-overlap[] + +tag::chunking-settings-strategy[] +Specifies the chunking strategy. +It could be either `sentence` or `word`. +end::chunking-settings-strategy[] + + diff --git a/docs/reference/inference/post-inference.asciidoc b/docs/reference/inference/post-inference.asciidoc index 52131c0b10776..07e557aa355bd 100644 --- a/docs/reference/inference/post-inference.asciidoc +++ b/docs/reference/inference/post-inference.asciidoc @@ -2,16 +2,17 @@ [[post-inference-api]] === Perform inference API -experimental[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs]. +-- Performs an inference task on an input text by using an {infer} endpoint. -IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in -{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI or -Hugging Face. For built-in models and models uploaded through Eland, the {infer} -APIs offer an alternative way to use and manage trained models. However, if you -do not plan to use the {infer} APIs to use these models or if you want to use -non-NLP models, use the <>. +IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face. +For built-in models and models uploaded through Eland, the {infer} APIs offer an alternative way to use and manage trained models. +However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the <>. [discrete] diff --git a/docs/reference/inference/put-inference.asciidoc b/docs/reference/inference/put-inference.asciidoc index b106e2c4a08fc..c203b610169e6 100644 --- a/docs/reference/inference/put-inference.asciidoc +++ b/docs/reference/inference/put-inference.asciidoc @@ -2,22 +2,20 @@ [[put-inference-api]] === Create {infer} API -experimental[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs]. +-- Creates an {infer} endpoint to perform an {infer} task. [IMPORTANT] ==== -* The {infer} APIs enable you to use certain services, such as built-in -{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Mistral, -Azure OpenAI, Google AI Studio, Google Vertex AI, Anthropic or Hugging Face. -* For built-in models and models uploaded through Eland, the {infer} APIs offer an -alternative way to use and manage trained models. However, if you do not plan to -use the {infer} APIs to use these models or if you want to use non-NLP models, -use the <>. +* The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Mistral, Azure OpenAI, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face. +* For built-in models and models uploaded through Eland, the {infer} APIs offer an alternative way to use and manage trained models. However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the <>. ==== - [discrete] [[put-inference-api-request]] ==== {api-request-title} @@ -54,11 +52,19 @@ Refer to the service list in the <> API. In the response, look for `"state": "fully_allocated"` and ensure the `"allocation_count"` matches the `"target_allocation_count"`. +* Avoid creating multiple endpoints for the same model unless required, as each endpoint consumes significant resources. +==== + + The following services are available through the {infer} API. You can find the available task types next to the service name. Click the links to review the configuration details of the services: -* <> (`rerank`, `sparse_embedding`, `text_embedding`) +* <> (`completion`, `rerank`, `sparse_embedding`, `text_embedding`) * <> (`completion`, `text_embedding`) * <> (`completion`) * <> (`completion`, `text_embedding`) @@ -71,6 +77,8 @@ Click the links to review the configuration details of the services: * <> (`text_embedding`) * <> (`text_embedding`) * <> (`completion`, `text_embedding`) +* <> (`text_embedding`) +* <> (`text_embedding`, `rerank`) The {es} and ELSER services run on a {ml} node in your {es} cluster. The rest of the services connect to external providers. \ No newline at end of file diff --git a/docs/reference/inference/service-alibabacloud-ai-search.asciidoc b/docs/reference/inference/service-alibabacloud-ai-search.asciidoc index 23a3d532635ac..5a11190549ce6 100644 --- a/docs/reference/inference/service-alibabacloud-ai-search.asciidoc +++ b/docs/reference/inference/service-alibabacloud-ai-search.asciidoc @@ -1,6 +1,12 @@ [[infer-service-alibabacloud-ai-search]] === AlibabaCloud AI Search {infer} service +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs]. +-- + Creates an {infer} endpoint to perform an {infer} task with the `alibabacloud-ai-search` service. [discrete] @@ -24,15 +30,36 @@ include::inference-shared.asciidoc[tag=task-type] -- Available task types: -* `text_embedding`, +* `completion`, +* `rerank` * `sparse_embedding`, -* `rerank`. +* `text_embedding`. -- [discrete] [[infer-service-alibabacloud-ai-search-api-request-body]] ==== {api-request-body-title} +`chunking_settings`:: +(Optional, object) +include::inference-shared.asciidoc[tag=chunking-settings] + +`max_chunking_size`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-max-chunking-size] + +`overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-overlap] + +`sentence_overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-sentence-overlap] + +`strategy`::: +(Optional, string) +include::inference-shared.asciidoc[tag=chunking-settings-strategy] + `service`:: (Required, string) The type of service supported for the specified task type. In this case, @@ -55,14 +82,21 @@ A valid API key for the AlibabaCloud AI Search API. The name of the model service to use for the {infer} task. + -- -Available service_ids for the `text_embedding` task: +Available service_ids for the `completion` task: -* `ops-text-embedding-001` -* `ops-text-embedding-zh-001` -* `ops-text-embedding-en-001` -* `ops-text-embedding-002` +* `ops-qwen-turbo` +* `qwen-turbo` +* `qwen-plus` +* `qwen-max` +÷ `qwen-max-longcontext` -For the supported `text_embedding` service_ids, refer to the https://help.aliyun.com/zh/open-search/search-platform/developer-reference/text-embedding-api-details[documentation]. +For the supported `completion` service_ids, refer to the https://help.aliyun.com/zh/open-search/search-platform/developer-reference/text-generation-api-details[documentation]. + +Available service_id for the `rerank` task is: + +* `ops-bge-reranker-larger` + +For the supported `rerank` service_id, refer to the https://help.aliyun.com/zh/open-search/search-platform/developer-reference/ranker-api-details[documentation]. Available service_id for the `sparse_embedding` task: @@ -70,16 +104,21 @@ Available service_id for the `sparse_embedding` task: For the supported `sparse_embedding` service_id, refer to the https://help.aliyun.com/zh/open-search/search-platform/developer-reference/text-sparse-embedding-api-details[documentation]. -Available service_id for the `rerank` task is: +Available service_ids for the `text_embedding` task: -* `ops-bge-reranker-larger` +* `ops-text-embedding-001` +* `ops-text-embedding-zh-001` +* `ops-text-embedding-en-001` +* `ops-text-embedding-002` + +For the supported `text_embedding` service_ids, refer to the https://help.aliyun.com/zh/open-search/search-platform/developer-reference/text-embedding-api-details[documentation]. -For the supported `rerank` service_id, refer to the https://help.aliyun.com/zh/open-search/search-platform/developer-reference/ranker-api-details[documentation]. -- `host`::: (Required, string) -The name of the host address used for the {infer} task. You can find the host address at https://opensearch.console.aliyun.com/cn-shanghai/rag/api-key[ the API keys section] of the documentation. +The name of the host address used for the {infer} task. +You can find the host address at https://opensearch.console.aliyun.com/cn-shanghai/rag/api-key[the API keys section] of the documentation. `workspace`::: (Required, string) @@ -95,7 +134,6 @@ To modify this, set the `requests_per_minute` setting of this object in your ser include::inference-shared.asciidoc[tag=request-per-minute-example] -- - `task_settings`:: (Optional, object) include::inference-shared.asciidoc[tag=task-settings] @@ -130,16 +168,34 @@ If `true`, the token name will be returned in the response. Defaults to `false` [[inference-example-alibabacloud-ai-search]] ==== AlibabaCloud AI Search service examples -The following example shows how to create an {infer} endpoint called `alibabacloud_ai_search_embeddings` to perform a `text_embedding` task type. +The following example shows how to create an {infer} endpoint called `alibabacloud_ai_search_completion` to perform a `completion` task type. [source,console] ------------------------------------------------------------ -PUT _inference/text_embedding/alibabacloud_ai_search_embeddings +PUT _inference/completion/alibabacloud_ai_search_completion +{ + "service": "alibabacloud-ai-search", + "service_settings": { + "host" : "default-j01.platform-cn-shanghai.opensearch.aliyuncs.com", + "api_key": "{{API_KEY}}", + "service_id": "ops-qwen-turbo", + "workspace" : "default" + } +} +------------------------------------------------------------ +// TEST[skip:TBD] + + +The next example shows how to create an {infer} endpoint called `alibabacloud_ai_search_rerank` to perform a `rerank` task type. + +[source,console] +------------------------------------------------------------ +PUT _inference/rerank/alibabacloud_ai_search_rerank { "service": "alibabacloud-ai-search", "service_settings": { "api_key": "", - "service_id": "ops-text-embedding-001", + "service_id": "ops-bge-reranker-larger", "host": "default-j01.platform-cn-shanghai.opensearch.aliyuncs.com", "workspace": "default" } @@ -147,8 +203,8 @@ PUT _inference/text_embedding/alibabacloud_ai_search_embeddings ------------------------------------------------------------ // TEST[skip:TBD] -The following example shows how to create an {infer} endpoint called -`alibabacloud_ai_search_sparse` to perform a `sparse_embedding` task type. + +The following example shows how to create an {infer} endpoint called `alibabacloud_ai_search_sparse` to perform a `sparse_embedding` task type. [source,console] ------------------------------------------------------------ @@ -165,20 +221,23 @@ PUT _inference/sparse_embedding/alibabacloud_ai_search_sparse ------------------------------------------------------------ // TEST[skip:TBD] -The next example shows how to create an {infer} endpoint called -`alibabacloud_ai_search_rerank` to perform a `rerank` task type. + +The following example shows how to create an {infer} endpoint called `alibabacloud_ai_search_embeddings` to perform a `text_embedding` task type. [source,console] ------------------------------------------------------------ -PUT _inference/rerank/alibabacloud_ai_search_rerank +PUT _inference/text_embedding/alibabacloud_ai_search_embeddings { "service": "alibabacloud-ai-search", "service_settings": { "api_key": "", - "service_id": "ops-bge-reranker-larger", + "service_id": "ops-text-embedding-001", "host": "default-j01.platform-cn-shanghai.opensearch.aliyuncs.com", "workspace": "default" } } ------------------------------------------------------------ // TEST[skip:TBD] + + + diff --git a/docs/reference/inference/service-amazon-bedrock.asciidoc b/docs/reference/inference/service-amazon-bedrock.asciidoc index dbffd5c26fbcc..ed25ce0d515b5 100644 --- a/docs/reference/inference/service-amazon-bedrock.asciidoc +++ b/docs/reference/inference/service-amazon-bedrock.asciidoc @@ -1,6 +1,12 @@ [[infer-service-amazon-bedrock]] === Amazon Bedrock {infer} service +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs]. +-- + Creates an {infer} endpoint to perform an {infer} task with the `amazonbedrock` service. [discrete] @@ -32,6 +38,26 @@ Available task types: [[infer-service-amazon-bedrock-api-request-body]] ==== {api-request-body-title} +`chunking_settings`:: +(Optional, object) +include::inference-shared.asciidoc[tag=chunking-settings] + +`max_chunking_size`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-max-chunking-size] + +`overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-overlap] + +`sentence_overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-sentence-overlap] + +`strategy`::: +(Optional, string) +include::inference-shared.asciidoc[tag=chunking-settings-strategy] + `service`:: (Required, string) The type of service supported for the specified task type. In this case, diff --git a/docs/reference/inference/service-anthropic.asciidoc b/docs/reference/inference/service-anthropic.asciidoc index 41419db7a6069..4ce76dc1d57bd 100644 --- a/docs/reference/inference/service-anthropic.asciidoc +++ b/docs/reference/inference/service-anthropic.asciidoc @@ -1,6 +1,12 @@ [[infer-service-anthropic]] === Anthropic {infer} service +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs]. +-- + Creates an {infer} endpoint to perform an {infer} task with the `anthropic` service. @@ -32,6 +38,26 @@ Available task types: [[infer-service-anthropic-api-request-body]] ==== {api-request-body-title} +`chunking_settings`:: +(Optional, object) +include::inference-shared.asciidoc[tag=chunking-settings] + +`max_chunking_size`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-max-chunking-size] + +`overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-overlap] + +`sentence_overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-sentence-overlap] + +`strategy`::: +(Optional, string) +include::inference-shared.asciidoc[tag=chunking-settings-strategy] + `service`:: (Required, string) The type of service supported for the specified task type. In this case, diff --git a/docs/reference/inference/service-azure-ai-studio.asciidoc b/docs/reference/inference/service-azure-ai-studio.asciidoc index 0d711a0d6171f..7ada8df1ecdaa 100644 --- a/docs/reference/inference/service-azure-ai-studio.asciidoc +++ b/docs/reference/inference/service-azure-ai-studio.asciidoc @@ -1,6 +1,12 @@ [[infer-service-azure-ai-studio]] === Azure AI studio {infer} service +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs]. +-- + Creates an {infer} endpoint to perform an {infer} task with the `azureaistudio` service. @@ -33,6 +39,26 @@ Available task types: [[infer-service-azure-ai-studio-api-request-body]] ==== {api-request-body-title} +`chunking_settings`:: +(Optional, object) +include::inference-shared.asciidoc[tag=chunking-settings] + +`max_chunking_size`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-max-chunking-size] + +`overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-overlap] + +`sentence_overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-sentence-overlap] + +`strategy`::: +(Optional, string) +include::inference-shared.asciidoc[tag=chunking-settings-strategy] + `service`:: (Required, string) The type of service supported for the specified task type. In this case, diff --git a/docs/reference/inference/service-azure-openai.asciidoc b/docs/reference/inference/service-azure-openai.asciidoc index 6f03c5966d9e6..170c0939166f7 100644 --- a/docs/reference/inference/service-azure-openai.asciidoc +++ b/docs/reference/inference/service-azure-openai.asciidoc @@ -1,6 +1,12 @@ [[infer-service-azure-openai]] === Azure OpenAI {infer} service +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs]. +-- + Creates an {infer} endpoint to perform an {infer} task with the `azureopenai` service. @@ -33,6 +39,26 @@ Available task types: [[infer-service-azure-openai-api-request-body]] ==== {api-request-body-title} +`chunking_settings`:: +(Optional, object) +include::inference-shared.asciidoc[tag=chunking-settings] + +`max_chunking_size`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-max-chunking-size] + +`overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-overlap] + +`sentence_overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-sentence-overlap] + +`strategy`::: +(Optional, string) +include::inference-shared.asciidoc[tag=chunking-settings-strategy] + `service`:: (Required, string) The type of service supported for the specified task type. In this case, diff --git a/docs/reference/inference/service-cohere.asciidoc b/docs/reference/inference/service-cohere.asciidoc index 84eae6e880617..70e311c810cdd 100644 --- a/docs/reference/inference/service-cohere.asciidoc +++ b/docs/reference/inference/service-cohere.asciidoc @@ -1,6 +1,12 @@ [[infer-service-cohere]] === Cohere {infer} service +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs]. +-- + Creates an {infer} endpoint to perform an {infer} task with the `cohere` service. @@ -34,6 +40,26 @@ Available task types: [[infer-service-cohere-api-request-body]] ==== {api-request-body-title} +`chunking_settings`:: +(Optional, object) +include::inference-shared.asciidoc[tag=chunking-settings] + +`max_chunking_size`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-max-chunking-size] + +`overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-overlap] + +`sentence_overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-sentence-overlap] + +`strategy`::: +(Optional, string) +include::inference-shared.asciidoc[tag=chunking-settings-strategy] + `service`:: (Required, string) The type of service supported for the specified task type. In this case, diff --git a/docs/reference/inference/service-elasticsearch.asciidoc b/docs/reference/inference/service-elasticsearch.asciidoc index 572cad591fba6..a388bcf253eab 100644 --- a/docs/reference/inference/service-elasticsearch.asciidoc +++ b/docs/reference/inference/service-elasticsearch.asciidoc @@ -1,13 +1,19 @@ [[infer-service-elasticsearch]] === Elasticsearch {infer} service -Creates an {infer} endpoint to perform an {infer} task with the `elasticsearch` -service. +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs]. +-- -NOTE: If you use the E5 model through the `elasticsearch` service, the API -request will automatically download and deploy the model if it isn't downloaded -yet. +Creates an {infer} endpoint to perform an {infer} task with the `elasticsearch` service. +[NOTE] +==== +* Your {es} deployment contains <>, you only need to create the enpoints using the API if you want to customize the settings. +* If you use the ELSER or the E5 model through the `elasticsearch` service, the API request will automatically download and deploy the model if it isn't downloaded yet. +==== [discrete] [[infer-service-elasticsearch-api-request]] @@ -39,6 +45,26 @@ Available task types: [[infer-service-elasticsearch-api-request-body]] ==== {api-request-body-title} +`chunking_settings`:: +(Optional, object) +include::inference-shared.asciidoc[tag=chunking-settings] + +`max_chunking_size`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-max-chunking-size] + +`overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-overlap] + +`sentence_overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-sentence-overlap] + +`strategy`::: +(Optional, string) +include::inference-shared.asciidoc[tag=chunking-settings-strategy] + `service`:: (Required, string) The type of service supported for the specified task type. In this case, @@ -52,6 +78,11 @@ include::inference-shared.asciidoc[tag=service-settings] These settings are specific to the `elasticsearch` service. -- +`deployment_id`::: +(Optional, string) +The `deployment_id` of an existing trained model deployment. +When `deployment_id` is used the `model_id` is optional. + `adaptive_allocations`::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=adaptive-allocation] @@ -71,7 +102,7 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=adaptive-allocation-min-number] `model_id`::: (Required, string) The name of the model to use for the {infer} task. -It can be the ID of either a built-in model (for example, `.multilingual-e5-small` for E5) or a text embedding model already +It can be the ID of either a built-in model (for example, `.multilingual-e5-small` for E5), a text embedding model already {ml-docs}/ml-nlp-import-model.html#ml-nlp-import-script[uploaded through Eland]. `num_allocations`::: @@ -97,16 +128,72 @@ include::inference-shared.asciidoc[tag=task-settings] Returns the document instead of only the index. Defaults to `true`. ===== +[discrete] +[[inference-example-elasticsearch-elser]] +==== ELSER via the `elasticsearch` service + +The following example shows how to create an {infer} endpoint called `my-elser-model` to perform a `sparse_embedding` task type. + +The API request below will automatically download the ELSER model if it isn't already downloaded and then deploy the model. + +[source,console] +------------------------------------------------------------ +PUT _inference/sparse_embedding/my-elser-model +{ + "service": "elasticsearch", + "service_settings": { + "adaptive_allocations": { <1> + "enabled": true, + "min_number_of_allocations": 1, + "max_number_of_allocations": 4 + }, + "num_threads": 1, + "model_id": ".elser_model_2" <2> + } +} +------------------------------------------------------------ +// TEST[skip:TBD] +<1> Adaptive allocations will be enabled with the minimum of 1 and the maximum of 10 allocations. +<2> The `model_id` must be the ID of one of the built-in ELSER models. +Valid values are `.elser_model_2` and `.elser_model_2_linux-x86_64`. +For further details, refer to the {ml-docs}/ml-nlp-elser.html[ELSER model documentation]. + +[discrete] +[[inference-example-elastic-reranker]] +==== Elastic Rerank via the `elasticsearch` service + +The following example shows how to create an {infer} endpoint called `my-elastic-rerank` to perform a `rerank` task type using the built-in Elastic Rerank cross-encoder model. + +The API request below will automatically download the Elastic Rerank model if it isn't already downloaded and then deploy the model. +Once deployed, the model can be used for semantic re-ranking with a <>. + +[source,console] +------------------------------------------------------------ +PUT _inference/rerank/my-elastic-rerank +{ + "service": "elasticsearch", + "service_settings": { + "model_id": ".rerank-v1", <1> + "num_threads": 1, + "adaptive_allocations": { <2> + "enabled": true, + "min_number_of_allocations": 1, + "max_number_of_allocations": 4 + } + } +} +------------------------------------------------------------ +// TEST[skip:TBD] +<1> The `model_id` must be the ID of the built-in Elastic Rerank model: `.rerank-v1`. +<2> {ml-docs}/ml-nlp-auto-scale.html#nlp-model-adaptive-allocations[Adaptive allocations] will be enabled with the minimum of 1 and the maximum of 10 allocations. [discrete] [[inference-example-elasticsearch]] ==== E5 via the `elasticsearch` service -The following example shows how to create an {infer} endpoint called -`my-e5-model` to perform a `text_embedding` task type. +The following example shows how to create an {infer} endpoint called `my-e5-model` to perform a `text_embedding` task type. -The API request below will automatically download the E5 model if it isn't -already downloaded and then deploy the model. +The API request below will automatically download the E5 model if it isn't already downloaded and then deploy the model. [source,console] ------------------------------------------------------------ @@ -135,7 +222,7 @@ If using the Python client, you can set the `timeout` parameter to a higher valu [discrete] [[inference-example-eland]] -==== Models uploaded by Eland via the elasticsearch service +==== Models uploaded by Eland via the `elasticsearch` service The following example shows how to create an {infer} endpoint called `my-msmarco-minilm-model` to perform a `text_embedding` task type. @@ -179,8 +266,52 @@ PUT _inference/text_embedding/my-e5-model "min_number_of_allocations": 3, "max_number_of_allocations": 10 }, + "num_threads": 1, "model_id": ".multilingual-e5-small" } } ------------------------------------------------------------ // TEST[skip:TBD] + + +[discrete] +[[inference-example-existing-deployment]] +==== Using an existing model deployment with the `elasticsearch` service + +The following example shows how to use an already existing model deployment when creating an {infer} endpoint. + +[source,console] +------------------------------------------------------------ +PUT _inference/sparse_embedding/use_existing_deployment +{ + "service": "elasticsearch", + "service_settings": { + "deployment_id": ".elser_model_2" <1> + } +} +------------------------------------------------------------ +// TEST[skip:TBD] +<1> The `deployment_id` of the already existing model deployment. + +The API response contains the `model_id`, and the threads and allocations settings from the model deployment: + +[source,console-result] +------------------------------------------------------------ +{ + "inference_id": "use_existing_deployment", + "task_type": "sparse_embedding", + "service": "elasticsearch", + "service_settings": { + "num_allocations": 2, + "num_threads": 1, + "model_id": ".elser_model_2", + "deployment_id": ".elser_model_2" + }, + "chunking_settings": { + "strategy": "sentence", + "max_chunk_size": 250, + "sentence_overlap": 1 + } +} +------------------------------------------------------------ +// NOTCONSOLE \ No newline at end of file diff --git a/docs/reference/inference/service-elser.asciidoc b/docs/reference/inference/service-elser.asciidoc index fdce94901984b..47aaa58814602 100644 --- a/docs/reference/inference/service-elser.asciidoc +++ b/docs/reference/inference/service-elser.asciidoc @@ -1,11 +1,27 @@ [[infer-service-elser]] === ELSER {infer} service +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs]. +-- + Creates an {infer} endpoint to perform an {infer} task with the `elser` service. +You can also deploy ELSER by using the <>. -NOTE: The API request will automatically download and deploy the ELSER model if -it isn't already downloaded. +[NOTE] +==== +* Your {es} deployment contains <>, you only need to create the enpoint using the API if you want to customize the settings. +* The API request will automatically download and deploy the ELSER model if it isn't already downloaded. +==== +[WARNING] +.Deprecated in 8.16 +==== +The `elser` service is deprecated and will be removed in a future release. +Use the <> instead, with `model_id` included in the `service_settings`. +==== [discrete] [[infer-service-elser-api-request]] @@ -35,6 +51,26 @@ Available task types: [[infer-service-elser-api-request-body]] ==== {api-request-body-title} +`chunking_settings`:: +(Optional, object) +include::inference-shared.asciidoc[tag=chunking-settings] + +`max_chunking_size`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-max-chunking-size] + +`overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-overlap] + +`sentence_overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-sentence-overlap] + +`strategy`::: +(Optional, string) +include::inference-shared.asciidoc[tag=chunking-settings-strategy] + `service`:: (Required, string) The type of service supported for the specified task type. In this case, @@ -75,17 +111,47 @@ If `adaptive_allocations` is enabled, do not set this value, because it's automa Sets the number of threads used by each model allocation during inference. This generally increases the speed per inference request. The inference process is a compute-bound process; `threads_per_allocations` must not exceed the number of available allocated processors per node. Must be a power of 2. Max allowed value is 32. +[discrete] +[[inference-example-elser-adaptive-allocation]] +==== ELSER service example with adaptive allocations + +When adaptive allocations are enabled, the number of allocations of the model is set automatically based on the current load. + +NOTE: For more information on how to optimize your ELSER endpoints, refer to {ml-docs}/ml-nlp-elser.html#elser-recommendations[the ELSER recommendations] section in the model documentation. +To learn more about model autoscaling, refer to the {ml-docs}/ml-nlp-auto-scale.html[trained model autoscaling] page. + +The following example shows how to create an {infer} endpoint called `my-elser-model` to perform a `sparse_embedding` task type and configure adaptive allocations. + +The request below will automatically download the ELSER model if it isn't already downloaded and then deploy the model. + +[source,console] +------------------------------------------------------------ +PUT _inference/sparse_embedding/my-elser-model +{ + "service": "elser", + "service_settings": { + "adaptive_allocations": { + "enabled": true, + "min_number_of_allocations": 3, + "max_number_of_allocations": 10 + }, + "num_threads": 1 + } +} +------------------------------------------------------------ +// TEST[skip:TBD] [discrete] [[inference-example-elser]] -==== ELSER service example +==== ELSER service example without adaptive allocations -The following example shows how to create an {infer} endpoint called -`my-elser-model` to perform a `sparse_embedding` task type. +The following example shows how to create an {infer} endpoint called `my-elser-model` to perform a `sparse_embedding` task type. Refer to the {ml-docs}/ml-nlp-elser.html[ELSER model documentation] for more info. -The request below will automatically download the ELSER model if it isn't -already downloaded and then deploy the model. +NOTE: If you want to optimize your ELSER endpoint for ingest, set the number of threads to `1` (`"num_threads": 1`). +If you want to optimize your ELSER endpoint for search, set the number of threads to greater than `1`. + +The request below will automatically download the ELSER model if it isn't already downloaded and then deploy the model. [source,console] ------------------------------------------------------------ @@ -100,7 +166,6 @@ PUT _inference/sparse_embedding/my-elser-model ------------------------------------------------------------ // TEST[skip:TBD] - Example response: [source,console-result] @@ -124,31 +189,4 @@ You might see a 502 bad gateway error in the response when using the {kib} Conso This error usually just reflects a timeout, while the model downloads in the background. You can check the download progress in the {ml-app} UI. If using the Python client, you can set the `timeout` parameter to a higher value. -==== - -[discrete] -[[inference-example-elser-adaptive-allocation]] -==== Setting adaptive allocation for the ELSER service - -The following example shows how to create an {infer} endpoint called -`my-elser-model` to perform a `sparse_embedding` task type and configure -adaptive allocations. - -The request below will automatically download the ELSER model if it isn't -already downloaded and then deploy the model. - -[source,console] ------------------------------------------------------------- -PUT _inference/sparse_embedding/my-elser-model -{ - "service": "elser", - "service_settings": { - "adaptive_allocations": { - "enabled": true, - "min_number_of_allocations": 3, - "max_number_of_allocations": 10 - } - } -} ------------------------------------------------------------- -// TEST[skip:TBD] \ No newline at end of file +==== \ No newline at end of file diff --git a/docs/reference/inference/service-google-ai-studio.asciidoc b/docs/reference/inference/service-google-ai-studio.asciidoc index 25aa89cd49110..5b30292fb9beb 100644 --- a/docs/reference/inference/service-google-ai-studio.asciidoc +++ b/docs/reference/inference/service-google-ai-studio.asciidoc @@ -1,6 +1,12 @@ [[infer-service-google-ai-studio]] === Google AI Studio {infer} service +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs]. +-- + Creates an {infer} endpoint to perform an {infer} task with the `googleaistudio` service. @@ -33,6 +39,26 @@ Available task types: [[infer-service-google-ai-studio-api-request-body]] ==== {api-request-body-title} +`chunking_settings`:: +(Optional, object) +include::inference-shared.asciidoc[tag=chunking-settings] + +`max_chunking_size`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-max-chunking-size] + +`overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-overlap] + +`sentence_overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-sentence-overlap] + +`strategy`::: +(Optional, string) +include::inference-shared.asciidoc[tag=chunking-settings-strategy] + `service`:: (Required, string) The type of service supported for the specified task type. In this case, diff --git a/docs/reference/inference/service-google-vertex-ai.asciidoc b/docs/reference/inference/service-google-vertex-ai.asciidoc index 640553ab74626..28fa65b6e5fcc 100644 --- a/docs/reference/inference/service-google-vertex-ai.asciidoc +++ b/docs/reference/inference/service-google-vertex-ai.asciidoc @@ -1,6 +1,12 @@ [[infer-service-google-vertex-ai]] === Google Vertex AI {infer} service +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs]. +-- + Creates an {infer} endpoint to perform an {infer} task with the `googlevertexai` service. @@ -33,6 +39,26 @@ Available task types: [[infer-service-google-vertex-ai-api-request-body]] ==== {api-request-body-title} +`chunking_settings`:: +(Optional, object) +include::inference-shared.asciidoc[tag=chunking-settings] + +`max_chunking_size`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-max-chunking-size] + +`overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-overlap] + +`sentence_overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-sentence-overlap] + +`strategy`::: +(Optional, string) +include::inference-shared.asciidoc[tag=chunking-settings-strategy] + `service`:: (Required, string) The type of service supported for the specified task type. In this case, diff --git a/docs/reference/inference/service-hugging-face.asciidoc b/docs/reference/inference/service-hugging-face.asciidoc index 177a15177d21f..862914c141740 100644 --- a/docs/reference/inference/service-hugging-face.asciidoc +++ b/docs/reference/inference/service-hugging-face.asciidoc @@ -1,6 +1,12 @@ [[infer-service-hugging-face]] === HuggingFace {infer} service +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs]. +-- + Creates an {infer} endpoint to perform an {infer} task with the `hugging_face` service. @@ -32,6 +38,26 @@ Available task types: [[infer-service-hugging-face-api-request-body]] ==== {api-request-body-title} +`chunking_settings`:: +(Optional, object) +include::inference-shared.asciidoc[tag=chunking-settings] + +`max_chunking_size`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-max-chunking-size] + +`overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-overlap] + +`sentence_overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-sentence-overlap] + +`strategy`::: +(Optional, string) +include::inference-shared.asciidoc[tag=chunking-settings-strategy] + `service`:: (Required, string) The type of service supported for the specified task type. In this case, diff --git a/docs/reference/inference/service-jinaai.asciidoc b/docs/reference/inference/service-jinaai.asciidoc new file mode 100644 index 0000000000000..7c5aebe5bcf8e --- /dev/null +++ b/docs/reference/inference/service-jinaai.asciidoc @@ -0,0 +1,255 @@ +[[infer-service-jinaai]] +=== JinaAI {infer} service + +Creates an {infer} endpoint to perform an {infer} task with the `jinaai` service. + + +[discrete] +[[infer-service-jinaai-api-request]] +==== {api-request-title} + +`PUT /_inference//` + +[discrete] +[[infer-service-jinaai-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) +include::inference-shared.asciidoc[tag=inference-id] + +``:: +(Required, string) +include::inference-shared.asciidoc[tag=task-type] ++ +-- +Available task types: + +* `text_embedding`, +* `rerank`. +-- + +[discrete] +[[infer-service-jinaai-api-request-body]] +==== {api-request-body-title} + +`chunking_settings`:: +(Optional, object) +include::inference-shared.asciidoc[tag=chunking-settings] + +`max_chunking_size`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-max-chunking-size] + +`overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-overlap] + +`sentence_overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-sentence-overlap] + +`strategy`::: +(Optional, string) +include::inference-shared.asciidoc[tag=chunking-settings-strategy] + +`service`:: +(Required, string) +The type of service supported for the specified task type. In this case, +`jinaai`. + +`service_settings`:: +(Required, object) +include::inference-shared.asciidoc[tag=service-settings] ++ +-- +These settings are specific to the `jinaai` service. +-- + +`api_key`::: +(Required, string) +A valid API key for your JinaAI account. +You can find it at https://jina.ai/embeddings/. ++ +-- +include::inference-shared.asciidoc[tag=api-key-admonition] +-- + +`rate_limit`::: +(Optional, object) +The default rate limit for the `jinaai` service is 2000 requests per minute for all task types. +You can modify this using the `requests_per_minute` setting in your service settings: ++ +-- +include::inference-shared.asciidoc[tag=request-per-minute-example] + +More information about JinaAI's rate limits can be found in https://jina.ai/contact-sales/#rate-limit. +-- ++ +.`service_settings` for the `rerank` task type +[%collapsible%closed] +===== +`model_id`:: +(Required, string) +The name of the model to use for the {infer} task. +To review the available `rerank` compatible models, refer to https://jina.ai/reranker. +===== ++ +.`service_settings` for the `text_embedding` task type +[%collapsible%closed] +===== +`model_id`::: +(Optional, string) +The name of the model to use for the {infer} task. +To review the available `text_embedding` models, refer to the +https://jina.ai/embeddings/. + +`similarity`::: +(Optional, string) +Similarity measure. One of `cosine`, `dot_product`, `l2_norm`. +Defaults based on the `embedding_type` (`float` -> `dot_product`, `int8/byte` -> `cosine`). +===== + + + +`task_settings`:: +(Optional, object) +include::inference-shared.asciidoc[tag=task-settings] ++ +.`task_settings` for the `rerank` task type +[%collapsible%closed] +===== +`return_documents`:: +(Optional, boolean) +Specify whether to return doc text within the results. + +`top_n`:: +(Optional, integer) +The number of most relevant documents to return, defaults to the number of the documents. +If this {infer} endpoint is used in a `text_similarity_reranker` retriever query and `top_n` is set, it must be greater than or equal to `rank_window_size` in the query. +===== ++ +.`task_settings` for the `text_embedding` task type +[%collapsible%closed] +===== +`task`::: +(Optional, string) +Specifies the task passed to the model. +Valid values are: +* `classification`: use it for embeddings passed through a text classifier. +* `clustering`: use it for the embeddings run through a clustering algorithm. +* `ingest`: use it for storing document embeddings in a vector database. +* `search`: use it for storing embeddings of search queries run against a vector database to find relevant documents. +===== + + +[discrete] +[[inference-example-jinaai]] +==== JinaAI service examples + +The following examples demonstrate how to create {infer} endpoints for `text_embeddings` and `rerank` tasks using the JinaAI service and use them in search requests. + +First, we create the `embeddings` service: + +[source,console] +------------------------------------------------------------ +PUT _inference/text_embedding/jinaai-embeddings +{ + "service": "jinaai", + "service_settings": { + "model_id": "jina-embeddings-v3", + "api_key": "" + } +} +------------------------------------------------------------ +// TEST[skip:uses ML] + +Then, we create the `rerank` service: +[source,console] +------------------------------------------------------------ +PUT _inference/rerank/jinaai-rerank +{ + "service": "jinaai", + "service_settings": { + "api_key": "", + "model_id": "jina-reranker-v2-base-multilingual" + }, + "task_settings": { + "top_n": 10, + "return_documents": true + } +} +------------------------------------------------------------ +// TEST[skip:uses ML] + +Now we can create an index that will use `jinaai-embeddings` service to index the documents. + +[source,console] +------------------------------------------------------------ +PUT jinaai-index +{ + "mappings": { + "properties": { + "content": { + "type": "semantic_text", + "inference_id": "jinaai-embeddings" + } + } + } +} +------------------------------------------------------------ +// TEST[skip:uses ML] + +[source,console] +------------------------------------------------------------ +PUT jinaai-index/_bulk +{ "index" : { "_index" : "jinaai-index", "_id" : "1" } } +{"content": "Sarah Johnson is a talented marine biologist working at the Oceanographic Institute. Her groundbreaking research on coral reef ecosystems has garnered international attention and numerous accolades."} +{ "index" : { "_index" : "jinaai-index", "_id" : "2" } } +{"content": "She spends months at a time diving in remote locations, meticulously documenting the intricate relationships between various marine species. "} +{ "index" : { "_index" : "jinaai-index", "_id" : "3" } } +{"content": "Her dedication to preserving these delicate underwater environments has inspired a new generation of conservationists."} +------------------------------------------------------------ +// TEST[skip:uses ML] + +Now, with the index created, we can search with and without the reranker service. + +[source,console] +------------------------------------------------------------ +GET jinaai-index/_search +{ + "query": { + "semantic": { + "field": "content", + "query": "who inspired taking care of the sea?" + } + } +} +------------------------------------------------------------ +// TEST[skip:uses ML] + +[source,console] +------------------------------------------------------------ +POST jinaai-index/_search +{ + "retriever": { + "text_similarity_reranker": { + "retriever": { + "standard": { + "query": { + "semantic": { + "field": "content", + "query": "who inspired taking care of the sea?" + } + } + } + }, + "field": "content", + "rank_window_size": 100, + "inference_id": "jinaai-rerank", + "inference_text": "who inspired taking care of the sea?" + } + } +} +------------------------------------------------------------ +// TEST[skip:uses ML] \ No newline at end of file diff --git a/docs/reference/inference/service-mistral.asciidoc b/docs/reference/inference/service-mistral.asciidoc index 077e610191705..326e8458be767 100644 --- a/docs/reference/inference/service-mistral.asciidoc +++ b/docs/reference/inference/service-mistral.asciidoc @@ -1,6 +1,12 @@ [[infer-service-mistral]] === Mistral {infer} service +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs]. +-- + Creates an {infer} endpoint to perform an {infer} task with the `mistral` service. @@ -32,6 +38,26 @@ Available task types: [[infer-service-mistral-api-request-body]] ==== {api-request-body-title} +`chunking_settings`:: +(Optional, object) +include::inference-shared.asciidoc[tag=chunking-settings] + +`max_chunking_size`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-max-chunking-size] + +`overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-overlap] + +`sentence_overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-sentence-overlap] + +`strategy`::: +(Optional, string) +include::inference-shared.asciidoc[tag=chunking-settings-strategy] + `service`:: (Required, string) The type of service supported for the specified task type. In this case, diff --git a/docs/reference/inference/service-openai.asciidoc b/docs/reference/inference/service-openai.asciidoc index 075e76dc7d741..e4be7f18e09dd 100644 --- a/docs/reference/inference/service-openai.asciidoc +++ b/docs/reference/inference/service-openai.asciidoc @@ -1,6 +1,12 @@ [[infer-service-openai]] === OpenAI {infer} service +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs]. +-- + Creates an {infer} endpoint to perform an {infer} task with the `openai` service. @@ -33,6 +39,26 @@ Available task types: [[infer-service-openai-api-request-body]] ==== {api-request-body-title} +`chunking_settings`:: +(Optional, object) +include::inference-shared.asciidoc[tag=chunking-settings] + +`max_chunking_size`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-max-chunking-size] + +`overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-overlap] + +`sentence_overlap`::: +(Optional, integer) +include::inference-shared.asciidoc[tag=chunking-settings-sentence-overlap] + +`strategy`::: +(Optional, string) +include::inference-shared.asciidoc[tag=chunking-settings-strategy] + `service`:: (Required, string) The type of service supported for the specified task type. In this case, @@ -56,6 +82,12 @@ https://platform.openai.com/api-keys[API keys section]. include::inference-shared.asciidoc[tag=api-key-admonition] -- +`dimensions`::: +(Optional, integer) +The number of dimensions the resulting output embeddings should have. +Only supported in `text-embedding-3` and later models. +If not set the OpenAI defined default for the model is used. + `model_id`::: (Required, string) The name of the model to use for the {infer} task. @@ -114,8 +146,8 @@ Specifies the user issuing the request, which can be used for abuse detection. [[inference-example-openai]] ==== OpenAI service example -The following example shows how to create an {infer} endpoint called -`openai-embeddings` to perform a `text_embedding` task type. +The following example shows how to create an {infer} endpoint called `openai-embeddings` to perform a `text_embedding` task type. +The embeddings created by requests to this endpoint will have 128 dimensions. [source,console] ------------------------------------------------------------ @@ -124,14 +156,14 @@ PUT _inference/text_embedding/openai-embeddings "service": "openai", "service_settings": { "api_key": "", - "model_id": "text-embedding-ada-002" + "model_id": "text-embedding-3-small", + "dimensions": 128 } } ------------------------------------------------------------ // TEST[skip:TBD] -The next example shows how to create an {infer} endpoint called -`openai-completion` to perform a `completion` task type. +The next example shows how to create an {infer} endpoint called `openai-completion` to perform a `completion` task type. [source,console] ------------------------------------------------------------ diff --git a/docs/reference/inference/service-watsonx-ai.asciidoc b/docs/reference/inference/service-watsonx-ai.asciidoc new file mode 100644 index 0000000000000..e7bba7b4e9a97 --- /dev/null +++ b/docs/reference/inference/service-watsonx-ai.asciidoc @@ -0,0 +1,121 @@ +[[infer-service-watsonx-ai]] +=== Watsonx {infer} service + +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs]. +-- + +Creates an {infer} endpoint to perform an {infer} task with the `watsonxai` service. + +You need an https://cloud.ibm.com/docs/databases-for-elasticsearch?topic=databases-for-elasticsearch-provisioning&interface=api[IBM Cloud® Databases for Elasticsearch deployment] to use the `watsonxai` {infer} service. +You can provision one through the https://cloud.ibm.com/databases/databases-for-elasticsearch/create[IBM catalog], the https://cloud.ibm.com/docs/databases-cli-plugin?topic=databases-cli-plugin-cdb-reference[Cloud Databases CLI plug-in], the https://cloud.ibm.com/apidocs/cloud-databases-api[Cloud Databases API], or https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/database[Terraform]. + + +[discrete] +[[infer-service-watsonx-ai-api-request]] +==== {api-request-title} + +`PUT /_inference//` + +[discrete] +[[infer-service-watsonx-ai-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) +include::inference-shared.asciidoc[tag=inference-id] + +``:: +(Required, string) +include::inference-shared.asciidoc[tag=task-type] ++ +-- +Available task types: + +* `text_embedding`. +-- + +[discrete] +[[infer-service-watsonx-ai-api-request-body]] +==== {api-request-body-title} + +`service`:: +(Required, string) +The type of service supported for the specified task type. In this case, +`watsonxai`. + +`service_settings`:: +(Required, object) +include::inference-shared.asciidoc[tag=service-settings] ++ +-- +These settings are specific to the `watsonxai` service. +-- + +`api_key`::: +(Required, string) +A valid API key of your Watsonx account. +You can find your Watsonx API keys or you can create a new one https://cloud.ibm.com/iam/apikeys[on the API keys page]. ++ +-- +include::inference-shared.asciidoc[tag=api-key-admonition] +-- + +`api_version`::: +(Required, string) +Version parameter that takes a version date in the format of `YYYY-MM-DD`. +For the active version data parameters, refer to the https://cloud.ibm.com/apidocs/watsonx-ai#active-version-dates[documentation]. + +`model_id`::: +(Required, string) +The name of the model to use for the {infer} task. +Refer to the IBM Embedding Models section in the https://www.ibm.com/products/watsonx-ai/foundation-models[Watsonx documentation] for the list of available text embedding models. + +`url`::: +(Required, string) +The URL endpoint to use for the requests. + +`project_id`::: +(Required, string) +The name of the project to use for the {infer} task. + +`rate_limit`::: +(Optional, object) +By default, the `watsonxai` service sets the number of requests allowed per minute to `120`. +This helps to minimize the number of rate limit errors returned from Watsonx. +To modify this, set the `requests_per_minute` setting of this object in your service settings: ++ +-- +include::inference-shared.asciidoc[tag=request-per-minute-example] +-- + + +[discrete] +[[inference-example-watsonx-ai]] +==== Watsonx AI service example + +The following example shows how to create an {infer} endpoint called `watsonx-embeddings` to perform a `text_embedding` task type. + +[source,console] +------------------------------------------------------------ +PUT _inference/text_embedding/watsonx-embeddings +{ + "service": "watsonxai", + "service_settings": { + "api_key": "", <1> + "url": "", <2> + "model_id": "ibm/slate-30m-english-rtrvr", + "project_id": "", <3> + "api_version": "2024-03-14" <4> + } +} + +------------------------------------------------------------ +// TEST[skip:TBD] +<1> A valid Watsonx API key. +You can find on the https://cloud.ibm.com/iam/apikeys[API keys page of your account]. +<2> The {infer} endpoint URL you created on Watsonx. +<3> The ID of your IBM Cloud project. +<4> A valid API version parameter. You can find the active version data parameters https://cloud.ibm.com/apidocs/watsonx-ai#active-version-dates[here]. \ No newline at end of file diff --git a/docs/reference/inference/stream-inference.asciidoc b/docs/reference/inference/stream-inference.asciidoc new file mode 100644 index 0000000000000..42abb589f9afd --- /dev/null +++ b/docs/reference/inference/stream-inference.asciidoc @@ -0,0 +1,128 @@ +[role="xpack"] +[[stream-inference-api]] +=== Stream inference API + +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs]. +-- + +Streams a chat completion response. + +IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face. +For built-in models and models uploaded through Eland, the {infer} APIs offer an alternative way to use and manage trained models. +However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the <>. + + +[discrete] +[[stream-inference-api-request]] +==== {api-request-title} + +`POST /_inference//_stream` + +`POST /_inference///_stream` + + +[discrete] +[[stream-inference-api-prereqs]] +==== {api-prereq-title} + +* Requires the `monitor_inference` <> +(the built-in `inference_admin` and `inference_user` roles grant this privilege) +* You must use a client that supports streaming. + + +[discrete] +[[stream-inference-api-desc]] +==== {api-description-title} + +The stream {infer} API enables real-time responses for completion tasks by delivering answers incrementally, reducing response times during computation. +It only works with the `completion` task type. + + +[discrete] +[[stream-inference-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) +The unique identifier of the {infer} endpoint. + + +``:: +(Optional, string) +The type of {infer} task that the model performs. + + +[discrete] +[[stream-inference-api-request-body]] +==== {api-request-body-title} + +`input`:: +(Required, string or array of strings) +The text on which you want to perform the {infer} task. +`input` can be a single string or an array. ++ +-- +[NOTE] +==== +Inference endpoints for the `completion` task type currently only support a +single string as input. +==== +-- + + +[discrete] +[[stream-inference-api-example]] +==== {api-examples-title} + +The following example performs a completion on the example question with streaming. + + +[source,console] +------------------------------------------------------------ +POST _inference/completion/openai-completion/_stream +{ + "input": "What is Elastic?" +} +------------------------------------------------------------ +// TEST[skip:TBD] + + +The API returns the following response: + + +[source,txt] +------------------------------------------------------------ +event: message +data: { + "completion":[{ + "delta":"Elastic" + }] +} + +event: message +data: { + "completion":[{ + "delta":" is" + }, + { + "delta":" a" + } + ] +} + +event: message +data: { + "completion":[{ + "delta":" software" + }, + { + "delta":" company" + }] +} + +(...) +------------------------------------------------------------ +// NOTCONSOLE diff --git a/docs/reference/inference/update-inference.asciidoc b/docs/reference/inference/update-inference.asciidoc new file mode 100644 index 0000000000000..d3a90f5d84e65 --- /dev/null +++ b/docs/reference/inference/update-inference.asciidoc @@ -0,0 +1,91 @@ +[role="xpack"] +[[update-inference-api]] +=== Update inference API + +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-inference[{infer-cap} APIs]. +-- + +Updates an {infer} endpoint. + +IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face. +For built-in models and models uploaded through Eland, the {infer} APIs offer an alternative way to use and manage trained models. +However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the <>. + + +[discrete] +[[update-inference-api-request]] +==== {api-request-title} + +`POST _inference//_update` + +`POST _inference///_update` + + +[discrete] +[[update-inference-api-prereqs]] +==== {api-prereq-title} + +* Requires the `manage_inference` <> (the built-in inference_admin role grants this privilege) +* Requires an existing {infer} endpoint, created by using the <> + + +[discrete] +[[update-inference-api-desc]] +==== {api-description-title} + +The update inference API enables you to update the task_settings, secrets, and/or num_allocations of an existing {infer} endpoint. + +To use the update API, you can modify `task_settings`, secrets (within `service_settings`), or `num_allocations`, depending on the specific endpoint service and task_type you've created. +To view the updatable `task_settings`, the field names of secrets (specific to each service), and the services where `num_allocations` is applicable (only for the `elasticsearch` service), refer to the following list of services available through the {infer} API. +You will find the available task types next to each service name. +Click the links to review the service configuration details: + +* <> (`completion`, `rerank`, `sparse_embedding`, `text_embedding`) +* <> (`completion`, `text_embedding`) +* <> (`completion`) +* <> (`completion`, `text_embedding`) +* <> (`completion`, `text_embedding`) +* <> (`completion`, `rerank`, `text_embedding`) +* <> (`rerank`, `sparse_embedding`, `text_embedding` - this service is for built-in models and models uploaded through Eland) +* <> (`sparse_embedding`) +* <> (`completion`, `text_embedding`) +* <> (`rerank`, `text_embedding`) +* <> (`text_embedding`) +* <> (`text_embedding`) +* <> (`completion`, `text_embedding`) + + +[discrete] +[[update-inference-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) +The unique identifier of the {infer} endpoint. + + +``:: +(Optional, string) +The type of {infer} task that the model performs. +Refer to the service list in the <> for the available task types. + + +[discrete] +[[update-inference-api-example]] +==== {api-examples-title} + +The following example shows how to update an API key of an {infer} endpoint called `my-inference-endpoint`: + +[source,console] +------------------------------------------------------------ +POST _inference/my-inference-endpoint/_update +{ + "service_settings": { + "api_key": "" + } +} +------------------------------------------------------------ +// TEST[skip:TBD] diff --git a/docs/reference/ingest.asciidoc b/docs/reference/ingest.asciidoc index e2b4cf6fa10db..8b63f9acc5a21 100644 --- a/docs/reference/ingest.asciidoc +++ b/docs/reference/ingest.asciidoc @@ -450,7 +450,7 @@ configuration. See {fleet-guide}/install-standalone-elastic-agent.html[Install s [[pipelines-in-enterprise-search]] === Pipelines for search indices -When you create Elasticsearch indices for search use cases, for example, using the {enterprise-search-ref}/crawler.html[web crawler^] or {enterprise-search-ref}/connectors.html[connectors^], these indices are automatically set up with specific ingest pipelines. +When you create Elasticsearch indices for search use cases, for example, using the {enterprise-search-ref}/crawler.html[web crawler^] or <>, these indices are automatically set up with specific ingest pipelines. These processors help optimize your content for search. See <> for more information. diff --git a/docs/reference/ingest/apis/delete-geoip-database.asciidoc b/docs/reference/ingest/apis/delete-geoip-database.asciidoc deleted file mode 100644 index 957e59f0f0de4..0000000000000 --- a/docs/reference/ingest/apis/delete-geoip-database.asciidoc +++ /dev/null @@ -1,55 +0,0 @@ -[[delete-geoip-database-api]] -=== Delete geoip database configuration API -++++ -Delete geoip database configuration -++++ - -Deletes a geoip database configuration. - -[source,console] ----- -DELETE /_ingest/geoip/database/my-database-id ----- -// TEST[skip:we don't want to leak the enterprise-geoip-downloader task, which touching these APIs would cause. Therefore, skip this test.] - - -[[delete-geoip-database-api-request]] -==== {api-request-title} - -`DELETE /_ingest/geoip/database/` - -[[delete-geoip-database-api-prereqs]] -==== {api-prereq-title} - -* If the {es} {security-features} are enabled, you must have the -`manage` <> to use this API. - -[[delete-geoip-database-api-path-params]] -==== {api-path-parms-title} - -``:: -+ --- -(Required, string) Database configuration ID used to limit the request. - --- - - -[[delete-geoip-database-api-query-params]] -==== {api-query-parms-title} - -include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] - - -[[delete-geoip-database-api-example]] -==== {api-examples-title} - - -[[delete-geoip-database-api-specific-ex]] -===== Delete a specific geoip database configuration - -[source,console] ----- -DELETE /_ingest/geoip/database/example-database-id ----- -// TEST[skip:we don't want to leak the enterprise-geoip-downloader task, which touching these APIs would cause. Therefore, skip this test.] diff --git a/docs/reference/ingest/apis/delete-ip-location-database.asciidoc b/docs/reference/ingest/apis/delete-ip-location-database.asciidoc new file mode 100644 index 0000000000000..23f1d15bfa7ee --- /dev/null +++ b/docs/reference/ingest/apis/delete-ip-location-database.asciidoc @@ -0,0 +1,61 @@ +[[delete-ip-location-database-api]] +=== Delete IP geolocation database configuration API +++++ +Delete IP geolocation database configuration +++++ + +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ingest[Ingest APIs]. +-- + +Deletes a IP geolocation database configuration. + +[source,console] +---- +DELETE /_ingest/ip_location/database/my-database-id +---- +// TEST[skip:we don't want to leak the enterprise-geoip-downloader task, which touching these APIs would cause. Therefore, skip this test.] + + +[[delete-ip-location-database-api-request]] +==== {api-request-title} + +`DELETE /_ingest/ip_location/database/` + +[[delete-ip-location-database-api-prereqs]] +==== {api-prereq-title} + +* If the {es} {security-features} are enabled, you must have the +`manage` <> to use this API. + +[[delete-ip-location-database-api-path-params]] +==== {api-path-parms-title} + +``:: ++ +-- +(Required, string) Database configuration ID used to limit the request. + +-- + + +[[delete-ip-location-database-api-query-params]] +==== {api-query-parms-title} + +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] + + +[[delete-ip-location-database-api-example]] +==== {api-examples-title} + + +[[delete-ip-location-database-api-specific-ex]] +===== Delete a specific IP geolocation database configuration + +[source,console] +---- +DELETE /_ingest/ip_location/database/example-database-id +---- +// TEST[skip:we don't want to leak the enterprise-geoip-downloader task, which touching these APIs would cause. Therefore, skip this test.] diff --git a/docs/reference/ingest/apis/delete-pipeline.asciidoc b/docs/reference/ingest/apis/delete-pipeline.asciidoc index 94ac87c61b56b..cd4cae1b636ca 100644 --- a/docs/reference/ingest/apis/delete-pipeline.asciidoc +++ b/docs/reference/ingest/apis/delete-pipeline.asciidoc @@ -4,6 +4,12 @@ Delete pipeline ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ingest[Ingest APIs]. +-- + Deletes one or more existing ingest pipeline. //// diff --git a/docs/reference/ingest/apis/enrich/delete-enrich-policy.asciidoc b/docs/reference/ingest/apis/enrich/delete-enrich-policy.asciidoc index 1d4b7e770325a..cd68bd1fcc67b 100644 --- a/docs/reference/ingest/apis/enrich/delete-enrich-policy.asciidoc +++ b/docs/reference/ingest/apis/enrich/delete-enrich-policy.asciidoc @@ -5,6 +5,12 @@ Delete enrich policy ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-enrich[Enrich APIs]. +-- + Deletes an existing <> and its <>. diff --git a/docs/reference/ingest/apis/enrich/execute-enrich-policy.asciidoc b/docs/reference/ingest/apis/enrich/execute-enrich-policy.asciidoc index ebad9f09250d3..8e72a51514a59 100644 --- a/docs/reference/ingest/apis/enrich/execute-enrich-policy.asciidoc +++ b/docs/reference/ingest/apis/enrich/execute-enrich-policy.asciidoc @@ -5,6 +5,12 @@ Execute enrich policy ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-enrich[Enrich APIs]. +-- + Executes an existing <>. //// diff --git a/docs/reference/ingest/apis/enrich/get-enrich-policy.asciidoc b/docs/reference/ingest/apis/enrich/get-enrich-policy.asciidoc index 88cf2cfa7cf60..2f9e069d5e43d 100644 --- a/docs/reference/ingest/apis/enrich/get-enrich-policy.asciidoc +++ b/docs/reference/ingest/apis/enrich/get-enrich-policy.asciidoc @@ -5,6 +5,12 @@ Get enrich policy ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-enrich[Enrich APIs]. +-- + Returns information about an <>. //// diff --git a/docs/reference/ingest/apis/enrich/index.asciidoc b/docs/reference/ingest/apis/enrich/index.asciidoc index a17c8179af1b1..7566b8812ec3c 100644 --- a/docs/reference/ingest/apis/enrich/index.asciidoc +++ b/docs/reference/ingest/apis/enrich/index.asciidoc @@ -2,6 +2,12 @@ [[enrich-apis]] == Enrich APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-enrich[Enrich APIs]. +-- + The following enrich APIs are available for managing <>: diff --git a/docs/reference/ingest/apis/enrich/put-enrich-policy.asciidoc b/docs/reference/ingest/apis/enrich/put-enrich-policy.asciidoc index ee33b0b320905..80223cb0ec711 100644 --- a/docs/reference/ingest/apis/enrich/put-enrich-policy.asciidoc +++ b/docs/reference/ingest/apis/enrich/put-enrich-policy.asciidoc @@ -5,6 +5,11 @@ Create enrich policy ++++ +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-enrich[Enrich APIs]. +-- + Creates an enrich policy. //// diff --git a/docs/reference/ingest/apis/geoip-stats.asciidoc b/docs/reference/ingest/apis/geoip-stats.asciidoc index 6135b7821f2a5..423fee1c5bceb 100644 --- a/docs/reference/ingest/apis/geoip-stats.asciidoc +++ b/docs/reference/ingest/apis/geoip-stats.asciidoc @@ -4,6 +4,12 @@ GeoIP stats ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ingest[Ingest APIs]. +-- + Gets statistics about the <>, including download statistics for GeoIP2 databases used with it. diff --git a/docs/reference/ingest/apis/get-geoip-database.asciidoc b/docs/reference/ingest/apis/get-geoip-database.asciidoc deleted file mode 100644 index f055e3e759db8..0000000000000 --- a/docs/reference/ingest/apis/get-geoip-database.asciidoc +++ /dev/null @@ -1,80 +0,0 @@ -[[get-geoip-database-api]] -=== Get geoip database configuration API -++++ -Get geoip database configuration -++++ - -Returns information about one or more geoip database configurations. - -[source,console] ----- -GET /_ingest/geoip/database/my-database-id ----- -// TEST[skip:we don't want to leak the enterprise-geoip-downloader task, which touching these APIs would cause. Therefore, skip this test.] - - - -[[get-geoip-database-api-request]] -==== {api-request-title} - -`GET /_ingest/geoip/database/` - -`GET /_ingest/geoip/database` - -[[get-geoip-database-api-prereqs]] -==== {api-prereq-title} - -* If the {es} {security-features} are enabled, you must have the - `manage` <> to use this API. - -[[get-geoip-database-api-path-params]] -==== {api-path-parms-title} - -``:: -(Optional, string) -Comma-separated list of database configuration IDs to retrieve. Wildcard (`*`) expressions are -supported. -+ -To get all database configurations, omit this parameter or use `*`. - - -[[get-geoip-database-api-query-params]] -==== {api-query-parms-title} - -include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] - - -[[get-geoip-database-api-example]] -==== {api-examples-title} - - -[[get-geoip-database-api-specific-ex]] -===== Get information for a specific geoip database configuration - -[source,console] ----- -GET /_ingest/geoip/database/my-database-id ----- -// TEST[skip:we don't want to leak the enterprise-geoip-downloader task, which touching these APIs would cause. Therefore, skip this test.] - -The API returns the following response: - -[source,console-result] ----- -{ - "databases" : [ - { - "id" : "my-database-id", - "version" : 1, - "modified_date_millis" : 1723040276114, - "database" : { - "name" : "GeoIP2-Domain", - "maxmind" : { - "account_id" : "1234567" - } - } - } - ] -} ----- -// TEST[skip:we don't want to leak the enterprise-geoip-downloader task, which touching these APIs would cause. Therefore, skip this test.] diff --git a/docs/reference/ingest/apis/get-ip-location-database.asciidoc b/docs/reference/ingest/apis/get-ip-location-database.asciidoc new file mode 100644 index 0000000000000..25f1c7ffdf695 --- /dev/null +++ b/docs/reference/ingest/apis/get-ip-location-database.asciidoc @@ -0,0 +1,86 @@ +[[get-ip-location-database-api]] +=== Get IP geolocation database configuration API +++++ +Get IP geolocation database configuration +++++ + +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ingest[Ingest APIs]. +-- + +Returns information about one or more IP geolocation database configurations. + +[source,console] +---- +GET /_ingest/ip_location/database/my-database-id +---- +// TEST[skip:we don't want to leak the enterprise-geoip-downloader task, which touching these APIs would cause. Therefore, skip this test.] + + + +[[get-ip-location-database-api-request]] +==== {api-request-title} + +`GET /_ingest/ip_location/database/` + +`GET /_ingest/ip_location/database` + +[[get-ip-location-database-api-prereqs]] +==== {api-prereq-title} + +* If the {es} {security-features} are enabled, you must have the + `manage` <> to use this API. + +[[get-ip-location-database-api-path-params]] +==== {api-path-parms-title} + +``:: +(Optional, string) +Comma-separated list of database configuration IDs to retrieve. Wildcard (`*`) expressions are +supported. ++ +To get all database configurations, omit this parameter or use `*`. + + +[[get-ip-location-database-api-query-params]] +==== {api-query-parms-title} + +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] + + +[[get-ip-location-database-api-example]] +==== {api-examples-title} + + +[[get-ip-location-database-api-specific-ex]] +===== Get information for a specific IP geolocation database configuration + +[source,console] +---- +GET /_ingest/ip_location/database/my-database-id +---- +// TEST[skip:we don't want to leak the enterprise-geoip-downloader task, which touching these APIs would cause. Therefore, skip this test.] + +The API returns the following response: + +[source,console-result] +---- +{ + "databases" : [ + { + "id" : "my-database-id", + "version" : 1, + "modified_date_millis" : 1723040276114, + "database" : { + "name" : "GeoIP2-Domain", + "maxmind" : { + "account_id" : "1234567" + } + } + } + ] +} +---- +// TEST[skip:we don't want to leak the enterprise-geoip-downloader task, which touching these APIs would cause. Therefore, skip this test.] diff --git a/docs/reference/ingest/apis/get-pipeline.asciidoc b/docs/reference/ingest/apis/get-pipeline.asciidoc index f2a1155bca12b..88fc22590004b 100644 --- a/docs/reference/ingest/apis/get-pipeline.asciidoc +++ b/docs/reference/ingest/apis/get-pipeline.asciidoc @@ -4,6 +4,12 @@ Get pipeline ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ingest[Ingest APIs]. +-- + Returns information about one or more ingest pipelines. This API returns a local reference of the pipeline. diff --git a/docs/reference/ingest/apis/index.asciidoc b/docs/reference/ingest/apis/index.asciidoc index e068f99ea0ad3..e83131b554452 100644 --- a/docs/reference/ingest/apis/index.asciidoc +++ b/docs/reference/ingest/apis/index.asciidoc @@ -1,6 +1,12 @@ [[ingest-apis]] == Ingest APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ingest[Ingest APIs]. +-- + Use ingest APIs to manage tasks and resources related to <> and processors. @@ -25,16 +31,14 @@ Use the following APIs to get statistics about ingest processing: the <>. [discrete] -[[ingest-geoip-database-apis]] -=== Ingest GeoIP Database APIs - -preview::["The commercial IP geolocation database download management APIs are in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but this feature is not subject to the support SLA of official GA features."] +[[ingest-ip-location-database-apis]] +=== Ingest IP Location Database APIs Use the following APIs to configure and manage commercial IP geolocation database downloads: -* <> to create or update a database configuration -* <> to retrieve a database configuration -* <> to delete a database configuration +* <> to create or update a database configuration +* <> to retrieve a database configuration +* <> to delete a database configuration include::put-pipeline.asciidoc[] include::get-pipeline.asciidoc[] @@ -42,6 +46,6 @@ include::delete-pipeline.asciidoc[] include::simulate-pipeline.asciidoc[] include::simulate-ingest.asciidoc[] include::geoip-stats.asciidoc[] -include::put-geoip-database.asciidoc[] -include::get-geoip-database.asciidoc[] -include::delete-geoip-database.asciidoc[] +include::put-ip-location-database.asciidoc[] +include::get-ip-location-database.asciidoc[] +include::delete-ip-location-database.asciidoc[] diff --git a/docs/reference/ingest/apis/put-geoip-database.asciidoc b/docs/reference/ingest/apis/put-geoip-database.asciidoc deleted file mode 100644 index 311c303002387..0000000000000 --- a/docs/reference/ingest/apis/put-geoip-database.asciidoc +++ /dev/null @@ -1,72 +0,0 @@ -[[put-geoip-database-api]] -=== Create or update geoip database configuration API -++++ -Create or update geoip database configuration -++++ - -Creates or updates an IP geolocation database configuration. - -IMPORTANT: The Maxmind `account_id` shown below requires a license key. Because the license key is sensitive information, -it is stored as a <> in {es} named `ingest.geoip.downloader.maxmind.license_key`. Only -one Maxmind license key is currently allowed per {es} cluster. A valid license key must be in the secure settings in order -to download from Maxmind. The license key setting does not take effect until all nodes are restarted. - -[source,console] ----- -PUT _ingest/geoip/database/my-database-id -{ - "name": "GeoIP2-Domain", - "maxmind": { - "account_id": "1025402" - } -} ----- -// TEST[skip:we don't want to leak the enterprise-geoip-downloader task, which touching these APIs would cause. Therefore, skip this test.] - -[[put-geoip-database-api-request]] -==== {api-request-title} - -`PUT /_ingest/geoip/database/` - -[[put-geoip-database-api-prereqs]] -==== {api-prereq-title} - -* If the {es} {security-features} are enabled, you must have the -`manage` <> to use this API. - - -[[put-geoip-database-api-path-params]] -==== {api-path-parms-title} - -``:: -+ -__ -(Required, string) ID of the database configuration to create or update. - -[[put-geoip-database-api-query-params]] -==== {api-query-parms-title} - -include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] - -[[put-geoip-database-api-request-body]] -==== {api-request-body-title} - -// tag::geoip-database-object[] -`name`:: -(Required, string) -The provider-assigned name of the IP geolocation database to download. - -``:: -(Required, a provider object and its associated configuration) -The configuration necessary to identify which IP geolocation provider to use to download -the database, as well as any provider-specific configuration necessary for such downloading. -+ -At present, the only supported provider is `maxmind`, and the maxmind provider -requires that an `account_id` (string) is configured. -// end::geoip-database-object[] - -[[geoip-database-configuration-licensing]] -==== Licensing - -Downloading databases from third party providers is a commercial feature that requires an -appropriate license. For more information, refer to https://www.elastic.co/subscriptions. diff --git a/docs/reference/ingest/apis/put-ip-location-database.asciidoc b/docs/reference/ingest/apis/put-ip-location-database.asciidoc new file mode 100644 index 0000000000000..1cdd789008881 --- /dev/null +++ b/docs/reference/ingest/apis/put-ip-location-database.asciidoc @@ -0,0 +1,98 @@ +[[put-ip-location-database-api]] +=== Create or update IP geolocation database configuration API +++++ +Create or update IP geolocation database configuration +++++ + +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ingest[Ingest APIs]. +-- + +Creates or updates an IP geolocation database configuration. + +IMPORTANT: The Maxmind `account_id` shown below requires a license key. Because the license key is sensitive information, +it is stored as a <> in {es} named `ingest.geoip.downloader.maxmind.license_key`. Only +one Maxmind license key is currently allowed per {es} cluster. A valid license key must be in the secure settings in order +to download from Maxmind. The license key setting does not take effect until all nodes are restarted or a +<> request is executed. + +[source,console] +---- +PUT _ingest/ip_location/database/my-database-1 +{ + "name": "GeoIP2-Domain", + "maxmind": { + "account_id": "1234567" + } +} +---- +// TEST[skip:we don't want to leak the enterprise-geoip-downloader task, which touching these APIs would cause. Therefore, skip this test.] + +IMPORTANT: The IPinfo configuration shown below requires a token. Because the token is sensitive information, +it is stored as a <> in {es} named `ingest.ip_location.downloader.ipinfo.token`. Only +one IPinfo token is currently allowed per {es} cluster. A valid token must be in the secure settings in order +to download from IPinfo. The token setting does not take effect until all nodes are restarted or a +<> request is executed. + +[source,console] +---- +PUT _ingest/ip_location/database/my-database-2 +{ + "name": "standard_location", + "ipinfo": { + } +} +---- +// TEST[skip:we don't want to leak the enterprise-geoip-downloader task, which touching these APIs would cause. Therefore, skip this test.] + + +[[put-ip-location-database-api-request]] +==== {api-request-title} + +`PUT /_ingest/ip_location/database/` + +[[put-ip-location-database-api-prereqs]] +==== {api-prereq-title} + +* If the {es} {security-features} are enabled, you must have the +`manage` <> to use this API. + + +[[put-ip-location-database-api-path-params]] +==== {api-path-parms-title} + +``:: ++ +__ +(Required, string) ID of the database configuration to create or update. + +[[put-ip-location-database-api-query-params]] +==== {api-query-parms-title} + +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] + +[[put-ip-location-database-api-request-body]] +==== {api-request-body-title} + +// tag::ip-location-database-object[] +`name`:: +(Required, string) +The provider-assigned name of the IP geolocation database to download. + +``:: +(Required, a provider object and its associated configuration) +The configuration necessary to identify which IP geolocation provider to use to download +the database, as well as any provider-specific configuration necessary for such downloading. ++ +At present, the only supported providers are `maxmind` and `ipinfo`. The maxmind provider +requires that an `account_id` (string) is configured. The ipinfo provider does not require +additional configuration in the request body. +// end::ip-location-database-object[] + +[[ip-location-database-configuration-licensing]] +==== Licensing + +Downloading databases from third party providers is a commercial feature that requires an +appropriate license. For more information, refer to https://www.elastic.co/subscriptions. diff --git a/docs/reference/ingest/apis/put-pipeline.asciidoc b/docs/reference/ingest/apis/put-pipeline.asciidoc index 5b532dedf8e82..7d9854a55758d 100644 --- a/docs/reference/ingest/apis/put-pipeline.asciidoc +++ b/docs/reference/ingest/apis/put-pipeline.asciidoc @@ -4,6 +4,12 @@ Create or update pipeline ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ingest[Ingest APIs]. +-- + Creates or updates an <>. Changes made using this API take effect immediately. diff --git a/docs/reference/ingest/apis/simulate-ingest.asciidoc b/docs/reference/ingest/apis/simulate-ingest.asciidoc index ee84a39ee6f65..a9c7e43e041bc 100644 --- a/docs/reference/ingest/apis/simulate-ingest.asciidoc +++ b/docs/reference/ingest/apis/simulate-ingest.asciidoc @@ -5,6 +5,12 @@ Simulate ingest ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ingest[Ingest APIs]. +-- + Executes ingest pipelines against a set of provided documents, optionally with substitute pipeline definitions. This API is meant to be used for troubleshooting or pipeline development, as it does not actually index any @@ -83,11 +89,49 @@ POST /_ingest/_simulate } ] } + }, + "component_template_substitutions": { <2> + "my-component-template": { + "template": { + "mappings": { + "dynamic": "true", + "properties": { + "field3": { + "type": "keyword" + } + } + }, + "settings": { + "index": { + "default_pipeline": "my-pipeline" + } + } + } + } + }, + "index_template_substitutions": { <3> + "my-index-template": { + "index_patterns": ["my-index-*"], + "composed_of": ["component_template_1", "component_template_2"] + } + }, + "mapping_addition": { <4> + "dynamic": "strict", + "properties": { + "foo": { + "type": "keyword" + } + } } } ---- <1> This replaces the existing `my-pipeline` pipeline with the contents given here for the duration of this request. +<2> This replaces the existing `my-component-template` component template with the contents given here for the duration of this request. +These templates can be used to change the pipeline(s) used, or to modify the mapping that will be used to validate the result. +<3> This replaces the existing `my-index-template` index template with the contents given here for the duration of this request. +These templates can be used to change the pipeline(s) used, or to modify the mapping that will be used to validate the result. +<4> This mapping is merged into the index's final mapping just before validation. It is used only for the duration of this request. [[simulate-ingest-api-request]] ==== {api-request-title} @@ -191,6 +235,36 @@ Map of pipeline IDs to substitute pipeline definition objects. include::put-pipeline.asciidoc[tag=pipeline-object] ==== +`component_template_substitutions`:: +(Optional, map of strings to objects) +Map of component template names to substitute component template definition objects. ++ +.Properties of component template definition objects +[%collapsible%open] + +==== + +include::{es-ref-dir}/indices/put-component-template.asciidoc[tag=template] + +==== + +`index_template_substitutions`:: +(Optional, map of strings to objects) +Map of index template names to substitute index template definition objects. ++ +.Properties of index template definition objects +[%collapsible%open] + +==== + +include::{es-ref-dir}/indices/put-index-template.asciidoc[tag=request-body] + +==== + +`mapping_addition`:: +(Optional, <>) +Definition of a mapping that will be merged into the index's mapping for validation during the course of this request. + [[simulate-ingest-api-example]] ==== {api-examples-title} @@ -268,7 +342,7 @@ The API returns the following response: [[simulate-ingest-api-request-body-ex]] ===== Specify a pipeline substitution in the request body -In this example the index `index` has a default pipeline called `my-pipeline` and a final +In this example the index `my-index` has a default pipeline called `my-pipeline` and a final pipeline called `my-final-pipeline`. But a substitute definition of `my-pipeline` is provided in `pipeline_substitutions`. The substitute `my-pipeline` will be used in place of the `my-pipeline` that is in the system, and then the `my-final-pipeline` that is already @@ -348,6 +422,87 @@ The API returns the following response: } ---- +[[simulate-ingest-api-substitute-component-templates-ex]] +===== Specify a component template substitution in the request body +In this example, imagine that the index `my-index` has a strict mapping with only the `foo` +keyword field defined. Say that field mapping came from a component template named +`my-mappings-template`. We want to test adding a new field, `bar`. So a substitute definition of +`my-mappings-template` is provided in `component_template_substitutions`. The substitute +`my-mappings-template` will be used in place of the existing mapping for `my-index` and in place +of the `my-mappings-template` that is in the system. + +[source,console] +---- +POST /_ingest/_simulate +{ + "docs": [ + { + "_index": "my-index", + "_id": "123", + "_source": { + "foo": "foo" + } + }, + { + "_index": "my-index", + "_id": "456", + "_source": { + "bar": "rab" + } + } + ], + "component_template_substitutions": { + "my-mappings_template": { + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "keyword" + }, + "bar": { + "type": "keyword" + } + } + } + } + } + } +} +---- + +The API returns the following response: + +[source,console-result] +---- +{ + "docs": [ + { + "doc": { + "_id": "123", + "_index": "my-index", + "_version": -3, + "_source": { + "foo": "foo" + }, + "executed_pipelines": [] + } + }, + { + "doc": { + "_id": "456", + "_index": "my-index", + "_version": -3, + "_source": { + "bar": "rab" + }, + "executed_pipelines": [] + } + } + ] +} +---- + //// [source,console] ---- diff --git a/docs/reference/ingest/apis/simulate-pipeline.asciidoc b/docs/reference/ingest/apis/simulate-pipeline.asciidoc index bb7c343383132..fe7e5f2e91c6a 100644 --- a/docs/reference/ingest/apis/simulate-pipeline.asciidoc +++ b/docs/reference/ingest/apis/simulate-pipeline.asciidoc @@ -5,6 +5,12 @@ Simulate pipeline ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ingest[Ingest APIs]. +-- + Executes an ingest pipeline against a set of provided documents. diff --git a/docs/reference/ingest/common-log-format-example.asciidoc b/docs/reference/ingest/common-log-format-example.asciidoc index 3fd296a167a6a..be20b49cf39f7 100644 --- a/docs/reference/ingest/common-log-format-example.asciidoc +++ b/docs/reference/ingest/common-log-format-example.asciidoc @@ -28,7 +28,7 @@ Pipelines**. image::images/ingest/ingest-pipeline-list.png[Kibana's Ingest Pipelines list view,align="center"] . Click **Create pipeline > New pipeline**. -. Set **Name** to `my-pipeline` and optionally add a description for the +. Set **Name** to `my-pipeline` and optionally add a description for the pipeline. . Add a <> to parse the log message: @@ -128,6 +128,7 @@ PUT _ingest/pipeline/my-pipeline // end::common-log-pipeline[] } ---- +// TEST[skip:This can output a warning, and asciidoc doesn't support allowed_warnings] //// -- diff --git a/docs/reference/ingest/processors.asciidoc b/docs/reference/ingest/processors.asciidoc index 8622e0b98602c..f4fcc0fc84d0d 100644 --- a/docs/reference/ingest/processors.asciidoc +++ b/docs/reference/ingest/processors.asciidoc @@ -77,7 +77,10 @@ Computes a hash of the document’s content. Converts geo-grid definitions of grid tiles or cells to regular bounding boxes or polygons which describe their shape. <>:: -Adds information about the geographical location of an IPv4 or IPv6 address. +Adds information about the geographical location of an IPv4 or IPv6 address from a Maxmind database. + +<>:: +Adds information about the geographical location of an IPv4 or IPv6 address from an ip geolocation database. <>:: Calculates the network direction given a source IP address, destination IP address, and a list of internal networks. @@ -185,6 +188,9 @@ Executes another pipeline. <>:: Reroutes documents to another target index or data stream. +<>:: +Terminates the current ingest pipeline, causing no further processors to be run. + [discrete] [[ingest-process-category-array-json-handling]] === Array/JSON handling processors @@ -242,6 +248,7 @@ include::processors/grok.asciidoc[] include::processors/gsub.asciidoc[] include::processors/html_strip.asciidoc[] include::processors/inference.asciidoc[] +include::processors/ip-location.asciidoc[] include::processors/join.asciidoc[] include::processors/json.asciidoc[] include::processors/kv.asciidoc[] @@ -258,6 +265,7 @@ include::processors/set.asciidoc[] include::processors/set-security-user.asciidoc[] include::processors/sort.asciidoc[] include::processors/split.asciidoc[] +include::processors/terminate.asciidoc[] include::processors/trim.asciidoc[] include::processors/uppercase.asciidoc[] include::processors/url-decode.asciidoc[] diff --git a/docs/reference/ingest/processors/attachment.asciidoc b/docs/reference/ingest/processors/attachment.asciidoc index fd2866906c1d0..bd5b8db562ae2 100644 --- a/docs/reference/ingest/processors/attachment.asciidoc +++ b/docs/reference/ingest/processors/attachment.asciidoc @@ -19,15 +19,15 @@ representation. The processor will skip the base64 decoding then. .Attachment options [options="header"] |====== -| Name | Required | Default | Description -| `field` | yes | - | The field to get the base64 encoded field from -| `target_field` | no | attachment | The field that will hold the attachment information -| `indexed_chars` | no | 100000 | The number of chars being used for extraction to prevent huge fields. Use `-1` for no limit. -| `indexed_chars_field` | no | `null` | Field name from which you can overwrite the number of chars being used for extraction. See `indexed_chars`. -| `properties` | no | all properties | Array of properties to select to be stored. Can be `content`, `title`, `name`, `author`, `keywords`, `date`, `content_type`, `content_length`, `language` -| `ignore_missing` | no | `false` | If `true` and `field` does not exist, the processor quietly exits without modifying the document -| `remove_binary` | no | `false` | If `true`, the binary `field` will be removed from the document -| `resource_name` | no | | Field containing the name of the resource to decode. If specified, the processor passes this resource name to the underlying Tika library to enable https://tika.apache.org/1.24.1/detection.html#Resource_Name_Based_Detection[Resource Name Based Detection]. +| Name | Required | Default | Description +| `field` | yes | - | The field to get the base64 encoded field from +| `target_field` | no | attachment | The field that will hold the attachment information +| `indexed_chars` | no | 100000 | The number of chars being used for extraction to prevent huge fields. Use `-1` for no limit. +| `indexed_chars_field` | no | `null` | Field name from which you can overwrite the number of chars being used for extraction. See `indexed_chars`. +| `properties` | no | all properties | Array of properties to select to be stored. Can be `content`, `title`, `name`, `author`, `keywords`, `date`, `content_type`, `content_length`, `language` +| `ignore_missing` | no | `false` | If `true` and `field` does not exist, the processor quietly exits without modifying the document +| `remove_binary` | encouraged | `false` | If `true`, the binary `field` will be removed from the document. This option is not required, but setting it explicitly is encouraged, and omitting it will result in a warning. +| `resource_name` | no | | Field containing the name of the resource to decode. If specified, the processor passes this resource name to the underlying Tika library to enable https://tika.apache.org/1.24.1/detection.html#Resource_Name_Based_Detection[Resource Name Based Detection]. |====== [discrete] @@ -58,7 +58,7 @@ PUT _ingest/pipeline/attachment { "attachment" : { "field" : "data", - "remove_binary": false + "remove_binary": true } } ] @@ -82,7 +82,6 @@ The document's `attachment` object contains extracted properties for the file: "_seq_no": 22, "_primary_term": 1, "_source": { - "data": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0=", "attachment": { "content_type": "application/rtf", "language": "ro", @@ -94,9 +93,6 @@ The document's `attachment` object contains extracted properties for the file: ---- // TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/] -NOTE: Keeping the binary as a field within the document might consume a lot of resources. It is highly recommended - to remove that field from the document. Set `remove_binary` to `true` to automatically remove the field. - [[attachment-fields]] ==== Exported fields @@ -143,7 +139,7 @@ PUT _ingest/pipeline/attachment "attachment" : { "field" : "data", "properties": [ "content", "title" ], - "remove_binary": false + "remove_binary": true } } ] @@ -154,6 +150,59 @@ NOTE: Extracting contents from binary data is a resource intensive operation and consumes a lot of resources. It is highly recommended to run pipelines using this processor in a dedicated ingest node. +[[attachment-keep-binary]] +==== Keeping the attachment binary + +Keeping the binary as a field within the document might consume a lot of resources. It is highly recommended to remove +that field from the document, by setting `remove_binary` to `true` to automatically remove the field, as in the other +examples shown on this page. If you _do_ want to keep the binary field, explicitly set `remove_binary` to `false` to +avoid the warning you get from omitting it: + +[source,console] +---- +PUT _ingest/pipeline/attachment +{ + "description" : "Extract attachment information including original binary", + "processors" : [ + { + "attachment" : { + "field" : "data", + "remove_binary": false + } + } + ] +} +PUT my-index-000001/_doc/my_id?pipeline=attachment +{ + "data": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0=" +} +GET my-index-000001/_doc/my_id +---- + +The document's `_source` object includes the original binary field: + +[source,console-result] +---- +{ + "found": true, + "_index": "my-index-000001", + "_id": "my_id", + "_version": 1, + "_seq_no": 22, + "_primary_term": 1, + "_source": { + "data": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0=", + "attachment": { + "content_type": "application/rtf", + "language": "ro", + "content": "Lorem ipsum dolor sit amet", + "content_length": 28 + } + } +} +---- +// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/] + [[attachment-cbor]] ==== Use the attachment processor with CBOR @@ -170,7 +219,7 @@ PUT _ingest/pipeline/cbor-attachment { "attachment" : { "field" : "data", - "remove_binary": false + "remove_binary": true } } ] @@ -226,7 +275,7 @@ PUT _ingest/pipeline/attachment "field" : "data", "indexed_chars" : 11, "indexed_chars_field" : "max_size", - "remove_binary": false + "remove_binary": true } } ] @@ -250,7 +299,6 @@ Returns this: "_seq_no": 35, "_primary_term": 1, "_source": { - "data": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0=", "attachment": { "content_type": "application/rtf", "language": "is", @@ -274,7 +322,7 @@ PUT _ingest/pipeline/attachment "field" : "data", "indexed_chars" : 11, "indexed_chars_field" : "max_size", - "remove_binary": false + "remove_binary": true } } ] @@ -299,7 +347,6 @@ Returns this: "_seq_no": 40, "_primary_term": 1, "_source": { - "data": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0=", "max_size": 5, "attachment": { "content_type": "application/rtf", @@ -358,7 +405,7 @@ PUT _ingest/pipeline/attachment "attachment": { "target_field": "_ingest._value.attachment", "field": "_ingest._value.data", - "remove_binary": false + "remove_binary": true } } } @@ -396,7 +443,6 @@ Returns this: "attachments" : [ { "filename" : "ipsum.txt", - "data" : "dGhpcyBpcwpqdXN0IHNvbWUgdGV4dAo=", "attachment" : { "content_type" : "text/plain; charset=ISO-8859-1", "language" : "en", @@ -406,7 +452,6 @@ Returns this: }, { "filename" : "test.txt", - "data" : "VGhpcyBpcyBhIHRlc3QK", "attachment" : { "content_type" : "text/plain; charset=ISO-8859-1", "language" : "en", diff --git a/docs/reference/ingest/processors/date.asciidoc b/docs/reference/ingest/processors/date.asciidoc index fd045abfa0c95..7c65c8c1468b3 100644 --- a/docs/reference/ingest/processors/date.asciidoc +++ b/docs/reference/ingest/processors/date.asciidoc @@ -67,3 +67,11 @@ the timezone and locale values. } -------------------------------------------------- // NOTCONSOLE + +[WARNING] +==== +// tag::locale-warning[] +The text strings accepted by textual date formats, and calculations for week-dates, depend on the JDK version +that Elasticsearch is running on. For more information see <>. +// end::locale-warning[] +==== diff --git a/docs/reference/ingest/processors/geoip.asciidoc b/docs/reference/ingest/processors/geoip.asciidoc index 3a9ba58dedbf0..78ebe3f5b5ee3 100644 --- a/docs/reference/ingest/processors/geoip.asciidoc +++ b/docs/reference/ingest/processors/geoip.asciidoc @@ -13,7 +13,7 @@ ASN IP geolocation databases from http://dev.maxmind.com/geoip/geoip2/geolite2/[ CC BY-SA 4.0 license. It automatically downloads these databases if your nodes can connect to `storage.googleapis.com` domain and either: * `ingest.geoip.downloader.eager.download` is set to true -* your cluster has at least one pipeline with a `geoip` processor +* your cluster has at least one pipeline with a `geoip` or `ip_location` processor {es} automatically downloads updates for these databases from the Elastic GeoIP endpoint: @@ -25,10 +25,10 @@ If your cluster can't connect to the Elastic GeoIP endpoint or you want to manage your own updates, see <>. If you would like to have {es} download database files directly from Maxmind using your own provided -license key, see <>. +license key, see <>. If {es} can't connect to the endpoint for 30 days all updated databases will become -invalid. {es} will stop enriching documents with geoip data and will add `tags: ["_geoip_expired_database"]` +invalid. {es} will stop enriching documents with ip geolocation data and will add `tags: ["_geoip_expired_database"]` field instead. [[using-ingest-geoip]] @@ -40,21 +40,23 @@ field instead. |====== | Name | Required | Default | Description | `field` | yes | - | The field to get the IP address from for the geographical lookup. -| `target_field` | no | geoip | The field that will hold the geographical information looked up from the MaxMind database. -| `database_file` | no | GeoLite2-City.mmdb | The database filename referring to one of the automatically downloaded GeoLite2 databases (GeoLite2-City.mmdb, GeoLite2-Country.mmdb, or GeoLite2-ASN.mmdb), or the name of a supported database file in the `ingest-geoip` config directory, or the name of a <> (with the `.mmdb` suffix appended). -| `properties` | no | [`continent_name`, `country_iso_code`, `country_name`, `region_iso_code`, `region_name`, `city_name`, `location`] * | Controls what properties are added to the `target_field` based on the geoip lookup. +| `target_field` | no | geoip | The field that will hold the geographical information looked up from the database. +| `database_file` | no | GeoLite2-City.mmdb | The database filename referring to one of the automatically downloaded GeoLite2 databases (GeoLite2-City.mmdb, GeoLite2-Country.mmdb, or GeoLite2-ASN.mmdb), or the name of a supported database file in the `ingest-geoip` config directory, or the name of a <> (with the `.mmdb` suffix appended). +| `properties` | no | [`continent_name`, `country_iso_code`, `country_name`, `region_iso_code`, `region_name`, `city_name`, `location`] * | Controls what properties are added to the `target_field` based on the ip geolocation lookup. | `ignore_missing` | no | `false` | If `true` and `field` does not exist, the processor quietly exits without modifying the document -| `first_only` | no | `true` | If `true` only first found geoip data will be returned, even if `field` contains array +| `first_only` | no | `true` | If `true` only first found ip geolocation data, will be returned, even if `field` contains array | `download_database_on_pipeline_creation` | no | `true` | If `true` (and if `ingest.geoip.downloader.eager.download` is `false`), the missing database is downloaded when the pipeline is created. Else, the download is triggered by when the pipeline is used as the `default_pipeline` or `final_pipeline` in an index. |====== *Depends on what is available in `database_file`: * If a GeoLite2 City or GeoIP2 City database is used, then the following fields may be added under the `target_field`: `ip`, -`country_iso_code`, `country_name`, `continent_code`, `continent_name`, `region_iso_code`, `region_name`, `city_name`, `timezone`, -and `location`. The fields actually added depend on what has been found and which properties were configured in `properties`. +`country_iso_code`, `country_name`, `country_in_european_union`, `registered_country_iso_code`, `registered_country_name`, `registered_country_in_european_union`, +`continent_code`, `continent_name`, `region_iso_code`, `region_name`, `city_name`, `postal_code`, `timezone`, +`location`, and `accuracy_radius`. The fields actually added depend on what has been found and which properties were configured in `properties`. * If a GeoLite2 Country or GeoIP2 Country database is used, then the following fields may be added under the `target_field`: `ip`, -`country_iso_code`, `country_name`, `continent_code`, and `continent_name`. The fields actually added depend on what has been found +`country_iso_code`, `country_name`, `country_in_european_union`, `registered_country_iso_code`, `registered_country_name`, `registered_country_in_european_union`, +`continent_code`, and `continent_name`. The fields actually added depend on what has been found and which properties were configured in `properties`. * If the GeoLite2 ASN database is used, then the following fields may be added under the `target_field`: `ip`, `asn`, `organization_name` and `network`. The fields actually added depend on what has been found and which properties were configured @@ -70,12 +72,12 @@ The fields actually added depend on what has been found and which properties wer `organization_name`, `network`, `isp`, `isp_organization_name`, `mobile_country_code`, and `mobile_network_code`. The fields actually added depend on what has been found and which properties were configured in `properties`. * If the GeoIP2 Enterprise database is used, then the following fields may be added under the `target_field`: `ip`, -`country_iso_code`, `country_name`, `continent_code`, `continent_name`, `region_iso_code`, `region_name`, `city_name`, `timezone`, -`location`, `asn`, `organization_name`, `network`, `hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`, +`country_iso_code`, `country_name`, `country_in_european_union`, `registered_country_iso_code`, `registered_country_name`, `registered_country_in_european_union`, +`continent_code`, `continent_name`, `region_iso_code`, `region_name`, `city_name`, `postal_code`, `timezone`, +`location`, `accuracy_radius`, `country_confidence`, `city_confidence`, `postal_confidence`, `asn`, `organization_name`, `network`, +`hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`, `residential_proxy`, `domain`, `isp`, `isp_organization_name`, `mobile_country_code`, `mobile_network_code`, `user_type`, and -`connection_type`. The fields actually added depend on what has been found and which properties were configured in `properties`. - -preview::["Do not use the GeoIP2 Anonymous IP, GeoIP2 Connection Type, GeoIP2 Domain, GeoIP2 ISP, and GeoIP2 Enterprise databases in production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] +`connection_type`. The fields actually added depend on what has been found and which properties were configured in `properties`. Here is an example that uses the default city database and adds the geographical information to the `geoip` field based on the `ip` field: @@ -83,7 +85,7 @@ Here is an example that uses the default city database and adds the geographical -------------------------------------------------- PUT _ingest/pipeline/geoip { - "description" : "Add geoip info", + "description" : "Add ip geolocation info", "processors" : [ { "geoip" : { @@ -134,7 +136,7 @@ this database is downloaded automatically. So this: -------------------------------------------------- PUT _ingest/pipeline/geoip { - "description" : "Add geoip info", + "description" : "Add ip geolocation info", "processors" : [ { "geoip" : { @@ -186,7 +188,7 @@ cannot be found: -------------------------------------------------- PUT _ingest/pipeline/geoip { - "description" : "Add geoip info", + "description" : "Add ip geolocation info", "processors" : [ { "geoip" : { @@ -252,7 +254,7 @@ PUT my_ip_locations -------------------------------------------------- PUT _ingest/pipeline/geoip { - "description" : "Add geoip info", + "description" : "Add ip geolocation info", "processors" : [ { "geoip" : { @@ -425,7 +427,7 @@ The `geoip` processor supports the following setting: The maximum number of results that should be cached. Defaults to `1000`. -Note that these settings are node settings and apply to all `geoip` processors, i.e. there is one cache for all defined `geoip` processors. +Note that these settings are node settings and apply to all `geoip` and `ip_location` processors, i.e. there is a single cache for all such processors. [[geoip-cluster-settings]] ===== Cluster settings @@ -454,7 +456,7 @@ each node's <> at `$ES_TMPDIR/geoip-databases/` | (String) Field added to incoming documents to contain results objects. | `field_map` | no | If defined the model's default field map | (Object) Maps the document field names to the known field names of the model. This mapping takes precedence over any default mappings provided in the model configuration. @@ -31,6 +31,7 @@ include::common-options.asciidoc[] `field_map` fields. For NLP models, use the `input_output` option. For {dfanalytics} models, use the `target_field` and `field_map` option. * Each {infer} input field must be single strings, not arrays of strings. +* The `input_field` is processed as is and ignores any <>'s <> at time of {infer} run. ================================================== [discrete] @@ -168,6 +169,18 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= + `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] @@ -223,6 +236,18 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= + `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] @@ -303,6 +328,23 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`span`:::: +(Optional, integer) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] + +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= + + `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] @@ -362,6 +404,18 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= + `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] @@ -423,6 +477,22 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`span`:::: +(Optional, integer) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] + +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= + `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] @@ -454,6 +524,29 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio ======= ===== +[discrete] +[[inference-processor-text-similarity-opt]] +==== Text similarity configuration options + +`text_similarity`::: +(Object, optional) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-similarity] ++ +.Properties of text_similarity inference +[%collapsible%open] +===== +`span_score_combination_function`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-similarity-span-score-func] + +`tokenization`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] ++ +Refer to <> to review the properties of the +`tokenization` object. +===== + [discrete] [[inference-processor-zero-shot-opt]] @@ -491,6 +584,18 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= + `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] @@ -630,3 +735,70 @@ You can also specify the target field as follows: In this case, {feat-imp} is exposed in the `my_field.foo.feature_importance` field. + + +[discrete] +[[inference-processor-examples]] +==== {infer-cap} processor examples + +The following example uses an <> in an {infer} processor named `query_helper_pipeline` to perform a chat completion task. +The processor generates an {es} query from natural language input using a prompt designed for a completion task type. +Refer to <> for the {infer} service you use and check the corresponding examples of setting up an endpoint with the chat completion task type. + + +[source,console] +-------------------------------------------------- +PUT _ingest/pipeline/query_helper_pipeline +{ + "processors": [ + { + "script": { + "source": "ctx.prompt = 'Please generate an elasticsearch search query on index `articles_index` for the following natural language query. Dates are in the field `@timestamp`, document types are in the field `type` (options are `news`, `publication`), categories in the field `category` and can be multiple (options are `medicine`, `pharmaceuticals`, `technology`), and document names are in the field `title` which should use a fuzzy match. Ignore fields which cannot be determined from the natural language query context: ' + ctx.content" <1> + } + }, + { + "inference": { + "model_id": "openai_chat_completions", <2> + "input_output": { + "input_field": "prompt", + "output_field": "query" + } + } + }, + { + "remove": { + "field": "prompt" + } + } + ] +} +-------------------------------------------------- +// TEST[skip: An inference endpoint is required.] +<1> The `prompt` field contains the prompt used for the completion task, created with <>. +`+ ctx.content` appends the natural language input to the prompt. +<2> The ID of the pre-configured {infer} endpoint, which utilizes the <> with the `completion` task type. + +The following API request will simulate running a document through the ingest pipeline created previously: + +[source,console] +-------------------------------------------------- +POST _ingest/pipeline/query_helper_pipeline/_simulate +{ + "docs": [ + { + "_source": { + "content": "artificial intelligence in medicine articles published in the last 12 months" <1> + } + } + ] +} +-------------------------------------------------- +// TEST[skip: An inference processor with an inference endpoint is required.] +<1> The natural language query used to generate an {es} query within the prompt created by the {infer} processor. + + +[discrete] +[[infer-proc-readings]] +==== Further readings + +* https://www.elastic.co/search-labs/blog/openwebcrawler-llms-semantic-text-resume-job-search[Which job is the best for you? Using LLMs and semantic_text to match resumes to jobs] \ No newline at end of file diff --git a/docs/reference/ingest/processors/ip-location.asciidoc b/docs/reference/ingest/processors/ip-location.asciidoc new file mode 100644 index 0000000000000..e2ca9dbbe2eb3 --- /dev/null +++ b/docs/reference/ingest/processors/ip-location.asciidoc @@ -0,0 +1,225 @@ +[[ip-location-processor]] +=== IP location processor +++++ +IP Location +++++ + +The `ip_location` processor adds information about the geographical location of an +IPv4 or IPv6 address. + +[[ip-location-automatic-updates]] +By default, the processor uses the GeoLite2 City, GeoLite2 Country, and GeoLite2 +ASN IP geolocation databases from http://dev.maxmind.com/geoip/geoip2/geolite2/[MaxMind], shared under the +CC BY-SA 4.0 license. It automatically downloads these databases if your nodes can connect to `storage.googleapis.com` domain and either: + +* `ingest.geoip.downloader.eager.download` is set to true +* your cluster has at least one pipeline with a `geoip` or `ip_location` processor + +{es} automatically downloads updates for these databases from the Elastic GeoIP +endpoint: +https://geoip.elastic.co/v1/database?elastic_geoip_service_tos=agree[https://geoip.elastic.co/v1/database]. +To get download statistics for these updates, use the <>. + +If your cluster can't connect to the Elastic GeoIP endpoint or you want to +manage your own updates, see <>. + +If you would like to have {es} download database files directly from Maxmind using your own provided +license key, see <>. + +If {es} can't connect to the endpoint for 30 days all updated databases will become +invalid. {es} will stop enriching documents with ip geolocation data and will add `tags: ["_ip_location_expired_database"]` +field instead. + +[[using-ingest-ip-location]] +==== Using the `ip_location` Processor in a Pipeline + +[[ingest-ip-location-options]] +.`ip-location` options +[options="header"] +|====== +| Name | Required | Default | Description +| `field` | yes | - | The field to get the IP address from for the geographical lookup. +| `target_field` | no | ip_location | The field that will hold the geographical information looked up from the database. +| `database_file` | no | GeoLite2-City.mmdb | The database filename referring to one of the automatically downloaded GeoLite2 databases (GeoLite2-City.mmdb, GeoLite2-Country.mmdb, or GeoLite2-ASN.mmdb), or the name of a supported database file in the `ingest-geoip` config directory, or the name of a <> (with the `.mmdb` suffix appended). +| `properties` | no | [`continent_name`, `country_iso_code`, `country_name`, `region_iso_code`, `region_name`, `city_name`, `location`] * | Controls what properties are added to the `target_field` based on the ip geolocation lookup. +| `ignore_missing` | no | `false` | If `true` and `field` does not exist, the processor quietly exits without modifying the document +| `first_only` | no | `true` | If `true` only first found ip geolocation data, will be returned, even if `field` contains array +| `download_database_on_pipeline_creation` | no | `true` | If `true` (and if `ingest.geoip.downloader.eager.download` is `false`), the missing database is downloaded when the pipeline is created. Else, the download is triggered by when the pipeline is used as the `default_pipeline` or `final_pipeline` in an index. +|====== + +*Depends on what is available in `database_file`: + +* If a GeoLite2 City or GeoIP2 City database is used, then the following fields may be added under the `target_field`: `ip`, +`country_iso_code`, `country_name`, `country_in_european_union`, `registered_country_iso_code`, `registered_country_name`, `registered_country_in_european_union`, +`continent_code`, `continent_name`, `region_iso_code`, `region_name`, `city_name`, `postal_code`, `timezone`, +`location`, and `accuracy_radius`. The fields actually added depend on what has been found and which properties were configured in `properties`. +* If a GeoLite2 Country or GeoIP2 Country database is used, then the following fields may be added under the `target_field`: `ip`, +`country_iso_code`, `country_name`, `country_in_european_union`, `registered_country_iso_code`, `registered_country_name`, `registered_country_in_european_union`, +`continent_code`, and `continent_name`. The fields actually added depend on what has been found +and which properties were configured in `properties`. +* If the GeoLite2 ASN database is used, then the following fields may be added under the `target_field`: `ip`, +`asn`, `organization_name` and `network`. The fields actually added depend on what has been found and which properties were configured +in `properties`. +* If the GeoIP2 Anonymous IP database is used, then the following fields may be added under the `target_field`: `ip`, +`hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`, and `residential_proxy`. The fields actually added +depend on what has been found and which properties were configured in `properties`. +* If the GeoIP2 Connection Type database is used, then the following fields may be added under the `target_field`: `ip`, and +`connection_type`. The fields actually added depend on what has been found and which properties were configured in `properties`. +* If the GeoIP2 Domain database is used, then the following fields may be added under the `target_field`: `ip`, and `domain`. +The fields actually added depend on what has been found and which properties were configured in `properties`. +* If the GeoIP2 ISP database is used, then the following fields may be added under the `target_field`: `ip`, `asn`, +`organization_name`, `network`, `isp`, `isp_organization_name`, `mobile_country_code`, and `mobile_network_code`. The fields actually added +depend on what has been found and which properties were configured in `properties`. +* If the GeoIP2 Enterprise database is used, then the following fields may be added under the `target_field`: `ip`, +`country_iso_code`, `country_name`, `country_in_european_union`, `registered_country_iso_code`, `registered_country_name`, `registered_country_in_european_union`, +`continent_code`, `continent_name`, `region_iso_code`, `region_name`, `city_name`, `postal_code`, `timezone`, +`location`, `accuracy_radius`, `country_confidence`, `city_confidence`, `postal_confidence`, `asn`, `organization_name`, `network`, +`hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`, +`residential_proxy`, `domain`, `isp`, `isp_organization_name`, `mobile_country_code`, `mobile_network_code`, `user_type`, and +`connection_type`. The fields actually added depend on what has been found and which properties were configured in `properties`. + +Here is an example that uses the default city database and adds the geographical information to the `ip_location` field based on the `ip` field: + +[source,console] +-------------------------------------------------- +PUT _ingest/pipeline/ip_location +{ + "description" : "Add ip geolocation info", + "processors" : [ + { + "ip_location" : { + "field" : "ip" + } + } + ] +} +PUT my-index-000001/_doc/my_id?pipeline=ip_location +{ + "ip": "89.160.20.128" +} +GET my-index-000001/_doc/my_id +-------------------------------------------------- + +Which returns: + +[source,console-result] +-------------------------------------------------- +{ + "found": true, + "_index": "my-index-000001", + "_id": "my_id", + "_version": 1, + "_seq_no": 55, + "_primary_term": 1, + "_source": { + "ip": "89.160.20.128", + "ip_location": { + "continent_name": "Europe", + "country_name": "Sweden", + "country_iso_code": "SE", + "city_name" : "Linköping", + "region_iso_code" : "SE-E", + "region_name" : "Östergötland County", + "location": { "lat": 58.4167, "lon": 15.6167 } + } + } +} +-------------------------------------------------- +// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term":1/"_primary_term" : $body._primary_term/] + +Here is an example that uses the default country database and adds the +geographical information to the `geo` field based on the `ip` field. Note that +this database is downloaded automatically. So this: + +[source,console] +-------------------------------------------------- +PUT _ingest/pipeline/ip_location +{ + "description" : "Add ip geolocation info", + "processors" : [ + { + "ip_location" : { + "field" : "ip", + "target_field" : "geo", + "database_file" : "GeoLite2-Country.mmdb" + } + } + ] +} +PUT my-index-000001/_doc/my_id?pipeline=ip_location +{ + "ip": "89.160.20.128" +} +GET my-index-000001/_doc/my_id +-------------------------------------------------- + +returns this: + +[source,console-result] +-------------------------------------------------- +{ + "found": true, + "_index": "my-index-000001", + "_id": "my_id", + "_version": 1, + "_seq_no": 65, + "_primary_term": 1, + "_source": { + "ip": "89.160.20.128", + "geo": { + "continent_name": "Europe", + "country_name": "Sweden", + "country_iso_code": "SE" + } + } +} +-------------------------------------------------- +// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/] + + +Not all IP addresses find geo information from the database, When this +occurs, no `target_field` is inserted into the document. + +Here is an example of what documents will be indexed as when information for "80.231.5.0" +cannot be found: + +[source,console] +-------------------------------------------------- +PUT _ingest/pipeline/ip_location +{ + "description" : "Add ip geolocation info", + "processors" : [ + { + "ip_location" : { + "field" : "ip" + } + } + ] +} + +PUT my-index-000001/_doc/my_id?pipeline=ip_location +{ + "ip": "80.231.5.0" +} + +GET my-index-000001/_doc/my_id +-------------------------------------------------- + +Which returns: + +[source,console-result] +-------------------------------------------------- +{ + "_index" : "my-index-000001", + "_id" : "my_id", + "_version" : 1, + "_seq_no" : 71, + "_primary_term": 1, + "found" : true, + "_source" : { + "ip" : "80.231.5.0" + } +} +-------------------------------------------------- +// TESTRESPONSE[s/"_seq_no" : \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/] diff --git a/docs/reference/ingest/processors/redact.asciidoc b/docs/reference/ingest/processors/redact.asciidoc index 6706106e92655..9b8ac1e15d1a8 100644 --- a/docs/reference/ingest/processors/redact.asciidoc +++ b/docs/reference/ingest/processors/redact.asciidoc @@ -39,6 +39,7 @@ patterns. Legacy Grok patterns are not supported. | `ignore_missing` | no | `true` | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document include::common-options.asciidoc[] | `skip_if_unlicensed` | no | `false` | If `true` and the current license does not support running redact processors, then the processor quietly exits without modifying the document +| `trace_redact` | no | `false` | If `true` then ingest metadata `_ingest._redact._is_redacted` is set to `true` if the document has been redacted |====== In this example the predefined `IP` Grok pattern is used to match diff --git a/docs/reference/ingest/processors/terminate.asciidoc b/docs/reference/ingest/processors/terminate.asciidoc new file mode 100644 index 0000000000000..a2643fbd955f1 --- /dev/null +++ b/docs/reference/ingest/processors/terminate.asciidoc @@ -0,0 +1,30 @@ +[[terminate-processor]] +=== Terminate processor + +++++ +Terminate +++++ + +Terminates the current ingest pipeline, causing no further processors to be run. +This will normally be executed conditionally, using the `if` option. + +If this pipeline is being called from another pipeline, the calling pipeline is *not* terminated. + +[[terminate-options]] +.Terminate Options +[options="header"] +|====== +| Name | Required | Default | Description +include::common-options.asciidoc[] +|====== + +[source,js] +-------------------------------------------------- +{ + "description" : "terminates the current pipeline if the error field is present", + "terminate": { + "if": "ctx.error != null" + } +} +-------------------------------------------------- +// NOTCONSOLE diff --git a/docs/reference/ingest/search-ingest-pipelines.asciidoc b/docs/reference/ingest/search-ingest-pipelines.asciidoc index f37e07f632810..e414dacaab964 100644 --- a/docs/reference/ingest/search-ingest-pipelines.asciidoc +++ b/docs/reference/ingest/search-ingest-pipelines.asciidoc @@ -37,7 +37,8 @@ For example: It can be a lot of work to set up and manage production-ready pipelines from scratch. Considerations such as error handling, conditional execution, sequencing, versioning, and modularization must all be taken into account. -To this end, when you create indices for search use cases, (including {enterprise-search-ref}/crawler.html[Elastic web crawler], {enterprise-search-ref}/connectors.html[Elastic connector], and API indices), each index already has a pipeline set up with several processors that optimize your content for search. +To this end, when you create indices for search use cases, (including {enterprise-search-ref}/crawler.html[Elastic web crawler], <>. +, and API indices), each index already has a pipeline set up with several processors that optimize your content for search. This pipeline is called `ent-search-generic-ingestion`. While it is a "managed" pipeline (meaning it should not be tampered with), you can view its details via the Kibana UI or the Elasticsearch API. @@ -45,7 +46,8 @@ You can also <>. +, you can opt out (or back in) per index, and your choices are saved. For API indices, you can opt out (or back in) by including specific fields in your documents. <>. diff --git a/docs/reference/intro.asciidoc b/docs/reference/intro.asciidoc index f80856368af2b..2fd0722bcd660 100644 --- a/docs/reference/intro.asciidoc +++ b/docs/reference/intro.asciidoc @@ -1,67 +1,97 @@ [[elasticsearch-intro]] -== What is {es}? +== {es} basics + +This guide covers the core concepts you need to understand to get started with {es}. +If you'd prefer to start working with {es} right away, set up a <> and jump to <>. + +This guide covers the following topics: + +* <>: Learn about {es} and some of its main use cases. +* <>: Understand your options for deploying {es} in different environments, including a fast local development setup. +* <>: Understand {es}'s most important primitives and how it stores data. +* <>: Understand your options for ingesting data into {es}. +* <>: Understand your options for searching and analyzing data in {es}. +* <>: Understand the basic concepts required for moving your {es} deployment to production. + +[[elasticsearch-intro-what-is-es]] +=== What is {es}? {es-repo}[{es}] is a distributed search and analytics engine, scalable data store, and vector database built on Apache Lucene. It's optimized for speed and relevance on production-scale workloads. Use {es} to search, index, store, and analyze data of all shapes and sizes in near real time. +{es} is the heart of the {estc-welcome-current}/stack-components.html[Elastic Stack]. +Combined with https://www.elastic.co/kibana[{kib}], it powers the following Elastic solutions: + +* https://www.elastic.co/observability[Observability] +* https://www.elastic.co/enterprise-search[Search] +* https://www.elastic.co/security[Security] + [TIP] ==== {es} has a lot of features. Explore the full list on the https://www.elastic.co/elasticsearch/features[product webpage^]. ==== -{es} is the heart of the {estc-welcome-current}/stack-components.html[Elastic Stack] and powers the Elastic https://www.elastic.co/enterprise-search[Search], https://www.elastic.co/observability[Observability] and https://www.elastic.co/security[Security] solutions. - -{es} is used for a wide and growing range of use cases. Here are a few examples: - -* *Monitor log and event data*: Store logs, metrics, and event data for observability and security information and event management (SIEM). -* *Build search applications*: Add search capabilities to apps or websites, or build search engines over internal data. -* *Vector database*: Store and search vectorized data, and create vector embeddings with built-in and third-party natural language processing (NLP) models. -* *Retrieval augmented generation (RAG)*: Use {es} as a retrieval engine to augment generative AI models. -* *Application and security monitoring*: Monitor and analyze application performance and security data. -* *Machine learning*: Use {ml} to automatically model the behavior of your data in real-time. - -This is just a sample of search, observability, and security use cases enabled by {es}. -Refer to our https://www.elastic.co/customers/success-stories[customer success stories] for concrete examples across a range of industries. -// Link to demos, search labs chatbots - [discrete] [[elasticsearch-intro-elastic-stack]] .What is the Elastic Stack? ******************************* {es} is the core component of the Elastic Stack, a suite of products for collecting, storing, searching, and visualizing data. -https://www.elastic.co/guide/en/starting-with-the-elasticsearch-platform-and-its-solutions/current/stack-components.html[Learn more about the Elastic Stack]. +{estc-welcome-current}/stack-components.html[Learn more about the Elastic Stack]. ******************************* -// TODO: Remove once we've moved Stack Overview to a subpage? [discrete] +[[elasticsearch-intro-use-cases]] +==== Use cases + +{es} is used for a wide and growing range of use cases. Here are a few examples: + +**Observability** + +* *Logs, metrics, and traces*: Collect, store, and analyze logs, metrics, and traces from applications, systems, and services. +* *Application performance monitoring (APM)*: Monitor and analyze the performance of business-critical software applications. +* *Real user monitoring (RUM)*: Monitor, quantify, and analyze user interactions with web applications. +* *OpenTelemetry*: Reuse your existing instrumentation to send telemetry data to the Elastic Stack using the OpenTelemetry standard. + +**Search** + +* *Full-text search*: Build a fast, relevant full-text search solution using inverted indexes, tokenization, and text analysis. +* *Vector database*: Store and search vectorized data, and create vector embeddings with built-in and third-party natural language processing (NLP) models. +* *Semantic search*: Understand the intent and contextual meaning behind search queries using tools like synonyms, dense vector embeddings, and learned sparse query-document expansion. +* *Hybrid search*: Combine full-text search with vector search using state-of-the-art ranking algorithms. +* *Build search experiences*: Add hybrid search capabilities to apps or websites, or build enterprise search engines over your organization's internal data sources. +* *Retrieval augmented generation (RAG)*: Use {es} as a retrieval engine to supplement generative AI models with more relevant, up-to-date, or proprietary data for a range of use cases. +* *Geospatial search*: Search for locations and calculate spatial relationships using geospatial queries. + +**Security** + +* *Security information and event management (SIEM)*: Collect, store, and analyze security data from applications, systems, and services. +* *Endpoint security*: Monitor and analyze endpoint security data. +* *Threat hunting*: Search and analyze data to detect and respond to security threats. + +This is just a sample of search, observability, and security use cases enabled by {es}. +Refer to Elastic https://www.elastic.co/customers/success-stories[customer success stories] for concrete examples across a range of industries. + [[elasticsearch-intro-deploy]] -=== Deployment options +=== Run {es} To use {es}, you need a running instance of the {es} service. -You can deploy {es} in various ways: +You can deploy {es} in various ways. -* <>: Get started quickly with a minimal local Docker setup. -* {cloud}/ec-getting-started-trial.html[*Elastic Cloud*]: {es} is available as part of our hosted Elastic Stack offering, deployed in the cloud with your provider of choice. Sign up for a https://cloud.elastic.co/registration[14-day free trial]. -* {serverless-docs}/general/sign-up-trial[*Elastic Cloud Serverless* (technical preview)]: Create serverless projects for autoscaled and fully managed {es} deployments. Sign up for a https://cloud.elastic.co/serverless-registration[14-day free trial]. +**Quick start option** -**Advanced deployment options** +* <>: Get started quickly with a minimal local Docker setup for development and testing. -* <>: Install, configure, and run {es} on your own premises. -* {ece-ref}/Elastic-Cloud-Enterprise-overview.html[*Elastic Cloud Enterprise*]: Deploy Elastic Cloud on public or private clouds, virtual machines, or your own premises. -* {eck-ref}/k8s-overview.html[*Elastic Cloud on Kubernetes*]: Deploy Elastic Cloud on Kubernetes. +**Hosted options** -[discrete] -[[elasticsearch-next-steps]] -=== Learn more +* {cloud}/ec-getting-started-trial.html[*Elastic Cloud Hosted*]: {es} is available as part of the hosted Elastic Stack offering, deployed in the cloud with your provider of choice. Sign up for a https://cloud.elastic.co/registration[14-day free trial]. +* {serverless-docs}/general/sign-up-trial[*Elastic Cloud Serverless*]: Create serverless projects for autoscaled and fully managed {es} deployments. Sign up for a https://cloud.elastic.co/serverless-registration[14-day free trial]. -Here are some resources to help you get started: +**Advanced options** -* <>: A beginner's guide to deploying your first {es} instance, indexing data, and running queries. -* https://elastic.co/webinars/getting-started-elasticsearch[Webinar: Introduction to {es}]: Register for our live webinars to learn directly from {es} experts. -* https://www.elastic.co/search-labs[Elastic Search Labs]: Tutorials and blogs that explore AI-powered search using the latest {es} features. -** Follow our tutorial https://www.elastic.co/search-labs/tutorials/search-tutorial/welcome[to build a hybrid search solution in Python]. -** Check out the https://github.com/elastic/elasticsearch-labs?tab=readme-ov-file#elasticsearch-examples--apps[`elasticsearch-labs` repository] for a range of Python notebooks and apps for various use cases. +* <>: Install, configure, and run {es} on your own premises. +* {ece-ref}/Elastic-Cloud-Enterprise-overview.html[*Elastic Cloud Enterprise*]: Deploy Elastic Cloud on public or private clouds, virtual machines, or your own premises. +* {eck-ref}/k8s-overview.html[*Elastic Cloud on Kubernetes*]: Deploy Elastic Cloud on Kubernetes. // new html page [[documents-indices]] @@ -73,20 +103,16 @@ Here are some resources to help you get started: The index is the fundamental unit of storage in {es}, a logical namespace for storing data that share similar characteristics. After you have {es} <>, you'll get started by creating an index to store your data. +An index is a collection of documents uniquely identified by a name or an <>. +This unique name is important because it's used to target the index in search queries and other operations. + [TIP] ==== A closely related concept is a <>. -This index abstraction is optimized for append-only time-series data, and is made up of hidden, auto-generated backing indices. -If you're working with time-series data, we recommend the {observability-guide}[Elastic Observability] solution. +This index abstraction is optimized for append-only timestamped data, and is made up of hidden, auto-generated backing indices. +If you're working with timestamped data, we recommend the {observability-guide}[Elastic Observability] solution for additional tools and optimized content. ==== -Some key facts about indices: - -* An index is a collection of documents -* An index has a unique name -* An index can also be referred to by an alias -* An index has a mapping that defines the schema of its documents - [discrete] [[elasticsearch-intro-documents-fields]] ==== Documents and fields @@ -126,14 +152,12 @@ A simple {es} document might look like this: [discrete] [[elasticsearch-intro-documents-fields-data-metadata]] -==== Data and metadata +==== Metadata fields -An indexed document contains data and metadata. +An indexed document contains data and metadata. <> are system fields that store information about the documents. In {es}, metadata fields are prefixed with an underscore. +For example, the following fields are metadata fields: -The most important metadata fields are: - -* `_source`: Contains the original JSON document. * `_index`: The name of the index where the document is stored. * `_id`: The document's ID. IDs must be unique per index. @@ -146,8 +170,8 @@ A mapping defines the <> for each field, how the field and how it should be stored. When adding documents to {es}, you have two options for mappings: -* <>: Let {es} automatically detect the data types and create the mappings for you. This is great for getting started quickly, but can lead to unexpected results for complex data. -* <>: Define the mappings up front by specifying data types for each field. Recommended for production use cases, because you have much more control over how your data is indexed. +* <>: Let {es} automatically detect the data types and create the mappings for you. Dynamic mapping helps you get started quickly, but might yield suboptimal results for your specific use case due to automatic field type inference. +* <>: Define the mappings up front by specifying data types for each field. Recommended for production use cases, because you have full control over how your data is indexed to suit your specific use case. [TIP] ==== @@ -155,164 +179,283 @@ You can use a combination of dynamic and explicit mapping on the same index. This is useful when you have a mix of known and unknown fields in your data. ==== +// New html page +[[es-ingestion-overview]] +=== Add data to {es} + +There are multiple ways to ingest data into {es}. +The option that you choose depends on whether you're working with timestamped data or non-timestamped data, where the data is coming from, its complexity, and more. + +[TIP] +==== +You can load {kibana-ref}/connect-to-elasticsearch.html#_add_sample_data[sample data] into your {es} cluster using {kib}, to get started quickly. +==== + +[discrete] +[[es-ingestion-overview-general-content]] +==== General content + +General content is data that does not have a timestamp. +This could be data like vector embeddings, website content, product catalogs, and more. +For general content, you have the following options for adding data to {es} indices: + +* <>: Use the {es} <> to index documents directly, using the Dev Tools {kibana-ref}/console-kibana.html[Console], or cURL. ++ +If you're building a website or app, then you can call Elasticsearch APIs using an https://www.elastic.co/guide/en/elasticsearch/client/index.html[{es} client] in the programming language of your choice. If you use the Python client, then check out the `elasticsearch-labs` repo for various https://github.com/elastic/elasticsearch-labs/tree/main/notebooks/search/python-examples[example notebooks]. +* {kibana-ref}/connect-to-elasticsearch.html#upload-data-kibana[File upload]: Use the {kib} file uploader to index single files for one-off testing and exploration. The GUI guides you through setting up your index and field mappings. +* https://github.com/elastic/crawler[Web crawler]: Extract and index web page content into {es} documents. +* <>: Sync data from various third-party data sources to create searchable, read-only replicas in {es}. + +[discrete] +[[es-ingestion-overview-timestamped]] +==== Timestamped data + +Timestamped data in {es} refers to datasets that include a timestamp field. If you use the {ecs-ref}/ecs-reference.html[Elastic Common Schema (ECS)], this field is named `@timestamp`. +This could be data like logs, metrics, and traces. + +For timestamped data, you have the following options for adding data to {es} data streams: + +* {fleet-guide}/fleet-overview.html[Elastic Agent and Fleet]: The preferred way to index timestamped data. Each Elastic Agent based integration includes default ingestion rules, dashboards, and visualizations to start analyzing your data right away. +You can use the Fleet UI in {kib} to centrally manage Elastic Agents and their policies. +* {beats-ref}/beats-reference.html[Beats]: If your data source isn't supported by Elastic Agent, use Beats to collect and ship data to Elasticsearch. You install a separate Beat for each type of data to collect. +* {logstash-ref}/introduction.html[Logstash]: Logstash is an open source data collection engine with real-time pipelining capabilities that supports a wide variety of data sources. You might use this option because neither Elastic Agent nor Beats supports your data source. You can also use Logstash to persist incoming data, or if you need to send the data to multiple destinations. +* {cloud}/ec-ingest-guides.html[Language clients]: The linked tutorials demonstrate how to use {es} programming language clients to ingest data from an application. In these examples, {es} is running on Elastic Cloud, but the same principles apply to any {es} deployment. + +[TIP] +==== +If you're interested in data ingestion pipelines for timestamped data, use the decision tree in the {cloud}/ec-cloud-ingest-data.html#ec-data-ingest-pipeline[Elastic Cloud docs] to understand your options. +==== + // New html page [[search-analyze]] -=== Search and analyze +=== Search and analyze data + +You can use {es} as a basic document store to retrieve documents and their +metadata. +However, the real power of {es} comes from its advanced search and analytics capabilities. + +You'll use a combination of an API endpoint and a query language to interact with your data. + +[discrete] +[[search-analyze-rest-api]] +==== REST API + +Use REST APIs to manage your {es} cluster, and to index +and search your data. +For testing purposes, you can submit requests +directly from the command line or through the Dev Tools {kibana-ref}/console-kibana.html[Console] in {kib}. +From your applications, you can use a +https://www.elastic.co/guide/en/elasticsearch/client/index.html[client] +in your programming language of choice. + +Refer to <> for a hands-on example of using the `_search` endpoint, adding data to {es}, and running basic searches in Query DSL syntax. -While you can use {es} as a document store and retrieve documents and their -metadata, the real power comes from being able to easily access the full suite -of search capabilities built on the Apache Lucene search engine library. +[discrete] +[[search-analyze-query-languages]] +==== Query languages + +{es} provides a number of query languages for interacting with your data. + +*Query DSL* is the primary query language for {es} today. + +*{esql}* is a new piped query language and compute engine which was first added in version *8.11*. -{es} provides a simple, coherent REST API for managing your cluster and indexing -and searching your data. For testing purposes, you can easily submit requests -directly from the command line or through the Developer Console in {kib}. From -your applications, you can use the -https://www.elastic.co/guide/en/elasticsearch/client/index.html[{es} client] -for your language of choice: Java, JavaScript, Go, .NET, PHP, Perl, Python -or Ruby. +{esql} does not yet support all the features of Query DSL. +Look forward to new {esql} features and functionalities in each release. + +Refer to <> for a full overview of the query languages available in {es}. [discrete] -[[search-data]] -==== Searching your data - -The {es} REST APIs support structured queries, full text queries, and complex -queries that combine the two. Structured queries are -similar to the types of queries you can construct in SQL. For example, you -could search the `gender` and `age` fields in your `employee` index and sort the -matches by the `hire_date` field. Full-text queries find all documents that -match the query string and return them sorted by _relevance_—how good a -match they are for your search terms. - -In addition to searching for individual terms, you can perform phrase searches, -similarity searches, and prefix searches, and get autocomplete suggestions. - -Have geospatial or other numerical data that you want to search? {es} indexes -non-textual data in optimized data structures that support -high-performance geo and numerical queries. - -You can access all of these search capabilities using {es}'s -comprehensive JSON-style query language (<>). You can also -construct <> to search and aggregate data -natively inside {es}, and JDBC and ODBC drivers enable a broad range of -third-party applications to interact with {es} via SQL. +[[search-analyze-query-dsl]] +===== Query DSL + +<> is a full-featured JSON-style query language that enables complex searching, filtering, and aggregations. +It is the original and most powerful query language for {es} today. + +The <> accepts queries written in Query DSL syntax. [discrete] -[[analyze-data]] -==== Analyzing your data +[[search-analyze-query-dsl-search-filter]] +====== Search and filter with Query DSL -{es} aggregations enable you to build complex summaries of your data and gain -insight into key metrics, patterns, and trends. Instead of just finding the -proverbial “needle in a haystack”, aggregations enable you to answer questions -like: +Query DSL support a wide range of search techniques, including the following: -* How many needles are in the haystack? -* What is the average length of the needles? -* What is the median length of the needles, broken down by manufacturer? -* How many needles were added to the haystack in each of the last six months? +* <>: Search text that has been analyzed and indexed to support phrase or proximity queries, fuzzy matches, and more. +* <>: Search for exact matches using `keyword` fields. +* <>: Search `semantic_text` fields using dense or sparse vector search on embeddings generated in your {es} cluster. +* <>: Search for similar dense vectors using the kNN algorithm for embeddings generated outside of {es}. +* <>: Search for locations and calculate spatial relationships using geospatial queries. -You can also use aggregations to answer more subtle questions, such as: +Learn about the full range of queries supported by <>. -* What are your most popular needle manufacturers? -* Are there any unusual or anomalous clumps of needles? +You can also filter data using Query DSL. +Filters enable you to include or exclude documents by retrieving documents that match specific field-level criteria. +A query that uses the `filter` parameter indicates <>. -Because aggregations leverage the same data-structures used for search, they are +[discrete] +[[search-analyze-data-query-dsl]] +====== Analyze with Query DSL + +<> are the primary tool for analyzing {es} data using Query DSL. +Aggregrations enable you to build complex summaries of your data and gain +insight into key metrics, patterns, and trends. + +Because aggregations leverage the same data structures used for search, they are also very fast. This enables you to analyze and visualize your data in real time. -Your reports and dashboards update as your data changes so you can take action -based on the latest information. +You can search documents, filter results, and perform analytics at the same time, on the same +data, in a single request. +That means aggregations are calculated in the context of the search query. + +The folowing aggregation types are available: -What’s more, aggregations operate alongside search requests. You can search -documents, filter results, and perform analytics at the same time, on the same -data, in a single request. And because aggregations are calculated in the -context of a particular search, you’re not just displaying a count of all -size 70 needles, you’re displaying a count of the size 70 needles -that match your users' search criteria--for example, all size 70 _non-stick -embroidery_ needles. +* <>: Calculate metrics, +such as a sum or average, from field values. +* <>: Group documents into buckets based on field values, ranges, +or other criteria. +* <>: Run aggregations on the results of other aggregations. + +Run aggregations by specifying the <>'s `aggs` parameter. +Learn more in <>. + +[discrete] +[[search-analyze-data-esql]] +===== {esql} +<> is a piped query language for filtering, transforming, and analyzing data. +{esql} is built on top of a new compute engine, where search, aggregation, and transformation functions are +directly executed within {es} itself. +{esql} syntax can also be used within various {kib} tools. + +The <> accepts queries written in {esql} syntax. + +Today, it supports a subset of the features available in Query DSL, but it is rapidly evolving. + +It comes with a comprehensive set of <> for working with data and has robust integration with {kib}'s Discover, dashboards and visualizations. + +Learn more in <>, or try https://www.elastic.co/training/introduction-to-esql[our training course]. + +[discrete] +[[search-analyze-data-query-languages-table]] +==== List of available query languages + +The following table summarizes all available {es} query languages, to help you choose the right one for your use case. + +[cols="1,2,2,1", options="header"] +|=== +| Name | Description | Use cases | API endpoint + +| <> +| The primary query language for {es}. A powerful and flexible JSON-style language that enables complex queries. +| Full-text search, semantic search, keyword search, filtering, aggregations, and more. +| <> + + +| <> +| Introduced in *8.11*, the Elasticsearch Query Language ({esql}) is a piped query language language for filtering, transforming, and analyzing data. +| Initially tailored towards working with time series data like logs and metrics. +Robust integration with {kib} for querying, visualizing, and analyzing data. +Does not yet support full-text search. +| <> + + +| <> +| Event Query Language (EQL) is a query language for event-based time series data. Data must contain the `@timestamp` field to use EQL. +| Designed for the threat hunting security use case. +| <> + +| <> +| Allows native, real-time SQL-like querying against {es} data. JDBC and ODBC drivers are available for integration with business intelligence (BI) tools. +| Enables users familiar with SQL to query {es} data using familiar syntax for BI and reporting. +| <> + +| {kibana-ref}/kuery-query.html[Kibana Query Language (KQL)] +| {kib} Query Language (KQL) is a text-based query language for filtering data when you access it through the {kib} UI. +| Use KQL to filter documents where a value for a field exists, matches a given value, or is within a given range. +| N/A + +|=== + +// New html page [[scalability]] -=== Scalability and resilience - -{es} is built to be always available and to scale with your needs. It does this -by being distributed by nature. You can add servers (nodes) to a cluster to -increase capacity and {es} automatically distributes your data and query load -across all of the available nodes. No need to overhaul your application, {es} -knows how to balance multi-node clusters to provide scale and high availability. -The more nodes, the merrier. - -How does this work? Under the covers, an {es} index is really just a logical -grouping of one or more physical shards, where each shard is actually a -self-contained index. By distributing the documents in an index across multiple -shards, and distributing those shards across multiple nodes, {es} can ensure -redundancy, which both protects against hardware failures and increases -query capacity as nodes are added to a cluster. As the cluster grows (or shrinks), -{es} automatically migrates shards to rebalance the cluster. - -There are two types of shards: primaries and replicas. Each document in an index -belongs to one primary shard. A replica shard is a copy of a primary shard. -Replicas provide redundant copies of your data to protect against hardware -failure and increase capacity to serve read requests -like searching or retrieving a document. - -The number of primary shards in an index is fixed at the time that an index is -created, but the number of replica shards can be changed at any time, without -interrupting indexing or query operations. +=== Get ready for production + +Many teams rely on {es} to run their key services. To keep these services running, you can design your {es} deployment +to keep {es} available, even in case of large-scale outages. To keep it running fast, you also can design your +deployment to be responsive to production workloads. + +{es} is built to be always available and to scale with your needs. It does this using a distributed architecture. +By distributing your cluster, you can keep Elastic online and responsive to requests. + +In case of failure, {es} offers tools for cross-cluster replication and cluster snapshots that can +help you fall back or recover quickly. You can also use cross-cluster replication to serve requests based on the +geographic location of your users and your resources. + +{es} also offers security and monitoring tools to help you keep your cluster highly available. [discrete] -[[it-depends]] -==== Shard size and number of shards +[[use-multiple-nodes-shards]] +==== Use multiple nodes and shards + +When you move to production, you need to introduce multiple nodes and shards to your cluster. Nodes and shards are what make {es} distributed and scalable. The size and number of these nodes and shards depends on your data, your use case, and your budget. -There are a number of performance considerations and trade offs with respect -to shard size and the number of primary shards configured for an index. The more -shards, the more overhead there is simply in maintaining those indices. The -larger the shard size, the longer it takes to move shards around when {es} -needs to rebalance a cluster. +These concepts aren't essential if you're just getting started. How you <> in production determines what you need to know: -Querying lots of small shards makes the processing per shard faster, but more -queries means more overhead, so querying a smaller -number of larger shards might be faster. In short...it depends. +* *Self-managed {es}*: You are responsible for setting up and managing nodes, clusters, shards, and replicas. This includes +managing the underlying infrastructure, scaling, and ensuring high availability through failover and backup strategies. +* *Elastic Cloud*: Elastic can autoscale resources in response to workload changes. Choose from different deployment types +to apply sensible defaults for your use case. A basic understanding of nodes, shards, and replicas is still important. +* *Elastic Cloud Serverless*: You don't need to worry about nodes, shards, or replicas. These resources are 100% automated +on the serverless platform, which is designed to scale with your workload. + +Learn more about <>. + +[discrete] +[[ccr-disaster-recovery-geo-proximity]] +==== CCR for disaster recovery and geo-proximity -As a starting point: +To effectively distribute read and write operations across nodes, the nodes in a cluster need good, reliable connections +to each other. To provide better connections, you typically co-locate the nodes in the same data center or nearby data centers. -* Aim to keep the average shard size between a few GB and a few tens of GB. For - use cases with time-based data, it is common to see shards in the 20GB to 40GB - range. +Co-locating nodes in a single location exposes you to the risk of a single outage taking your entire cluster offline. To +maintain high availability, you can prepare a second cluster that can take over in case of disaster by implementing +cross-cluster replication (CCR). -* Avoid the gazillion shards problem. The number of shards a node can hold is - proportional to the available heap space. As a general rule, the number of - shards per GB of heap space should be less than 20. +CCR provides a way to automatically synchronize indices from your primary cluster to a secondary remote cluster that +can serve as a hot backup. If the primary cluster fails, the secondary cluster can take over. -The best way to determine the optimal configuration for your use case is -through https://www.elastic.co/elasticon/conf/2016/sf/quantitative-cluster-sizing[ -testing with your own data and queries]. +You can also use CCR to create secondary clusters to serve read requests in geo-proximity to your users. + +Learn more about <> and about <>. + +[TIP] +==== +You can also take <> of your cluster that can be restored in case of failure. +==== [discrete] -[[disaster-ccr]] -==== Disaster recovery +[[security-and-monitoring]] +==== Security and monitoring -A cluster's nodes need good, reliable connections to each other. To provide -better connections, you typically co-locate the nodes in the same data center or -nearby data centers. However, to maintain high availability, you -also need to avoid any single point of failure. In the event of a major outage -in one location, servers in another location need to be able to take over. The -answer? {ccr-cap} (CCR). +As with any enterprise system, you need tools to secure, manage, and monitor your {es} clusters. Security, +monitoring, and administrative features that are integrated into {es} enable you to use {kibana-ref}/introduction.html[Kibana] as a +control center for managing a cluster. -CCR provides a way to automatically synchronize indices from your primary cluster -to a secondary remote cluster that can serve as a hot backup. If the primary -cluster fails, the secondary cluster can take over. You can also use CCR to -create secondary clusters to serve read requests in geo-proximity to your users. +<>. -{ccr-cap} is active-passive. The index on the primary cluster is -the active leader index and handles all write requests. Indices replicated to -secondary clusters are read-only followers. +<>. [discrete] -[[admin]] -==== Security, management, and monitoring +[[cluster-design]] +==== Cluster design + +{es} offers many options that allow you to configure your cluster to meet your organization’s goals, requirements, +and restrictions. You can review the following guides to learn how to tune your cluster to meet your needs: -As with any enterprise system, you need tools to secure, manage, and -monitor your {es} clusters. Security, monitoring, and administrative features -that are integrated into {es} enable you to use {kibana-ref}/introduction.html[{kib}] -as a control center for managing a cluster. Features like <> and <> -help you intelligently manage your data over time. +* <> +* <> +* <> +* <> +* <> -Refer to <> for more information. \ No newline at end of file +Many {es} options come with different performance considerations and trade-offs. The best way to determine the +optimal configuration for your use case is through https://www.elastic.co/elasticon/conf/2016/sf/quantitative-cluster-sizing[testing with your own data and queries]. diff --git a/docs/reference/landing-page.asciidoc b/docs/reference/landing-page.asciidoc index e781dc0aff4e3..6449a799ffd16 100644 --- a/docs/reference/landing-page.asciidoc +++ b/docs/reference/landing-page.asciidoc @@ -62,7 +62,7 @@ Elasticsearch is the search and analytics engine that powers the Elastic Stack.

- +

@@ -79,6 +79,11 @@

Get to know Elasticsearch

+

+Demos: + Hands-on learning for Search +

+

New webinar: Architect search apps with Google Cloud @@ -128,7 +133,7 @@ Adding data to Elasticsearch

  • - Connectors + Connectors
  • Web crawler diff --git a/docs/reference/licensing/delete-license.asciidoc b/docs/reference/licensing/delete-license.asciidoc index 0f0aac416ccf8..fd9236664e1c0 100644 --- a/docs/reference/licensing/delete-license.asciidoc +++ b/docs/reference/licensing/delete-license.asciidoc @@ -5,6 +5,12 @@ Delete license ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-license[Licensing APIs]. +-- + This API enables you to delete licensing information. [discrete] diff --git a/docs/reference/licensing/get-basic-status.asciidoc b/docs/reference/licensing/get-basic-status.asciidoc index 44bc246e72bc3..0e354e0dc07cd 100644 --- a/docs/reference/licensing/get-basic-status.asciidoc +++ b/docs/reference/licensing/get-basic-status.asciidoc @@ -5,6 +5,12 @@ Get basic status ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-license[Licensing APIs]. +-- + This API enables you to check the status of your basic license. [discrete] diff --git a/docs/reference/licensing/get-license.asciidoc b/docs/reference/licensing/get-license.asciidoc index 816bdd36ff4bd..1b50a5be8fa20 100644 --- a/docs/reference/licensing/get-license.asciidoc +++ b/docs/reference/licensing/get-license.asciidoc @@ -5,6 +5,12 @@ Get license ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-license[Licensing APIs]. +-- + This API enables you to retrieve licensing information. [discrete] diff --git a/docs/reference/licensing/get-trial-status.asciidoc b/docs/reference/licensing/get-trial-status.asciidoc index 912d90bde850f..b0ae110faa8cc 100644 --- a/docs/reference/licensing/get-trial-status.asciidoc +++ b/docs/reference/licensing/get-trial-status.asciidoc @@ -5,6 +5,12 @@ Get trial status ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-license[Licensing APIs]. +-- + Enables you to check the status of your trial. [discrete] diff --git a/docs/reference/licensing/index.asciidoc b/docs/reference/licensing/index.asciidoc index a1dfd398acfe7..12df7b606c5bd 100644 --- a/docs/reference/licensing/index.asciidoc +++ b/docs/reference/licensing/index.asciidoc @@ -2,6 +2,12 @@ [[licensing-apis]] == Licensing APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-license[Licensing APIs]. +-- + You can use the following APIs to manage your licenses: * <> diff --git a/docs/reference/licensing/start-basic.asciidoc b/docs/reference/licensing/start-basic.asciidoc index 8ae25e1940a62..291a0ec452caf 100644 --- a/docs/reference/licensing/start-basic.asciidoc +++ b/docs/reference/licensing/start-basic.asciidoc @@ -5,6 +5,12 @@ Start basic ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-license[Licensing APIs]. +-- + This API starts an indefinite basic license. [discrete] diff --git a/docs/reference/licensing/update-license.asciidoc b/docs/reference/licensing/update-license.asciidoc index 4207dbb092fb0..03b6c3ce40edd 100644 --- a/docs/reference/licensing/update-license.asciidoc +++ b/docs/reference/licensing/update-license.asciidoc @@ -5,6 +5,12 @@ Update license ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-license[Licensing APIs]. +-- + Updates the license for your {es} cluster. [[update-license-api-request]] diff --git a/docs/reference/mapping.asciidoc b/docs/reference/mapping.asciidoc index 239614345d782..5d6245a964104 100644 --- a/docs/reference/mapping.asciidoc +++ b/docs/reference/mapping.asciidoc @@ -75,9 +75,29 @@ reindexing. You can use runtime fields in conjunction with indexed fields to balance resource usage and performance. Your index will be smaller, but with slower search performance. +[discrete] +[[mapping-manage-update]] +== Managing and updating mappings + +Explicit mappings should be defined at index creation for fields you know in advance. +You can still add _new fields_ to mappings at any time, as your data evolves. + +Use the <> to update an existing mapping. + +In most cases, you can't change mappings for fields that are already mapped. +These changes require <>. + +However, you can _update_ mappings under certain conditions: + +* You can add new fields to an existing mapping at any time, explicitly or dynamically. +* You can add new <> for existing fields. +** Documents indexed before the mapping update will not have values for the new multi-fields until they are updated or reindexed. Documents indexed after the mapping change will automatically have values for the new multi-fields. +* Some <> can be updated for existing fields of certain <>. + [discrete] [[mapping-limit-settings]] -== Settings to prevent mapping explosion +== Prevent mapping explosions + Defining too many fields in an index can lead to a mapping explosion, which can cause out of memory errors and difficult situations to recover from. diff --git a/docs/reference/mapping/fields/synthetic-source.asciidoc b/docs/reference/mapping/fields/synthetic-source.asciidoc index ccea38cf602da..ddbefb73f4522 100644 --- a/docs/reference/mapping/fields/synthetic-source.asciidoc +++ b/docs/reference/mapping/fields/synthetic-source.asciidoc @@ -1,49 +1,53 @@ [[synthetic-source]] ==== Synthetic `_source` -IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices -(indices that have `index.mode` set to `time_series`). For other indices -synthetic `_source` is in technical preview. Features in technical preview may -be changed or removed in a future release. Elastic will work to fix -any issues, but features in technical preview are not subject to the support SLA -of official GA features. - Though very handy to have around, the source field takes up a significant amount of space on disk. Instead of storing source documents on disk exactly as you send them, Elasticsearch can reconstruct source content on the fly upon retrieval. -Enable this by setting `mode: synthetic` in `_source`: +To enable this https://www.elastic.co/subscriptions[subscription] feature, use the value `synthetic` for the index setting `index.mapping.source.mode`: [source,console,id=enable-synthetic-source-example] ---- PUT idx { - "mappings": { - "_source": { - "mode": "synthetic" + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } } } } ---- // TESTSETUP -While this on the fly reconstruction is *generally* slower than saving the source +While this on-the-fly reconstruction is _generally_ slower than saving the source documents verbatim and loading them at query time, it saves a lot of storage space. Additional latency can be avoided by not loading `_source` field in queries when it is not needed. [[synthetic-source-fields]] ===== Supported fields -Synthetic `_source` is supported by all field types. Depending on implementation details, field types have different properties when used with synthetic `_source`. +Synthetic `_source` is supported by all field types. Depending on implementation details, field types have different +properties when used with synthetic `_source`. -<> construct synthetic `_source` using existing data, most commonly <> and <>. For these field types, no additional space is needed to store the contents of `_source` field. Due to the storage layout of <>, the generated `_source` field undergoes <> compared to original document. +<> construct synthetic `_source` using existing data, most +commonly <> and <>. For these field types, no additional space +is needed to store the contents of `_source` field. Due to the storage layout of <>, the +generated `_source` field undergoes <> compared to the original document. -For all other field types, the original value of the field is stored as is, in the same way as the `_source` field in non-synthetic mode. In this case there are no modifications and field data in `_source` is the same as in the original document. Similarly, malformed values of fields that use <> or <> need to be stored as is. This approach is less storage efficient since data needed for `_source` reconstruction is stored in addition to other data required to index the field (like `doc_values`). +For all other field types, the original value of the field is stored as is, in the same way as the `_source` field in +non-synthetic mode. In this case there are no modifications and field data in `_source` is the same as in the original +document. Similarly, malformed values of fields that use <> or +<> need to be stored as is. This approach is less storage efficient since data needed for +`_source` reconstruction is stored in addition to other data required to index the field (like `doc_values`). [[synthetic-source-restrictions]] ===== Synthetic `_source` restrictions -Synthetic `_source` cannot be used together with field mappings that use <>. - -Some field types have additional restrictions. These restrictions are documented in the **synthetic `_source`** section of the field type's <>. +Some field types have additional restrictions. These restrictions are documented in the **synthetic `_source`** section +of the field type's <>. [[synthetic-source-modifications]] ===== Synthetic `_source` modifications @@ -144,6 +148,42 @@ Will become: ---- // TEST[s/^/{"_source":/ s/\n$/}/] +This impacts how source contents can be referenced in <>. For instance, referencing +a script in its original source form will return null: + +[source,js] +---- +"script": { "source": """ emit(params._source['foo.bar.baz']) """ } +---- +// NOTCONSOLE + +Instead, source references need to be in line with the mapping structure: + +[source,js] +---- +"script": { "source": """ emit(params._source['foo']['bar']['baz']) """ } +---- +// NOTCONSOLE + +or simply + +[source,js] +---- +"script": { "source": """ emit(params._source.foo.bar.baz) """ } +---- +// NOTCONSOLE + +The following <> are preferable as, in addition to being agnostic to the +mapping structure, they make use of docvalues if available and fall back to synthetic source only when needed. This +reduces source synthesizing, a slow and costly operation. + +[source,js] +---- +"script": { "source": """ emit(field('foo.bar.baz').get(null)) """ } +"script": { "source": """ emit($('foo.bar.baz', null)) """ } +---- +// NOTCONSOLE + [[synthetic-source-modifications-alphabetical]] ====== Alphabetical sorting Synthetic `_source` fields are sorted alphabetically. The @@ -155,18 +195,105 @@ that ordering. [[synthetic-source-modifications-ranges]] ====== Representation of ranges -Range field values (e.g. `long_range`) are always represented as inclusive on both sides with bounds adjusted accordingly. See <>. +Range field values (e.g. `long_range`) are always represented as inclusive on both sides with bounds adjusted +accordingly. See <>. [[synthetic-source-precision-loss-for-point-types]] ====== Reduced precision of `geo_point` values -Values of `geo_point` fields are represented in synthetic `_source` with reduced precision. See <>. +Values of `geo_point` fields are represented in synthetic `_source` with reduced precision. See +<>. + +[[synthetic-source-keep]] +====== Minimizing source modifications + +It is possible to avoid synthetic source modifications for a particular object or field, at extra storage cost. +This is controlled through param `synthetic_source_keep` with the following option: + + - `none`: synthetic source diverges from the original source as described above (default). + - `arrays`: arrays of the corresponding field or object preserve the original element ordering and duplicate elements. +The synthetic source fragment for such arrays is not guaranteed to match the original source exactly, e.g. array +`[1, 2, [5], [[4, [3]]], 5]` may appear as-is or in an equivalent format like `[1, 2, 5, 4, 3, 5]`. The exact format +may change in the future, in an effort to reduce the storage overhead of this option. +- `all`: the source for both singleton instances and arrays of the corresponding field or object gets recorded. When +applied to objects, the source of all sub-objects and sub-fields gets captured. Furthermore, the original source of +arrays gets captured and appears in synthetic source with no modifications. + +For instance: + +[source,console,id=create-index-with-synthetic-source-keep] +---- +PUT idx_keep +{ + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, + "mappings": { + "properties": { + "path": { + "type": "object", + "synthetic_source_keep": "all" + }, + "ids": { + "type": "integer", + "synthetic_source_keep": "arrays" + } + } + } +} +---- +// TEST + +[source,console,id=synthetic-source-keep-example] +---- +PUT idx_keep/_doc/1 +{ + "path": { + "to": [ + { "foo": [3, 2, 1] }, + { "foo": [30, 20, 10] } + ], + "bar": "baz" + }, + "ids": [ 200, 100, 300, 100 ] +} +---- +// TEST[s/$/\nGET idx_keep\/_doc\/1?filter_path=_source\n/] + +returns the original source, with no array deduplication and sorting: + +[source,console-result] +---- +{ + "path": { + "to": [ + { "foo": [3, 2, 1] }, + { "foo": [30, 20, 10] } + ], + "bar": "baz" + }, + "ids": [ 200, 100, 300, 100 ] +} +---- +// TEST[s/^/{"_source":/ s/\n$/}/] +The option for capturing the source of arrays can be applied at index level, by setting +`index.mapping.synthetic_source_keep` to `arrays`. This applies to all objects and fields in the index, except for +the ones with explicit overrides of `synthetic_source_keep` set to `none`. In this case, the storage overhead grows +with the number and sizes of arrays present in source of each document, naturally. [[synthetic-source-fields-native-list]] ===== Field types that support synthetic source with no storage overhead -The following field types support synthetic source using data from <> or <>, and require no additional storage space to construct the `_source` field. +The following field types support synthetic source using data from <> or +>, and require no additional storage space to construct the `_source` field. -NOTE: If you enable the <> or <> settings, then additional storage is required to store ignored field values for these types. +NOTE: If you enable the <> or <> settings, then +additional storage is required to store ignored field values for these types. ** <> ** {plugins}/mapper-annotated-text-usage.html#annotated-text-synthetic-source[`annotated-text`] diff --git a/docs/reference/mapping/params/format.asciidoc b/docs/reference/mapping/params/format.asciidoc index 5babb4def2320..29380fca83ca5 100644 --- a/docs/reference/mapping/params/format.asciidoc +++ b/docs/reference/mapping/params/format.asciidoc @@ -31,11 +31,41 @@ down to the nearest day. [[custom-date-formats]] ==== Custom date formats -Completely customizable date formats are supported. The syntax for these is explained -https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html[DateTimeFormatter docs]. +Completely customizable date formats are supported. The syntax for these is explained in +https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/time/format/DateTimeFormatter.html[DateTimeFormatter docs]. + +[[custom-date-format-locales]] +===== Differences in locale information between JDK versions + +There can be some differences in date formats between JDK versions and different locales. In particular, +there can be differences in text strings used for textual date formats, and there can be differences +in the results of week-date calculations. + +There can be differences in text strings used by the following field specifiers: + +* `B`, `E`, `G`, `O`, `a`, `v`, `z` of any length +* `L`, `M`, `Q`, `q`, `c`, `e` of length 3 or greater +* `Z` of length 4 + +If the text format changes between Elasticsearch or JDK versions, it can cause significant problems +with ingest, output, and re-indexing. It is recommended to always use numerical fields in custom date formats, +which are not affected by locale information. + +There can also be differences in week-date calculations using the `Y`, `W`, and `w` field specifiers. +The underlying data used to calculate week-dates can vary depending on the JDK version and locale; +this can cause differences in the calculated week-date for the same calendar dates. +It is recommended that the built-in week-date formats are used, which will always use ISO rules +for calculating week-dates. + +In particular, there is a significant change in locale information between JDK releases 22 and 23. +Elasticsearch will use the _COMPAT_ locale database when run on JDK 22 and before, +and will use the _CLDR_ locale database when run on JDK 23 and above. This change can cause significant differences +to the textual date formats accepted by Elasticsearch, and to calculated week-dates. If you are using +affected specifiers, you may need to modify your ingest or output integration code to account +for the differences between these two JDK versions. [[built-in-date-formats]] -==== Built In Formats +==== Built-in formats Most of the below formats have a `strict` companion format, which means that year, month and day parts of the month must use respectively 4, 2 and 2 digits @@ -256,31 +286,37 @@ The following tables lists all the defaults ISO formats supported: `week_date` or `strict_week_date`:: A formatter for a full date as four digit weekyear, two digit week of - weekyear, and one digit day of week: `xxxx-'W'ww-e`. + weekyear, and one digit day of week: `YYYY-'W'ww-e`. + This uses the ISO week-date definition. `week_date_time` or `strict_week_date_time`:: A formatter that combines a full weekyear date and time, separated by a - 'T': `xxxx-'W'ww-e'T'HH:mm:ss.SSSZ`. + 'T': `YYYY-'W'ww-e'T'HH:mm:ss.SSSZ`. + This uses the ISO week-date definition. `week_date_time_no_millis` or `strict_week_date_time_no_millis`:: A formatter that combines a full weekyear date and time without millis, - separated by a 'T': `xxxx-'W'ww-e'T'HH:mm:ssZ`. + separated by a 'T': `YYYY-'W'ww-e'T'HH:mm:ssZ`. + This uses the ISO week-date definition. `weekyear` or `strict_weekyear`:: - A formatter for a four digit weekyear: `xxxx`. + A formatter for a four digit weekyear: `YYYY`. + This uses the ISO week-date definition. `weekyear_week` or `strict_weekyear_week`:: A formatter for a four digit weekyear and two digit week of weekyear: - `xxxx-'W'ww`. + `YYYY-'W'ww`. + This uses the ISO week-date definition. `weekyear_week_day` or `strict_weekyear_week_day`:: A formatter for a four digit weekyear, two digit week of weekyear, and one - digit day of week: `xxxx-'W'ww-e`. + digit day of week: `YYYY-'W'ww-e`. + This uses the ISO week-date definition. `year` or `strict_year`:: diff --git a/docs/reference/mapping/params/ignore-above.asciidoc b/docs/reference/mapping/params/ignore-above.asciidoc index 7d04bc82dcbb3..526f2d6205961 100644 --- a/docs/reference/mapping/params/ignore-above.asciidoc +++ b/docs/reference/mapping/params/ignore-above.asciidoc @@ -57,3 +57,33 @@ NOTE: The value for `ignore_above` is the _character count_, but Lucene counts bytes. If you use UTF-8 text with many non-ASCII characters, you may want to set the limit to `32766 / 4 = 8191` since UTF-8 characters may occupy at most 4 bytes. + +[[index-mapping-ignore-above]] +=== `index.mapping.ignore_above` + +The `ignore_above` setting, typically used at the field level, can also be applied at the index level using +`index.mapping.ignore_above`. This setting lets you define a maximum string length for all applicable fields across +the index, including `keyword`, `wildcard`, and keyword values in `flattened` fields. Any values that exceed this +limit will be ignored during indexing and won’t be stored. + +This index-wide setting ensures a consistent approach to managing excessively long values. It works the same as the +field-level setting—if a string’s length goes over the specified limit, that string won’t be indexed or stored. +When dealing with arrays, each element is evaluated separately, and only the elements that exceed the limit are ignored. + +[source,console] +-------------------------------------------------- +PUT my-index-000001 +{ + "settings": { + "index.mapping.ignore_above": 256 + } +} +-------------------------------------------------- + +In this example, all applicable fields in `my-index-000001` will ignore any strings longer than 256 characters. + +TIP: You can override this index-wide setting for specific fields by specifying a custom `ignore_above` value in the +field mapping. + +NOTE: Just like the field-level `ignore_above`, this setting only affects indexing and storage. The original values +are still available in the `_source` field if `_source` is enabled, which is the default behavior in Elasticsearch. diff --git a/docs/reference/mapping/params/index-prefixes.asciidoc b/docs/reference/mapping/params/index-prefixes.asciidoc index a143c5531c81b..1d5e844467b6f 100644 --- a/docs/reference/mapping/params/index-prefixes.asciidoc +++ b/docs/reference/mapping/params/index-prefixes.asciidoc @@ -54,3 +54,30 @@ PUT my-index-000001 } } -------------------------------- + +`index_prefixes` parameter instructs {ES} to create a subfield "._index_prefix". This +field will be used to do fast prefix queries. When doing highlighting, add "._index_prefix" +subfield to the `matched_fields` parameter to highlight the main field based on the +found matches of the prefix field, like in the request below: + +[source,console] +-------------------------------- +GET my-index-000001/_search +{ + "query": { + "prefix": { + "full_name": { + "value": "ki" + } + } + }, + "highlight": { + "fields": { + "full_name": { + "matched_fields": ["full_name._index_prefix"] + } + } + } +} +-------------------------------- +// TEST[continued] diff --git a/docs/reference/mapping/params/subobjects.asciidoc b/docs/reference/mapping/params/subobjects.asciidoc index b0a5d3817c332..ff91f07cfb359 100644 --- a/docs/reference/mapping/params/subobjects.asciidoc +++ b/docs/reference/mapping/params/subobjects.asciidoc @@ -111,6 +111,7 @@ PUT my-index-000001/_doc/metric_1 The `subobjects` setting for existing fields and the top-level mapping definition cannot be updated. +[[subobjects-auto-flattening]] ==== Auto-flattening object mappings It is generally recommended to define the properties of an object that is configured with `subobjects: false` with dotted field names diff --git a/docs/reference/mapping/runtime.asciidoc b/docs/reference/mapping/runtime.asciidoc index 190081fa801b4..1ee1194279061 100644 --- a/docs/reference/mapping/runtime.asciidoc +++ b/docs/reference/mapping/runtime.asciidoc @@ -821,8 +821,6 @@ address. [[lookup-runtime-fields]] ==== Retrieve fields from related indices -experimental[] - The <> parameter on the `_search` API can also be used to retrieve fields from the related indices via runtime fields with a type of `lookup`. diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc index 7e2e7083fa70b..e5155b7d4ce5b 100644 --- a/docs/reference/mapping/types.asciidoc +++ b/docs/reference/mapping/types.asciidoc @@ -35,12 +35,13 @@ Dates:: Date types, including <> and [[object-types]] ==== Objects and relational types -<>:: A JSON object. -<>:: An entire JSON object as a single field value. -<>:: A JSON object that preserves the relationship - between its subfields. -<>:: Defines a parent/child relationship for documents - in the same index. +<>:: A JSON object. +<>:: An entire JSON object as a single field value. +<>:: A JSON object that preserves the relationship + between its subfields. +<>:: Defines a parent/child relationship for documents + in the same index. +<>:: Provides aliases for sub-fields at the same level. [discrete] @@ -167,6 +168,8 @@ include::types/numeric.asciidoc[] include::types/object.asciidoc[] +include::types/passthrough.asciidoc[] + include::types/percolator.asciidoc[] include::types/point.asciidoc[] @@ -177,6 +180,8 @@ include::types/rank-feature.asciidoc[] include::types/rank-features.asciidoc[] +include::types/rank-vectors.asciidoc[] + include::types/search-as-you-type.asciidoc[] include::types/semantic-text.asciidoc[] diff --git a/docs/reference/mapping/types/aggregate-metric-double.asciidoc b/docs/reference/mapping/types/aggregate-metric-double.asciidoc index 8e14fba976360..faae5118e42bb 100644 --- a/docs/reference/mapping/types/aggregate-metric-double.asciidoc +++ b/docs/reference/mapping/types/aggregate-metric-double.asciidoc @@ -259,16 +259,21 @@ be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features. -`aggregate_metric-double` fields support <> in their default -configuration. - For example: [source,console,id=synthetic-source-aggregate-metric-double-example] ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "agg_metric": { "type": "aggregate_metric_double", diff --git a/docs/reference/mapping/types/binary.asciidoc b/docs/reference/mapping/types/binary.asciidoc index a06e5b4f572e0..5733a28eb711a 100644 --- a/docs/reference/mapping/types/binary.asciidoc +++ b/docs/reference/mapping/types/binary.asciidoc @@ -63,7 +63,7 @@ be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features. -`binary` fields support <> only when <> are enabled. Synthetic source always sorts `binary` values in order of their byte representation. For example: +Synthetic source may sort `binary` values in order of their byte representation. For example: [source,console,id=synthetic-source-binary-example] ---- PUT idx diff --git a/docs/reference/mapping/types/boolean.asciidoc b/docs/reference/mapping/types/boolean.asciidoc index 32f3d13edf581..268be9016987f 100644 --- a/docs/reference/mapping/types/boolean.asciidoc +++ b/docs/reference/mapping/types/boolean.asciidoc @@ -241,16 +241,23 @@ any issues, but features in technical preview are not subject to the support SLA of official GA features. `boolean` fields support <> in their -default configuration. Synthetic `_source` cannot be used together with -<> or with <> disabled. +default configuration. -Synthetic source always sorts `boolean` fields. For example: +Synthetic source may sort `boolean` field values. For example: [source,console,id=synthetic-source-boolean-example] ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "bool": { "type": "boolean" } } diff --git a/docs/reference/mapping/types/date.asciidoc b/docs/reference/mapping/types/date.asciidoc index a29db79167d2e..32b0278fccd5b 100644 --- a/docs/reference/mapping/types/date.asciidoc +++ b/docs/reference/mapping/types/date.asciidoc @@ -81,6 +81,14 @@ on those dates so they should be avoided. // end::decimal-warning[] ==== +[WARNING] +==== +// tag::locale-warning[] +The text strings accepted by textual date formats, and calculations for week-dates, depend on the JDK version +that Elasticsearch is running on. For more information see <>. +// end::locale-warning[] +==== + [[multiple-date-formats]] ==== Multiple date formats @@ -125,13 +133,12 @@ The following parameters are accepted by `date` fields: `locale`:: The locale to use when parsing dates since months do not have the same names - and/or abbreviations in all languages. The default is the - https://docs.oracle.com/javase/8/docs/api/java/util/Locale.html#ROOT[`ROOT` locale], + and/or abbreviations in all languages. The default is ENGLISH. <>:: If `true`, malformed numbers are ignored. If `false` (default), malformed - numbers throw an exception and reject the whole document. Note that this + numbers throw an exception and reject the whole document. Note that this cannot be set if the `script` parameter is used. <>:: @@ -240,17 +247,21 @@ be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features. -`date` fields support <> in their -default configuration. Synthetic `_source` cannot be used together with -<> or with <> disabled. - -Synthetic source always sorts `date` fields. For example: +Synthetic source may sort `date` field values. For example: [source,console,id=synthetic-source-date-example] ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "date": { "type": "date" } } diff --git a/docs/reference/mapping/types/date_nanos.asciidoc b/docs/reference/mapping/types/date_nanos.asciidoc index 1a3b390b1690c..31f5ae09e7a63 100644 --- a/docs/reference/mapping/types/date_nanos.asciidoc +++ b/docs/reference/mapping/types/date_nanos.asciidoc @@ -150,18 +150,21 @@ be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features. -`date_nanos` fields support <> in their -default configuration. Synthetic `_source` cannot be used together with -<>, <> set to true -or with <> disabled. - -Synthetic source always sorts `date_nanos` fields. For example: +Synthetic source may sort `date_nanos` field values. For example: [source,console,id=synthetic-source-date-nanos-example] ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "date": { "type": "date_nanos" } } diff --git a/docs/reference/mapping/types/dense-vector.asciidoc b/docs/reference/mapping/types/dense-vector.asciidoc index 0cd9ee0578b70..b56704b896706 100644 --- a/docs/reference/mapping/types/dense-vector.asciidoc +++ b/docs/reference/mapping/types/dense-vector.asciidoc @@ -1,4 +1,3 @@ -[role="xpack"] [[dense-vector]] === Dense vector field type ++++ @@ -115,22 +114,25 @@ that sacrifices result accuracy for improved speed. ==== Automatically quantize vectors for kNN search The `dense_vector` type supports quantization to reduce the memory footprint required when <> `float` vectors. -The two following quantization strategies are supported: +The three following quantization strategies are supported: + +* `int8` - Quantizes each dimension of the vector to 1-byte integers. This reduces the memory footprint by 75% (or 4x) at the cost of some accuracy. +* `int4` - Quantizes each dimension of the vector to half-byte integers. This reduces the memory footprint by 87% (or 8x) at the cost of accuracy. +* `bbq` - experimental:[] Better binary quantization which reduces each dimension to a single bit precision. This reduces the memory footprint by 96% (or 32x) at a larger cost of accuracy. Generally, oversampling during query time and reranking can help mitigate the accuracy loss. -+ --- -`int8` - Quantizes each dimension of the vector to 1-byte integers. This can reduce the memory footprint by 75% at the cost of some accuracy. -`int4` - Quantizes each dimension of the vector to half-byte integers. This can reduce the memory footprint by 87% at the cost of some accuracy. --- -To use a quantized index, you can set your index type to `int8_hnsw` or `int4_hnsw`. When indexing `float` vectors, the current default +When using a quantized format, you may want to oversample and rescore the results to improve accuracy. See <> for more information. + +To use a quantized index, you can set your index type to `int8_hnsw`, `int4_hnsw`, or `bbq_hnsw`. When indexing `float` vectors, the current default index type is `int8_hnsw`. NOTE: Quantization will continue to keep the raw float vector values on disk for reranking, reindexing, and quantization improvements over the lifetime of the data. -This means disk usage will increase by ~25% for `int8` and ~12.5% for `int4` due to the overhead of storing the quantized and raw vectors. +This means disk usage will increase by ~25% for `int8`, ~12.5% for `int4`, and ~3.1% for `bbq` due to the overhead of storing the quantized and raw vectors. NOTE: `int4` quantization requires an even number of vector dimensions. +NOTE: experimental:[] `bbq` quantization only supports vector dimensions that are greater than 64. + Here is an example of how to create a byte-quantized index: [source,console] @@ -173,6 +175,27 @@ PUT my-byte-quantized-index } -------------------------------------------------- +experimental:[] Here is an example of how to create a binary quantized index: + +[source,console] +-------------------------------------------------- +PUT my-byte-quantized-index +{ + "mappings": { + "properties": { + "my_vector": { + "type": "dense_vector", + "dims": 64, + "index": true, + "index_options": { + "type": "bbq_hnsw" + } + } + } + } +} +-------------------------------------------------- + [role="child_attributes"] [[dense-vector-params]] ==== Parameters for dense vector fields @@ -219,12 +242,11 @@ their vector field's similarity to the query vector. The `_score` of each document will be derived from the similarity, in a way that ensures scores are positive and that a larger score corresponds to a higher ranking. Defaults to `l2_norm` when `element_type: bit` otherwise defaults to `cosine`. - -NOTE: `bit` vectors only support `l2_norm` as their similarity metric. - + ^*^ This parameter can only be specified when `index` is `true`. + +NOTE: `bit` vectors only support `l2_norm` as their similarity metric. + .Valid values for `similarity` [%collapsible%open] ==== @@ -301,21 +323,26 @@ by 4x at the cost of some accuracy. See <>. +* experimental:[] `bbq_hnsw` - This utilizes the https://arxiv.org/abs/1603.09320[HNSW algorithm] in addition to automatically binary +quantization for scalable approximate kNN search with `element_type` of `float`. This can reduce the memory footprint +by 32x at the cost of accuracy. See <>. * `flat` - This utilizes a brute-force search algorithm for exact kNN search. This supports all `element_type` values. * `int8_flat` - This utilizes a brute-force search algorithm in addition to automatically scalar quantization. Only supports `element_type` of `float`. * `int4_flat` - This utilizes a brute-force search algorithm in addition to automatically half-byte scalar quantization. Only supports `element_type` of `float`. +* experimental:[] `bbq_flat` - This utilizes a brute-force search algorithm in addition to automatically binary quantization. Only supports +`element_type` of `float`. -- `m`::: (Optional, integer) The number of neighbors each node will be connected to in the HNSW graph. -Defaults to `16`. Only applicable to `hnsw`, `int8_hnsw`, and `int4_hnsw` index types. +Defaults to `16`. Only applicable to `hnsw`, `int8_hnsw`, `int4_hnsw` and `bbq_hnsw` index types. `ef_construction`::: (Optional, integer) The number of candidates to track while assembling the list of nearest -neighbors for each new node. Defaults to `100`. Only applicable to `hnsw`, `int8_hnsw`, and `int4_hnsw` index types. +neighbors for each new node. Defaults to `100`. Only applicable to `hnsw`, `int8_hnsw`, `int4_hnsw` and `bbq_hnsw` index types. `confidence_interval`::: (Optional, float) diff --git a/docs/reference/mapping/types/flattened.asciidoc b/docs/reference/mapping/types/flattened.asciidoc index 0a72ebc98ecef..96b230794003a 100644 --- a/docs/reference/mapping/types/flattened.asciidoc +++ b/docs/reference/mapping/types/flattened.asciidoc @@ -325,17 +325,24 @@ any issues, but features in technical preview are not subject to the support SLA of official GA features. Flattened fields support <> in their default -configuration. Synthetic `_source` cannot be used with <> -disabled. +configuration. -Synthetic source always sorts alphabetically and de-duplicates flattened fields. +Synthetic source may sort `flattened` field values and remove duplicates. For example: [source,console,id=synthetic-source-flattened-sorting-example] ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "flattened": { "type": "flattened" } } @@ -367,8 +374,16 @@ For example: ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "flattened": { "type": "flattened" } } @@ -407,8 +422,16 @@ For example: ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "flattened": { "type": "flattened" } } diff --git a/docs/reference/mapping/types/geo-point.asciidoc b/docs/reference/mapping/types/geo-point.asciidoc index 6db05188dfb98..0958997d3fb00 100644 --- a/docs/reference/mapping/types/geo-point.asciidoc +++ b/docs/reference/mapping/types/geo-point.asciidoc @@ -219,18 +219,22 @@ be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features. -`geo_point` fields support <> in their -default configuration. Synthetic `_source` cannot be used together with <> or with -<> disabled. - -Synthetic source always sorts `geo_point` fields (first by latitude and then +Synthetic source may sort `geo_point` fields (first by latitude and then longitude) and reduces them to their stored precision. For example: [source,console,id=synthetic-source-geo-point-example] ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "point": { "type": "geo_point" } } diff --git a/docs/reference/mapping/types/geo-shape.asciidoc b/docs/reference/mapping/types/geo-shape.asciidoc index e50c7d73b1b76..affebc6f721e4 100644 --- a/docs/reference/mapping/types/geo-shape.asciidoc +++ b/docs/reference/mapping/types/geo-shape.asciidoc @@ -502,6 +502,3 @@ synthetic `_source` is in technical preview. Features in technical preview may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features. - -`geo_shape` fields support <> in their -default configuration. diff --git a/docs/reference/mapping/types/histogram.asciidoc b/docs/reference/mapping/types/histogram.asciidoc index 8cd30110250bf..cdebe97000d68 100644 --- a/docs/reference/mapping/types/histogram.asciidoc +++ b/docs/reference/mapping/types/histogram.asciidoc @@ -79,7 +79,7 @@ any issues, but features in technical preview are not subject to the support SLA of official GA features. `histogram` fields support <> in their -default configuration. Synthetic `_source` cannot be used together with <>. +default configuration. NOTE: To save space, zero-count buckets are not stored in the histogram doc values. As a result, when indexing a histogram field in an index with synthetic source enabled, diff --git a/docs/reference/mapping/types/ip.asciidoc b/docs/reference/mapping/types/ip.asciidoc index f068916478a78..bafc25a977caa 100644 --- a/docs/reference/mapping/types/ip.asciidoc +++ b/docs/reference/mapping/types/ip.asciidoc @@ -161,17 +161,21 @@ be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features. -`ip` fields support <> in their default -configuration. Synthetic `_source` cannot be used together with -<> or with <> disabled. - -Synthetic source always sorts `ip` fields and removes duplicates. For example: +Synthetic source may sort `ip` field values and remove duplicates. For example: [source,console,id=synthetic-source-ip-example] ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "ip": { "type": "ip" } } diff --git a/docs/reference/mapping/types/keyword.asciidoc b/docs/reference/mapping/types/keyword.asciidoc index 59d307c4df0ad..165d9d7900441 100644 --- a/docs/reference/mapping/types/keyword.asciidoc +++ b/docs/reference/mapping/types/keyword.asciidoc @@ -163,7 +163,6 @@ index setting limits the number of dimensions in an index. Dimension fields have the following constraints: * The `doc_values` and `index` mapping parameters must be `true`. -* Field values cannot be an <>. // end::dimension[] * Dimension values are used to identify a document’s time series. If dimension values are altered in any way during indexing, the document will be stored as belonging to different from intended time series. As a result there are additional constraints: ** The field cannot use a <>. @@ -179,18 +178,22 @@ be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features. -`keyword` fields support <> in their -default configuration. Synthetic `_source` cannot be used together with -a <> or <>. - -By default, synthetic source sorts `keyword` fields and removes duplicates. +Synthetic source may sort `keyword` fields and remove duplicates. For example: [source,console,id=synthetic-source-keyword-example-default] ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "kwd": { "type": "keyword" } } @@ -219,8 +222,16 @@ are preserved. For example: ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "kwd": { "type": "keyword", "store": true } } @@ -249,8 +260,16 @@ For example: ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "kwd": { "type": "keyword", "ignore_above": 3 } } diff --git a/docs/reference/mapping/types/numeric.asciidoc b/docs/reference/mapping/types/numeric.asciidoc index d1e1c037e571e..2fba1931a2a29 100644 --- a/docs/reference/mapping/types/numeric.asciidoc +++ b/docs/reference/mapping/types/numeric.asciidoc @@ -254,13 +254,21 @@ All numeric fields support <>, or with <> disabled. -Synthetic source always sorts numeric fields. For example: +Synthetic source may sort numeric field values. For example: [source,console,id=synthetic-source-numeric-example] ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "long": { "type": "long" } } @@ -287,8 +295,16 @@ Scaled floats will always apply their scaling factor so: ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "f": { "type": "scaled_float", "scaling_factor": 0.01 } } diff --git a/docs/reference/mapping/types/passthrough.asciidoc b/docs/reference/mapping/types/passthrough.asciidoc new file mode 100644 index 0000000000000..f4f1945d21537 --- /dev/null +++ b/docs/reference/mapping/types/passthrough.asciidoc @@ -0,0 +1,218 @@ +[[passthrough]] +=== Pass-through object field type +++++ +Pass-through object +++++ + +Pass-through objects extend the functionality of <> by allowing to access +their subfields without including the name of the pass-through object as prefix. For instance: + +[source,console] +-------------------------------------------------- +PUT my-index-000001 +{ + "mappings": { + "properties": { + "attributes": { + "type": "passthrough", <1> + "priority": 10, + "properties": { + "id": { + "type": "keyword" + } + } + } + } + } +} + +PUT my-index-000001/_doc/1 +{ + "attributes" : { <2> + "id": "foo", + "zone": 10 + } +} + +GET my-index-000001/_search +{ + "query": { + "bool": { + "must": [ + { "match": { "id": "foo" }}, <3> + { "match": { "zone": 10 }} + ] + } + } +} + +GET my-index-000001/_search +{ + "query": { + "bool": { + "must": [ + { "match": { "attributes.id": "foo" }}, <4> + { "match": { "attributes.zone": 10 }} + ] + } + } +} + +-------------------------------------------------- + +<1> An object is defined as pass-through. Its priority (required) is used for conflict resolution. +<2> Object contents get indexed as usual, including dynamic mappings. +<3> Sub-fields can be referenced in queries as if they're defined at the root level. +<4> Sub-fields can also be referenced including the object name as prefix. + +[[passthrough-conflicts]] +==== Conflict resolution + +It's possible for conflicting names to arise, for fields that are defined within different scopes: + + a. A pass-through object is defined next to a field that has the same name as one of the pass-through object +sub-fields, e.g. ++ +[source,console] +-------------------------------------------------- +PUT my-index-000001/_doc/1 +{ + "attributes" : { + "id": "foo" + }, + "id": "bar" +} +-------------------------------------------------- ++ +In this case, references to `id` point to the field at the root level, while field `attributes.id` +can only be accessed using the full path. + + b. Two (or more) pass-through objects are defined within the same object and contain fields with the same name, e.g. ++ +[source,console] +-------------------------------------------------- +PUT my-index-000002 +{ + "mappings": { + "properties": { + "attributes": { + "type": "passthrough", + "priority": 10, + "properties": { + "id": { + "type": "keyword" + } + } + }, + "resource.attributes": { + "type": "passthrough", + "priority": 20, + "properties": { + "id": { + "type": "keyword" + } + } + } + } + } +} +-------------------------------------------------- ++ +In this case, param `priority` is used for conflict resolution, with the higher values taking precedence. In the +example above, `resource.attributes` has higher priority than `attributes`, so references to `id` point to the field +within `resource.attributes`. `attributes.id` can still be accessed using its full path. + +[[passthrough-dimensions]] +==== Defining sub-fields as time-series dimensions + +It is possible to configure a pass-through field as a container for <>. +In this case, all sub-fields get annotated with the same parameter under the covers, and they're also +included in <> and <> calculations, thus simplifying +the <> setup: + +[source,console] +-------------------------------------------------- +PUT _index_template/my-metrics +{ + "index_patterns": ["metrics-mymetrics-*"], + "priority": 200, + "data_stream": { }, + "template": { + "settings": { + "index.mode": "time_series" + }, + "mappings": { + "properties": { + "attributes": { + "type": "passthrough", + "priority": 10, + "time_series_dimension": true, + "properties": { + "host.name": { + "type": "keyword" + } + } + }, + "cpu": { + "type": "integer", + "time_series_metric": "counter" + } + } + } + } +} + +POST metrics-mymetrics-test/_doc +{ + "@timestamp": "2020-01-01T00:00:00.000Z", + "attributes" : { + "host.name": "foo", + "zone": "bar" + }, + "cpu": 10 +} +-------------------------------------------------- +// TEST[skip: The @timestamp value won't match an accepted range in the TSDS] + +In the example above, `attributes` is defined as a dimension container. Its sub-fields `host.name` (static) and `zone` +(dynamic) get included in the routing path and tsid, and can be referenced in queries without the `attributes.` prefix. + +[[passthrough-flattening]] +==== Sub-field auto-flattening + +Pass-through fields apply <> to sub-fields by default, to reduce dynamic +mapping conflicts. As a consequence, no sub-object definitions are allowed within pass-through fields. + +[[passthrough-params]] +==== Parameters for `passthrough` fields + +The following parameters are accepted by `passthrough` fields: + +[horizontal] + +<>:: + + (Required) used for naming conflict resolution between pass-through fields. The field with the highest value wins. + Accepts non-negative integer values. + +<>:: + + Whether or not to treat sub-fields as <>. + Accepts `false` (default) or `true`. + +<>:: + + Whether or not new `properties` should be added dynamically to an existing object. + Accepts `true` (default), `runtime`, `false` and `strict`. + +<>:: + + Whether the JSON value given for the object field should be parsed and indexed (`true`, default) + or completely ignored (`false`). + +<>:: + + The fields within the object, which can be of any <>, including `object`. + New properties may be added to an existing object. + +IMPORTANT: If you need to index arrays of objects instead of single objects, read <> first. diff --git a/docs/reference/mapping/types/range.asciidoc b/docs/reference/mapping/types/range.asciidoc index fa52722c4c7d1..3b31a1885e5b9 100644 --- a/docs/reference/mapping/types/range.asciidoc +++ b/docs/reference/mapping/types/range.asciidoc @@ -247,15 +247,23 @@ any issues, but features in technical preview are not subject to the support SLA of official GA features. `range` fields support <> in their default -configuration. Synthetic `_source` cannot be used with <> disabled. +configuration. -Synthetic source always sorts values and removes duplicates for all `range` fields except `ip_range` . Ranges are sorted by their lower bound and then by upper bound. For example: +Synthetic source may sort `range` field values and remove duplicates for all `range` fields except `ip_range`. Ranges are sorted by their lower bound and then by upper bound. For example: [source,console,id=synthetic-source-range-sorting-example] ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "my_range": { "type": "long_range" } } @@ -316,8 +324,16 @@ For example: ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "my_range": { "type": "ip_range" } } @@ -352,13 +368,21 @@ Will become: // TEST[s/^/{"_source":/ s/\n$/}/] [[range-synthetic-source-inclusive]] -Range field vales are always represented as inclusive on both sides with bounds adjusted accordingly. Default values for range bounds are represented as `null`. This is true even if range bound was explicitly provided. For example: +Range field values are always represented as inclusive on both sides with bounds adjusted accordingly. Default values for range bounds are represented as `null`. This is true even if range bound was explicitly provided. For example: [source,console,id=synthetic-source-range-normalization-example] ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "my_range": { "type": "long_range" } } @@ -394,8 +418,16 @@ Default values for range bounds are represented as `null` in synthetic source. T ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "my_range": { "type": "integer_range" } } @@ -429,8 +461,16 @@ Will become: ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "my_range": { "type": "date_range" } } @@ -465,7 +505,7 @@ Will become: }, { "gte": "2017-09-01T00:00:00.000Z", - "lte": "2017-09-10T00:00:00.000Z" + "lte": "2017-09-10T23:59:59.999Z" } ] } diff --git a/docs/reference/mapping/types/rank-vectors.asciidoc b/docs/reference/mapping/types/rank-vectors.asciidoc new file mode 100644 index 0000000000000..a718a5e47ec85 --- /dev/null +++ b/docs/reference/mapping/types/rank-vectors.asciidoc @@ -0,0 +1,201 @@ +[role="xpack"] +[[rank-vectors]] +=== Rank Vectors +++++ + Rank Vectors +++++ +experimental::[] + +The `rank_vectors` field type enables late-interaction dense vector scoring in Elasticsearch. The number of vectors +per field can vary, but they must all share the same number of dimensions and element type. + +The purpose of vectors stored in this field is second order ranking documents with max-sim similarity. + +Here is a simple example of using this field with `float` elements. + +[source,console] +-------------------------------------------------- +PUT my-rank-vectors-float +{ + "mappings": { + "properties": { + "my_vector": { + "type": "rank_vectors" + } + } + } +} + +PUT my-rank-vectors-float/_doc/1 +{ + "my_vector" : [[0.5, 10, 6], [-0.5, 10, 10]] +} + +-------------------------------------------------- +// TESTSETUP + +In addition to the `float` element type, `byte` and `bit` element types are also supported. + +Here is an example of using this field with `byte` elements. + +[source,console] +-------------------------------------------------- +PUT my-rank-vectors-byte +{ + "mappings": { + "properties": { + "my_vector": { + "type": "rank_vectors", + "element_type": "byte" + } + } + } +} + +PUT my-rank-vectors-byte/_doc/1 +{ + "my_vector" : [[1, 2, 3], [4, 5, 6]] +} +-------------------------------------------------- + +Here is an example of using this field with `bit` elements. + +[source,console] +-------------------------------------------------- +PUT my-rank-vectors-bit +{ + "mappings": { + "properties": { + "my_vector": { + "type": "rank_vectors", + "element_type": "bit" + } + } + } +} + +POST /my-rank-vectors-bit/_bulk?refresh +{"index": {"_id" : "1"}} +{"my_vector": [127, -127, 0, 1, 42]} +{"index": {"_id" : "2"}} +{"my_vector": "8100012a7f"} +-------------------------------------------------- + +[role="child_attributes"] +[[rank-vectors-params]] +==== Parameters for rank vectors fields + +The `rank_vectors` field type supports the following parameters: + +[[rank-vectors-element-type]] +`element_type`:: +(Optional, string) +The data type used to encode vectors. The supported data types are +`float` (default), `byte`, and bit. + +.Valid values for `element_type` +[%collapsible%open] +==== +`float`::: +indexes a 4-byte floating-point +value per dimension. This is the default value. + +`byte`::: +indexes a 1-byte integer value per dimension. + +`bit`::: +indexes a single bit per dimension. Useful for very high-dimensional vectors or models that specifically support bit vectors. +NOTE: when using `bit`, the number of dimensions must be a multiple of 8 and must represent the number of bits. + +==== + +`dims`:: +(Optional, integer) +Number of vector dimensions. Can't exceed `4096`. If `dims` is not specified, +it will be set to the length of the first vector added to the field. + +[[rank-vectors-synthetic-source]] +==== Synthetic `_source` + +IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices +(indices that have `index.mode` set to `time_series`). For other indices +synthetic `_source` is in technical preview. Features in technical preview may +be changed or removed in a future release. Elastic will work to fix +any issues, but features in technical preview are not subject to the support SLA +of official GA features. + +`rank_vectors` fields support <> . + +[[rank-vectors-scoring]] +==== Scoring with rank vectors + +Rank vectors can be accessed and used in <>. + +For example, the following query scores documents based on the maxSim similarity between the query vector and the vectors stored in the `my_vector` field: + +[source,console] +-------------------------------------------------- +GET my-rank-vectors-float/_search +{ + "query": { + "script_score": { + "query": { + "match_all": {} + }, + "script": { + "source": "maxSimDotProduct(params.query_vector, 'my_vector')", + "params": { + "query_vector": [[0.5, 10, 6], [-0.5, 10, 10]] + } + } + } + } +} +-------------------------------------------------- + +Additionally, asymmetric similarity functions can be used to score against `bit` vectors. For example, the following query scores documents based on the maxSimDotProduct similarity between a floating point query vector and bit vectors stored in the `my_vector` field: + +[source,console] +-------------------------------------------------- +PUT my-rank-vectors-bit +{ + "mappings": { + "properties": { + "my_vector": { + "type": "rank_vectors", + "element_type": "bit" + } + } + } +} + +POST /my-rank-vectors-bit/_bulk?refresh +{"index": {"_id" : "1"}} +{"my_vector": [127, -127, 0, 1, 42]} +{"index": {"_id" : "2"}} +{"my_vector": "8100012a7f"} + +GET my-rank-vectors-bit/_search +{ + "query": { + "script_score": { + "query": { + "match_all": {} + }, + "script": { + "source": "maxSimDotProduct(params.query_vector, 'my_vector')", + "params": { + "query_vector": [ + [0.35, 0.77, 0.95, 0.15, 0.11, 0.08, 0.58, 0.06, 0.44, 0.52, 0.21, + 0.62, 0.65, 0.16, 0.64, 0.39, 0.93, 0.06, 0.93, 0.31, 0.92, 0.0, + 0.66, 0.86, 0.92, 0.03, 0.81, 0.31, 0.2 , 0.92, 0.95, 0.64, 0.19, + 0.26, 0.77, 0.64, 0.78, 0.32, 0.97, 0.84] + ] <1> + } + } + } + } +} +-------------------------------------------------- +<1> Note that the query vector has 40 elements, matching the number of bits in the bit vectors. + diff --git a/docs/reference/mapping/types/search-as-you-type.asciidoc b/docs/reference/mapping/types/search-as-you-type.asciidoc index c0bdc75f13392..c2673a614c265 100644 --- a/docs/reference/mapping/types/search-as-you-type.asciidoc +++ b/docs/reference/mapping/types/search-as-you-type.asciidoc @@ -97,11 +97,21 @@ GET my-index-000001/_search "my_field._3gram" ] } + }, + "highlight": { + "fields": { + "my_field": { + "matched_fields": ["my_field._index_prefix"] <1> + } + } } } -------------------------------------------------- // TEST[continued] +<1> Adding "my_field._index_prefix" to the `matched_fields` allows to highlight + "my_field" also based on matches from "my_field._index_prefix" field. + [source,console-result] -------------------------------------------------- { @@ -126,6 +136,11 @@ GET my-index-000001/_search "_score" : 0.8630463, "_source" : { "my_field" : "quick brown fox jump lazy dog" + }, + "highlight": { + "my_field": [ + "quick brown fox jump lazy dog" + ] } } ] @@ -266,5 +281,4 @@ any issues, but features in technical preview are not subject to the support SLA of official GA features. `search_as_you_type` fields support <> in their -default configuration. Synthetic `_source` cannot be used together with -<>. +default configuration. diff --git a/docs/reference/mapping/types/semantic-text.asciidoc b/docs/reference/mapping/types/semantic-text.asciidoc index a006f288dc66d..96dc402e10c60 100644 --- a/docs/reference/mapping/types/semantic-text.asciidoc +++ b/docs/reference/mapping/types/semantic-text.asciidoc @@ -12,27 +12,68 @@ Long passages are <> to smaller secti The `semantic_text` field type specifies an inference endpoint identifier that will be used to generate embeddings. You can create the inference endpoint by using the <>. -This field type and the <> type make it simpler to perform semantic search on your data. +This field type and the <> type make it simpler to perform semantic search on your data. -Using `semantic_text`, you won't need to specify how to generate embeddings for -your data, or how to index it. The inference endpoint automatically determines -the embedding generation, indexing, and query to use. +If you don’t specify an inference endpoint, the `inference_id` field defaults to `.elser-2-elasticsearch`, a preconfigured endpoint for the elasticsearch service. + +Using `semantic_text`, you won't need to specify how to generate embeddings for your data, or how to index it. +The {infer} endpoint automatically determines the embedding generation, indexing, and query to use. + +If you use the preconfigured `.elser-2-elasticsearch` endpoint, you can set up `semantic_text` with the following API request: [source,console] ------------------------------------------------------------ PUT my-index-000001 +{ + "mappings": { + "properties": { + "inference_field": { + "type": "semantic_text" + } + } + } +} +------------------------------------------------------------ + +To use a custom {infer} endpoint instead of the default `.elser-2-elasticsearch`, you must <> and specify its `inference_id` when setting up the `semantic_text` field type. + +[source,console] +------------------------------------------------------------ +PUT my-index-000002 { "mappings": { "properties": { "inference_field": { "type": "semantic_text", - "inference_id": "my-elser-endpoint" + "inference_id": "my-openai-endpoint" <1> } } } } ------------------------------------------------------------ -// TEST[skip:TBD] +// TEST[skip:Requires inference endpoint] +<1> The `inference_id` of the {infer} endpoint to use to generate embeddings. + +The recommended way to use `semantic_text` is by having dedicated {infer} endpoints for ingestion and search. +This ensures that search speed remains unaffected by ingestion workloads, and vice versa. +After creating dedicated {infer} endpoints for both, you can reference them using the `inference_id` and `search_inference_id` parameters when setting up the index mapping for an index that uses the `semantic_text` field. + +[source,console] +------------------------------------------------------------ +PUT my-index-000003 +{ + "mappings": { + "properties": { + "inference_field": { + "type": "semantic_text", + "inference_id": "my-elser-endpoint-for-ingest", + "search_inference_id": "my-elser-endpoint-for-search" + } + } + } +} +------------------------------------------------------------ +// TEST[skip:Requires inference endpoint] [discrete] @@ -41,9 +82,18 @@ PUT my-index-000001 `inference_id`:: (Required, string) -Inference endpoint that will be used to generate the embeddings for the field. +{infer-cap} endpoint that will be used to generate embeddings for the field. +By default, `.elser-2-elasticsearch` is used. +This parameter cannot be updated. Use the <> to create the endpoint. +If `search_inference_id` is specified, the {infer} endpoint will only be used at index time. +`search_inference_id`:: +(Optional, string) +{infer-cap} endpoint that will be used to generate embeddings at query time. +You can update this parameter by using the <>. +Use the <> to create the endpoint. +If not specified, the {infer} endpoint defined by `inference_id` will be used at both index and query time. [discrete] [[infer-endpoint-validation]] @@ -55,58 +105,51 @@ When the first document is indexed, the `inference_id` will be used to generate WARNING: Removing an {infer} endpoint will cause ingestion of documents and semantic queries to fail on indices that define `semantic_text` fields with that {infer} endpoint as their `inference_id`. Trying to <> that is used on a `semantic_text` field will result in an error. + [discrete] [[auto-text-chunking]] -==== Automatic text chunking +==== Text chunking {infer-cap} endpoints have a limit on the amount of text they can process. To allow for large amounts of text to be used in semantic search, `semantic_text` automatically generates smaller passages if needed, called _chunks_. -Each chunk will include the text subpassage and the corresponding embedding generated from it. +Each chunk refers to a passage of the text and the corresponding embedding generated from it. When querying, the individual passages will be automatically searched for each document, and the most relevant passage will be used to compute a score. -Documents are split into 250-word sections with a 100-word overlap so that each section shares 100 words with the previous section. -This overlap ensures continuity and prevents vital contextual information in the input text from being lost by a hard break. +For more details on chunking and how to configure chunking settings, see <> in the Inference API documentation. +Refer to <> to learn more about +semantic search using `semantic_text` and the `semantic` query. [discrete] -[[semantic-text-structure]] -==== `semantic_text` structure +[[semantic-text-highlighting]] +==== Extracting Relevant Fragments from Semantic Text -Once a document is ingested, a `semantic_text` field will have the following structure: +You can extract the most relevant fragments from a semantic text field by using the <> in the <>. -[source,console-result] +[source,console] ------------------------------------------------------------ -"inference_field": { - "text": "these are not the droids you're looking for", <1> - "inference": { - "inference_id": "my-elser-endpoint", <2> - "model_settings": { <3> - "task_type": "sparse_embedding" +PUT test-index +{ + "query": { + "semantic": { + "field": "my_semantic_field" + } }, - "chunks": [ <4> - { - "text": "these are not the droids you're looking for", - "embeddings": { - (...) + "highlight": { + "fields": { + "my_semantic_field": { + "type": "semantic", + "number_of_fragments": 2, <1> + "order": "score" <2> + } } - } - ] - } + } } ------------------------------------------------------------ -// TEST[skip:TBD] -<1> The field will become an object structure to accommodate both the original -text and the inference results. -<2> The `inference_id` used to generate the embeddings. -<3> Model settings, including the task type and dimensions/similarity if -applicable. -<4> Inference results will be grouped in chunks, each with its corresponding -text and embeddings. - -Refer to <> to learn more about -semantic search using `semantic_text` and the `semantic` query. - +// TEST[skip:Requires inference endpoint] +<1> Specifies the maximum number of fragments to return. +<2> Sorts highlighted fragments by score when set to `score`. By default, fragments will be output in the order they appear in the field (order: none). [discrete] [[custom-indexing]] @@ -159,7 +202,7 @@ PUT test-index "properties": { "infer_field": { "type": "semantic_text", - "inference_id": "my-elser-endpoint" + "inference_id": ".elser-2-elasticsearch" }, "source_field": { "type": "text", @@ -183,6 +226,7 @@ PUT test-index/_bulk Notice that both the `semantic_text` field and the source field are updated in the bulk request. + [discrete] [[limitations]] ==== Limitations @@ -190,4 +234,5 @@ Notice that both the `semantic_text` field and the source field are updated in t `semantic_text` field types have the following limitations: * `semantic_text` fields are not currently supported as elements of <>. +* `semantic_text` fields can't currently be set as part of <>. * `semantic_text` fields can't be defined as <> of another field, nor can they contain other fields as multi-fields. diff --git a/docs/reference/mapping/types/sparse-vector.asciidoc b/docs/reference/mapping/types/sparse-vector.asciidoc index b24f65fcf97ca..22d4644ede490 100644 --- a/docs/reference/mapping/types/sparse-vector.asciidoc +++ b/docs/reference/mapping/types/sparse-vector.asciidoc @@ -26,6 +26,23 @@ PUT my-index See <> for a complete example on adding documents to a `sparse_vector` mapped field using ELSER. +[[sparse-vectors-params]] +==== Parameters for `sparse_vector` fields + +The following parameters are accepted by `sparse_vector` fields: + +[horizontal] + +<>:: + +Indicates whether the field value should be stored and retrievable independently of the <> field. +Accepted values: true or false (default). +The field's data is stored using term vectors, a disk-efficient structure compared to the original JSON input. +The input map can be retrieved during a search request via the <>. +To benefit from reduced disk usage, you must either: + * Exclude the field from <>. + * Use <>. + [[index-multi-value-sparse-vectors]] ==== Multi-value sparse vectors diff --git a/docs/reference/mapping/types/text.asciidoc b/docs/reference/mapping/types/text.asciidoc index c33af69df5607..b10484fc5ded8 100644 --- a/docs/reference/mapping/types/text.asciidoc +++ b/docs/reference/mapping/types/text.asciidoc @@ -134,10 +134,6 @@ The following parameters are accepted by `text` fields: Whether the field value should be stored and retrievable separately from the <> field. Accepts `true` or `false` (default). - This parameter will be automatically set to `true` for TSDB indices - (indices that have `index.mode` set to `time_series`) - if there is no <> - sub-field that supports synthetic `_source`. <>:: @@ -177,15 +173,23 @@ a <> sub-field that supports synthetic `_source` or if the `text` field sets `store` to `true`. Either way, it may not have <>. -If using a sub-`keyword` field then the values are sorted in the same way as +If using a sub-`keyword` field, then the values are sorted in the same way as a `keyword` field's values are sorted. By default, that means sorted with duplicates removed. So: [source,console,id=synthetic-source-text-example-default] ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "text": { "type": "text", @@ -233,8 +237,16 @@ are preserved. ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "text": { "type": "text", "store": true } } diff --git a/docs/reference/mapping/types/token-count.asciidoc b/docs/reference/mapping/types/token-count.asciidoc index 7d9dffcc82082..2e5bd111122c8 100644 --- a/docs/reference/mapping/types/token-count.asciidoc +++ b/docs/reference/mapping/types/token-count.asciidoc @@ -103,5 +103,4 @@ any issues, but features in technical preview are not subject to the support SLA of official GA features. `token_count` fields support <> in their -default configuration. Synthetic `_source` cannot be used together with -<>. +default configuration. diff --git a/docs/reference/mapping/types/version.asciidoc b/docs/reference/mapping/types/version.asciidoc index 8da0fcae80fcd..1d9f927a80ce4 100644 --- a/docs/reference/mapping/types/version.asciidoc +++ b/docs/reference/mapping/types/version.asciidoc @@ -63,31 +63,38 @@ The following parameters are accepted by `version` fields: [discrete] ==== Limitations -This field type isn't optimized for heavy wildcard, regex or fuzzy searches. While those -type of queries work in this field, you should consider using a regular `keyword` field if -you strongly rely on these kind of queries. - +This field type isn't optimized for heavy wildcard, regex, or fuzzy searches. While those +types of queries work in this field, you should consider using a regular `keyword` field if +you strongly rely on these kinds of queries. [[version-synthetic-source]] ==== Synthetic `_source` IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices -(indices that have `index.mode` set to `time_series`). For other indices +(indices that have `index.mode` set to `time_series`). For other indices, synthetic `_source` is in technical preview. Features in technical preview may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features. -`version` fields support <> so long as they don't -declare <>. +`version` fields support <> in their +default configuration.. -Synthetic source always sorts `version` fields and removes duplicates. For example: +Synthetic source may sort `version` field values and remove duplicates. For example: [source,console,id=synthetic-source-version-example] ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "versions": { "type": "version" } } diff --git a/docs/reference/mapping/types/wildcard.asciidoc b/docs/reference/mapping/types/wildcard.asciidoc index 79fc953051d54..255e34ecd959b 100644 --- a/docs/reference/mapping/types/wildcard.asciidoc +++ b/docs/reference/mapping/types/wildcard.asciidoc @@ -133,16 +133,22 @@ The following parameters are accepted by `wildcard` fields: [[wildcard-synthetic-source]] ==== Synthetic `_source` -`wildcard` fields support <> so long as they don't -declare <>. -Synthetic source always sorts `wildcard` fields. For example: +Synthetic source may sort `wildcard` field values. For example: [source,console,id=synthetic-source-wildcard-example] ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "card": { "type": "wildcard" } } diff --git a/docs/reference/migration/apis/deprecation.asciidoc b/docs/reference/migration/apis/deprecation.asciidoc index 67b4c113af2bc..71453a001eeeb 100644 --- a/docs/reference/migration/apis/deprecation.asciidoc +++ b/docs/reference/migration/apis/deprecation.asciidoc @@ -5,6 +5,12 @@ Deprecation info ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-migration[Migration APIs]. +-- + include::{es-ref-dir}/migration/apis/shared-migration-apis-tip.asciidoc[] The deprecation API is to be used to retrieve information about different diff --git a/docs/reference/migration/apis/feature-migration.asciidoc b/docs/reference/migration/apis/feature-migration.asciidoc index e38639ac44531..717a46442bd92 100644 --- a/docs/reference/migration/apis/feature-migration.asciidoc +++ b/docs/reference/migration/apis/feature-migration.asciidoc @@ -5,6 +5,12 @@ Feature migration ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-migration[Migration APIs]. +-- + include::{es-ref-dir}/migration/apis/shared-migration-apis-tip.asciidoc[] Version upgrades sometimes require changes to how features store configuration diff --git a/docs/reference/migration/index.asciidoc b/docs/reference/migration/index.asciidoc index 0690f60495c97..c3e3251894bdd 100644 --- a/docs/reference/migration/index.asciidoc +++ b/docs/reference/migration/index.asciidoc @@ -1,5 +1,7 @@ include::migration_intro.asciidoc[] +* <> +* <> * <> * <> * <> @@ -18,6 +20,8 @@ include::migration_intro.asciidoc[] * <> * <> +include::migrate_8_18.asciidoc[] +include::migrate_8_17.asciidoc[] include::migrate_8_16.asciidoc[] include::migrate_8_15.asciidoc[] include::migrate_8_14.asciidoc[] diff --git a/docs/reference/migration/migrate_8_0/rest-api-changes.asciidoc b/docs/reference/migration/migrate_8_0/rest-api-changes.asciidoc index 99c09b9b05385..85fc4160ea071 100644 --- a/docs/reference/migration/migrate_8_0/rest-api-changes.asciidoc +++ b/docs/reference/migration/migrate_8_0/rest-api-changes.asciidoc @@ -1136,3 +1136,20 @@ for both cases. *Impact* + To detect a server timeout, check the `timed_out` field of the JSON response. ==== + +.The `Content-Type` response header no longer specifies the charset. +[%collapsible] +==== +*Details* + +The `Content-Type` response header no longer specifies the charset. This information is not required when transferring JSON data, because JSON text will always be encoded in Unicode, with UTF-8 being the default encoding. + +*Impact* + +Some applications and utilities, such as PowerShell's https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.utility/invoke-restmethod[Invoke-RestMethod], must receive charset information to display data correctly. If your application or utility relies on charset information in the `Content-Type` response header, UTF-8 encoded characters will be rendered incorrectly in the response body. + +As a workaround, to render non-ASCII characters, include an HTTP `Accept` header in your requests, specifying the charset: + +[source,sh] +---- +Accept: application/json; charset=utf-8 +---- +==== \ No newline at end of file diff --git a/docs/reference/migration/migrate_8_16.asciidoc b/docs/reference/migration/migrate_8_16.asciidoc index aea6322f292bf..7baa5fee9cbae 100644 --- a/docs/reference/migration/migrate_8_16.asciidoc +++ b/docs/reference/migration/migrate_8_16.asciidoc @@ -9,12 +9,206 @@ your application to {es} 8.16. See also <> and <>. -coming::[8.16.0] - - [discrete] [[breaking-changes-8.16]] === Breaking changes -There are no breaking changes in {es} 8.16. +The following changes in {es} 8.16 might affect your applications +and prevent them from operating normally. +Before upgrading to 8.16, review these changes and take the described steps +to mitigate the impact. + +[discrete] +[[breaking_816_analysis_changes]] +==== Analysis changes + +[[set_lenient_to_true_by_default_when_using_updateable_synonyms]] +.Set lenient to true by default when using updateable synonyms +[%collapsible] +==== +*Details* + +When a `synonym` or `synonym_graph` token filter is configured with `updateable: true`, the default `lenient` +value will now be `true`. + +*Impact* + +`synonym` or `synonym_graph` token filters configured with `updateable: true` will ignore invalid synonyms by +default. This prevents shard initialization errors on invalid synonyms. +==== + +[discrete] +[[breaking_816_mapping_changes]] +==== Mapping changes + +[[jdk_locale_database_change]] +.JDK locale database change +[%collapsible] +==== +*Details* + +{es} 8.16 changes the version of the JDK that is included from version 22 to version 23. This changes the locale database that is used by Elasticsearch from the COMPAT database to the CLDR database. This change can cause significant differences to the textual date formats accepted by Elasticsearch, and to calculated week-dates. + +If you run {es} 8.16 on JDK version 22 or below, it will use the COMPAT locale database to match the behavior of 8.15. However, starting with {es} 9.0, {es} will use the CLDR database regardless of JDK version it is run on. + +*Impact* + +This affects you if you use custom date formats using textual or week-date field specifiers. If you use date fields or calculated week-dates that change between the COMPAT and CLDR databases, then this change will cause Elasticsearch to reject previously valid date fields as invalid data. You might need to modify your ingest or output integration code to account for the differences between these two JDK versions. + +Starting in version 8.15.2, Elasticsearch will log deprecation warnings if you are using date format specifiers that might change on upgrading to JDK 23. These warnings are visible in Kibana. + +For detailed guidance, refer to <> and the https://ela.st/jdk-23-locales[Elastic blog]. +==== + +[discrete] +[[breaking_816_es_ql_changes]] +==== ES|QL changes + +[[esql_entirely_remove_meta_functions]] +.ESQL: Entirely remove META FUNCTIONS +[%collapsible] +==== +*Details* + +Removes an undocumented syntax from ESQL: META FUNCTION. This was never +reliable or really useful. Consult the documentation instead. + +*Impact* + +Removes an undocumented syntax from ESQL: META FUNCTION +==== + +[discrete] +[[breaking_816_rest_api_changes]] +==== REST API changes + +[[reworking_rrf_retriever_to_be_evaluated_during_rewrite_phase]] +.Reworking RRF retriever to be evaluated during rewrite phase +[%collapsible] +==== +*Details* + +In this release (8.16), we have introduced major changes to the retrievers framework +and how they can be evaluated, focusing mainly on compound retrievers +like `rrf` and `text_similarity_reranker`, which allowed us to support full +composability (i.e. any retriever can be nested under any compound retriever), +as well as supporting additional search features like collapsing, explaining, +aggregations, and highlighting. + +To ensure consistency, and given that this rework is not available until 8.16, +`rrf` and `text_similarity_reranker` retriever queries would now +throw an exception in a mixed cluster scenario, where there are nodes +both in current or later (i.e. >= 8.16) and previous ( <= 8.15) versions. + +As part of the rework, we have also removed the `_rank` property from +the responses of an `rrf` retriever. + +*Impact* + +- Users will not be able to use the `rrf` and `text_similarity_reranker` retrievers in a mixed cluster scenario +with previous releases (i.e. prior to 8.16), and the request will throw an `IllegalArgumentException`. +- `_rank` has now been removed from the output of the `rrf` retrievers so trying to directly parse the field +will throw an exception +==== + +[[update_data_stream_lifecycle_telemetry_to_track_global_retention]] +.Update data stream lifecycle telemetry to track global retention +[%collapsible] +==== +*Details* + +In this release we introduced global retention settings that fulfil the following criteria: + +- a data stream managed by the data stream lifecycle, +- a data stream that is not an internal data stream. + +As a result, we defined different types of retention: + +- **data retention**: the retention configured on data stream level by the data stream user or owner +- **default global retention:** the retention configured by an admin on a cluster level and applied to any +data stream that doesn't have data retention and fulfils the criteria. +- **max global retention:** the retention configured by an admin to guard against having long retention periods. +Any data stream that fulfills the criteria will adhere to the data retention unless it exceeds the max retention, +in which case the max global retention applies. +- **effective retention:** the retention that applies on the data stream that fulfill the criteria at a given moment +in time. It takes into consideration all the retention above and resolves it to the retention that will take effect. + +Considering the above changes, having a field named `retention` in the usage API was confusing. For this reason, we +renamed it to `data_retention` and added telemetry about the other configurations too. + +*Impact* + +Users that use the field `data_lifecycle.retention` should use the `data_lifecycle.data_retention` +==== + + +[discrete] +[[deprecated-8.16]] +=== Deprecations + +The following functionality has been deprecated in {es} 8.16 +and will be removed in a future version. +While this won't have an immediate impact on your applications, +we strongly encourage you to take the described steps to update your code +after upgrading to 8.16. + +To find out if you are using any deprecated functionality, +enable <>. + +[discrete] +[[deprecations_816_analysis]] +==== Analysis deprecations + +[[deprecate_dutch_kp_lovins_stemmer_as_they_are_removed_in_lucene_10]] +.Deprecate dutch_kp and lovins stemmer as they are removed in Lucene 10 +[%collapsible] +==== +*Details* + +kp, dutch_kp, dutchKp and lovins stemmers are deprecated and will be removed. + +*Impact* + +These stemmers will be removed and will be no longer supported. +==== + +[[deprecate_edge_ngram_side_parameter]] +.deprecate `edge_ngram` side parameter +[%collapsible] +==== +*Details* + +edge_ngram will no longer accept the side parameter. + +*Impact* + +Users will need to update any usage of edge_ngram token filter that utilizes `side`. If the `back` value was used, they can achieve the same behavior by using the `reverse` token filter. +==== + +[discrete] +[[deprecations_816_crud]] +==== CRUD deprecations + +[[deprecate_dot_prefixed_indices_composable_template_index_patterns]] +.Deprecate dot-prefixed indices and composable template index patterns +[%collapsible] +==== +*Details* + +Indices beginning with a dot '.' are reserved for system and internal indices, and should not be used by and end-user. Additionally, composable index templates that contain patterns for dot-prefixed indices should also be avoided, as these patterns are meant for internal use only. In a future Elasticsearch version, creation of these dot-prefixed indices will no longer be allowed. + +*Impact* + +Requests performing an action that would create an index beginning with a dot (indexing a document, manual creation, reindex), or creating an index template with index patterns beginning with a dot, will contain a deprecation header warning about dot-prefixed indices in the response. +==== + +[discrete] +[[deprecations_816_rest_api]] +==== REST API deprecations + +[[adding_deprecation_warnings_for_rrf_using_rank_sub_searches]] +.Adding deprecation warnings for rrf using rank and `sub_searches` +[%collapsible] +==== +*Details* + +Search API parameter `sub_searches` will no longer be a supported and will be removed in future releases. Similarly, `rrf` can only be used through the specified `retriever` and no longer though the `rank` parameter + +*Impact* + +Requests specifying rrf through `rank` and/or `sub_searches` elements will be disallowed in a future version. Users should instead utilize the new `retriever` parameter. +==== + +[[deprecate_legacy_params_from_range_query]] +.Deprecate legacy params from range query +[%collapsible] +==== +*Details* + +Range query will not longer accept `to`, `from`, `include_lower`, and `include_upper` parameters. + +*Impact* + +Instead use `gt`, `gte`, `lt` and `lte` parameters. +==== diff --git a/docs/reference/migration/migrate_8_17.asciidoc b/docs/reference/migration/migrate_8_17.asciidoc new file mode 100644 index 0000000000000..55f731da6f4bc --- /dev/null +++ b/docs/reference/migration/migrate_8_17.asciidoc @@ -0,0 +1,68 @@ +[[migrating-8.17]] +== Migrating to 8.17 +++++ +8.17 +++++ + +This section discusses the changes that you need to be aware of when migrating +your application to {es} 8.17. + +See also <> and <>. + +[discrete] +[[breaking-changes-8.17]] +=== Breaking changes + +There are no breaking changes in {es} 8.17. + + +[discrete] +[[deprecated-8.17]] +=== Deprecations + +The following functionality has been deprecated in {es} 8.17 +and will be removed in a future version. +While this won't have an immediate impact on your applications, +we strongly encourage you to take the described steps to update your code +after upgrading to 8.17. + +To find out if you are using any deprecated functionality, +enable <>. + +[discrete] +[[deprecations_817_mapping]] +==== Mapping deprecations + +[[deprecate_source_mode_in_mappings]] +.Deprecate `_source.mode` in mappings +[%collapsible] +==== +*Details* + +Configuring `_source.mode` in mappings is deprecated and will be removed in future versions. Use `index.mapping.source.mode` index setting instead. + +*Impact* + +Use `index.mapping.source.mode` index setting instead +==== + +[discrete] +[[deprecations_817_rest_api]] +==== REST API deprecations + +[[format_of_non_detailed_error_responses_changing_in_v9]] +.The format of non-detailed error responses is changing in v9 +[%collapsible] +==== +*Details* + +When an error occurs when processing a request, Elasticsearch returns information on that error in the REST response. +If `http:detailed_errors.enabled: false` is specified in node settings with the v8 REST API and below, +the format of this response changes significantly. +Starting with the v9 REST API, the JSON structure of responses with errors when the `http.detailed_errors.enabled: false` option is set +will be the same as when detailed errors are enabled (which is the default). +To keep using the existing format for non-detailed error responses, use the v8 REST API. + +*Impact* + +If you have set `http.detailed_errors.enabled: false` (the default is `true`) +the structure of JSON when any exceptions occur will change with the v9 REST API. +To keep using the existing format, use the v8 REST API. +==== + diff --git a/docs/reference/migration/migrate_8_18.asciidoc b/docs/reference/migration/migrate_8_18.asciidoc new file mode 100644 index 0000000000000..c989ff9f85b6d --- /dev/null +++ b/docs/reference/migration/migrate_8_18.asciidoc @@ -0,0 +1,20 @@ +[[migrating-8.18]] +== Migrating to 8.18 +++++ +8.18 +++++ + +This section discusses the changes that you need to be aware of when migrating +your application to {es} 8.18. + +See also <> and <>. + +coming::[8.18.0] + + +[discrete] +[[breaking-changes-8.18]] +=== Breaking changes + +There are no breaking changes in {es} 8.18. + diff --git a/docs/reference/migration/migration.asciidoc b/docs/reference/migration/migration.asciidoc index ffb2ca7a7859d..57b6c88aefea4 100644 --- a/docs/reference/migration/migration.asciidoc +++ b/docs/reference/migration/migration.asciidoc @@ -2,6 +2,12 @@ [[migration-api]] == Migration APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-migration[Migration APIs]. +-- + The migration APIs power {kib}'s **Upgrade Assistant** feature. include::apis/shared-migration-apis-tip.asciidoc[] diff --git a/docs/reference/ml/anomaly-detection/apis/close-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/close-job.asciidoc index f0cb968e082c5..865213dad3f50 100644 --- a/docs/reference/ml/anomaly-detection/apis/close-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/close-job.asciidoc @@ -5,6 +5,12 @@ Close jobs ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Closes one or more {anomaly-jobs}. [[ml-close-job-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/delete-calendar-event.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-calendar-event.asciidoc index b80a248038aea..d67f7cce9d1f6 100644 --- a/docs/reference/ml/anomaly-detection/apis/delete-calendar-event.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/delete-calendar-event.asciidoc @@ -5,6 +5,12 @@ Delete events from calendar ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Deletes scheduled events from a calendar. [[ml-delete-calendar-event-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/delete-calendar-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-calendar-job.asciidoc index 6720e236fd635..17a581f964f54 100644 --- a/docs/reference/ml/anomaly-detection/apis/delete-calendar-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/delete-calendar-job.asciidoc @@ -5,6 +5,12 @@ Delete jobs from calendar ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Deletes {anomaly-jobs} from a calendar. [[ml-delete-calendar-job-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/delete-calendar.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-calendar.asciidoc index 6684366c6f336..23ce74c8fb310 100644 --- a/docs/reference/ml/anomaly-detection/apis/delete-calendar.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/delete-calendar.asciidoc @@ -5,6 +5,12 @@ Delete calendars ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Deletes a calendar. [[ml-delete-calendar-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/delete-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-datafeed.asciidoc index 64a1e4c336fe6..61d3624ef50dc 100644 --- a/docs/reference/ml/anomaly-detection/apis/delete-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/delete-datafeed.asciidoc @@ -7,6 +7,12 @@ Delete {dfeeds} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Deletes an existing {dfeed}. [[ml-delete-datafeed-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/delete-expired-data.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-expired-data.asciidoc index a8c1279ac1b47..bee453f8d2082 100644 --- a/docs/reference/ml/anomaly-detection/apis/delete-expired-data.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/delete-expired-data.asciidoc @@ -5,6 +5,12 @@ Delete expired data ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Deletes expired and unused machine learning data. [[ml-delete-expired-data-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/delete-filter.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-filter.asciidoc index 4b41347543e8e..84346ae012300 100644 --- a/docs/reference/ml/anomaly-detection/apis/delete-filter.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/delete-filter.asciidoc @@ -5,6 +5,12 @@ Delete filters ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Deletes a filter. [[ml-delete-filter-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/delete-forecast.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-forecast.asciidoc index 74e6ce27084ad..ee3167036f66d 100644 --- a/docs/reference/ml/anomaly-detection/apis/delete-forecast.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/delete-forecast.asciidoc @@ -5,6 +5,12 @@ Delete forecasts ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Deletes forecasts from a {ml} job. [[ml-delete-forecast-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/delete-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-job.asciidoc index 1bbe07fd44f49..595c5acc041d6 100644 --- a/docs/reference/ml/anomaly-detection/apis/delete-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/delete-job.asciidoc @@ -5,6 +5,12 @@ Delete jobs ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Deletes an existing {anomaly-job}. [[ml-delete-job-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/delete-snapshot.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-snapshot.asciidoc index ad10de7a2ba0f..345b933a19622 100644 --- a/docs/reference/ml/anomaly-detection/apis/delete-snapshot.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/delete-snapshot.asciidoc @@ -5,6 +5,12 @@ Delete model snapshots ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Deletes an existing model snapshot. [[ml-delete-snapshot-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/estimate-model-memory.asciidoc b/docs/reference/ml/anomaly-detection/apis/estimate-model-memory.asciidoc index 1cc4df42f083d..8be8e8df72d4e 100644 --- a/docs/reference/ml/anomaly-detection/apis/estimate-model-memory.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/estimate-model-memory.asciidoc @@ -5,6 +5,12 @@ Estimate model memory ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Makes an estimation of the memory usage for an {anomaly-job} model. It is based on analysis configuration details for the job and cardinality estimates for the fields it references. diff --git a/docs/reference/ml/anomaly-detection/apis/flush-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/flush-job.asciidoc index 68ff601749b4b..dcf4d7a6b1d60 100644 --- a/docs/reference/ml/anomaly-detection/apis/flush-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/flush-job.asciidoc @@ -5,6 +5,12 @@ Flush jobs ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Forces any buffered data to be processed by the job. [[ml-flush-job-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/forecast.asciidoc b/docs/reference/ml/anomaly-detection/apis/forecast.asciidoc index 3e6067ab05857..ac4e3eb4ed29a 100644 --- a/docs/reference/ml/anomaly-detection/apis/forecast.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/forecast.asciidoc @@ -5,6 +5,12 @@ Forecast jobs ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Predicts the future behavior of a time series by using its historical behavior. [[ml-forecast-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/get-bucket.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-bucket.asciidoc index bca839d1db318..d33d6f64021a3 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-bucket.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-bucket.asciidoc @@ -5,6 +5,12 @@ Get buckets ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Retrieves {anomaly-job} results for one or more buckets. [[ml-get-bucket-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/get-calendar-event.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-calendar-event.asciidoc index fc06e286bf46c..565a553e18974 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-calendar-event.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-calendar-event.asciidoc @@ -5,6 +5,12 @@ Get scheduled events ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Retrieves information about the scheduled events in calendars. [[ml-get-calendar-event-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/get-calendar.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-calendar.asciidoc index b2c46bbe16c0e..bf00458299633 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-calendar.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-calendar.asciidoc @@ -5,6 +5,12 @@ Get calendars ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Retrieves configuration information for calendars. [[ml-get-calendar-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/get-category.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-category.asciidoc index 33de5e0f71a08..0f5fa1504c9fe 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-category.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-category.asciidoc @@ -5,6 +5,12 @@ Get categories ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Retrieves {anomaly-job} results for one or more categories. [[ml-get-category-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/get-datafeed-stats.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-datafeed-stats.asciidoc index a224f3880a1ad..950d12e9a503f 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-datafeed-stats.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-datafeed-stats.asciidoc @@ -7,6 +7,12 @@ Get {dfeed} statistics ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Retrieves usage information for {dfeeds}. [[ml-get-datafeed-stats-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc index a986e2220f928..3b86c5cc22a87 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc @@ -7,6 +7,12 @@ Get {dfeeds} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Retrieves configuration information for {dfeeds}. [[ml-get-datafeed-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/get-filter.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-filter.asciidoc index f73dcd236f1af..dc167238701cd 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-filter.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-filter.asciidoc @@ -5,6 +5,12 @@ Get filters ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Retrieves filters. [[ml-get-filter-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/get-influencer.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-influencer.asciidoc index 31489e361a848..f2e26344207a8 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-influencer.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-influencer.asciidoc @@ -5,6 +5,12 @@ Get influencers ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Retrieves {anomaly-job} results for one or more influencers. [[ml-get-influencer-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/get-job-model-snapshot-upgrade-stats.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-job-model-snapshot-upgrade-stats.asciidoc index 0939282a75916..32a9b67258b2d 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-job-model-snapshot-upgrade-stats.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-job-model-snapshot-upgrade-stats.asciidoc @@ -7,6 +7,12 @@ Get model snapshot upgrade statistics ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Retrieves usage information for {anomaly-job} model snapshot upgrades. [[ml-get-job-model-snapshot-upgrade-stats-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/get-job-stats.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-job-stats.asciidoc index a5bd188397bb9..a3c329edb3d83 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-job-stats.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-job-stats.asciidoc @@ -5,6 +5,12 @@ Get job statistics ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Retrieves usage information for {anomaly-jobs}. [[ml-get-job-stats-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc index 4ee6c429ce730..8331be16df82a 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc @@ -5,6 +5,12 @@ Get jobs ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Retrieves configuration information for {anomaly-jobs}. [[ml-get-job-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/get-overall-buckets.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-overall-buckets.asciidoc index b581b5c3a2eb2..45ae61647bfda 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-overall-buckets.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-overall-buckets.asciidoc @@ -5,6 +5,12 @@ Get overall buckets ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Retrieves overall bucket results that summarize the bucket results of multiple {anomaly-jobs}. diff --git a/docs/reference/ml/anomaly-detection/apis/get-record.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-record.asciidoc index e74ab3ecb4b12..266ab64ba5e19 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-record.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-record.asciidoc @@ -5,6 +5,12 @@ Get records ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Retrieves anomaly records for an {anomaly-job}. [[ml-get-record-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/get-snapshot.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-snapshot.asciidoc index d94bd4060854a..47fdc3db46b60 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-snapshot.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-snapshot.asciidoc @@ -5,6 +5,12 @@ Get model snapshots ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Retrieves information about model snapshots. [[ml-get-snapshot-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/ml-ad-apis.asciidoc b/docs/reference/ml/anomaly-detection/apis/ml-ad-apis.asciidoc index 856232c933432..951f9522db74b 100644 --- a/docs/reference/ml/anomaly-detection/apis/ml-ad-apis.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/ml-ad-apis.asciidoc @@ -2,6 +2,12 @@ [[ml-ad-apis]] = {ml-cap} {anomaly-detect} APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + You can use the following APIs to perform {ml} {anomaly-detect} activities. See also <>, <>, <>. diff --git a/docs/reference/ml/anomaly-detection/apis/open-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/open-job.asciidoc index 385f672f467f6..9a5153952b5a2 100644 --- a/docs/reference/ml/anomaly-detection/apis/open-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/open-job.asciidoc @@ -5,6 +5,12 @@ Open jobs ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Opens one or more {anomaly-jobs}. [[ml-open-job-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/post-calendar-event.asciidoc b/docs/reference/ml/anomaly-detection/apis/post-calendar-event.asciidoc index 8db1eb1ae84df..a717d76ea4e55 100644 --- a/docs/reference/ml/anomaly-detection/apis/post-calendar-event.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/post-calendar-event.asciidoc @@ -5,6 +5,12 @@ Add events to calendar ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Posts scheduled events in a calendar. [[ml-post-calendar-event-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/post-data.asciidoc b/docs/reference/ml/anomaly-detection/apis/post-data.asciidoc index 931efcf8c2a52..83e5dd77e814a 100644 --- a/docs/reference/ml/anomaly-detection/apis/post-data.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/post-data.asciidoc @@ -7,6 +7,12 @@ deprecated::[7.11.0, "Posting data directly to anomaly detection jobs is deprecated, in a future major version a <> will be required."] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Sends data to an anomaly detection job for analysis. [[ml-post-data-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/preview-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/preview-datafeed.asciidoc index 243cd2a5f32a3..c57965401e074 100644 --- a/docs/reference/ml/anomaly-detection/apis/preview-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/preview-datafeed.asciidoc @@ -7,6 +7,12 @@ Preview {dfeeds} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Previews a {dfeed}. [[ml-preview-datafeed-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/put-calendar-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/put-calendar-job.asciidoc index 0c19a08cbd74b..612f472bba37d 100644 --- a/docs/reference/ml/anomaly-detection/apis/put-calendar-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/put-calendar-job.asciidoc @@ -5,6 +5,12 @@ Add jobs to calendar ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Adds an {anomaly-job} to a calendar. [[ml-put-calendar-job-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/put-calendar.asciidoc b/docs/reference/ml/anomaly-detection/apis/put-calendar.asciidoc index fd2b58a31737a..69a4498cc29da 100644 --- a/docs/reference/ml/anomaly-detection/apis/put-calendar.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/put-calendar.asciidoc @@ -5,6 +5,12 @@ Create calendars ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Instantiates a calendar. [[ml-put-calendar-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/put-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/put-datafeed.asciidoc index 47e3059666d76..18260d3538e4b 100644 --- a/docs/reference/ml/anomaly-detection/apis/put-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/put-datafeed.asciidoc @@ -7,6 +7,12 @@ Create {dfeeds} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Instantiates a {dfeed}. [[ml-put-datafeed-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/put-filter.asciidoc b/docs/reference/ml/anomaly-detection/apis/put-filter.asciidoc index b50ba8cb1e23b..3f3157f008e9f 100644 --- a/docs/reference/ml/anomaly-detection/apis/put-filter.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/put-filter.asciidoc @@ -5,6 +5,12 @@ Create filters ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Instantiates a filter. [[ml-put-filter-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc index 7bf02e7a0dd6e..e3c292cc534bf 100644 --- a/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc @@ -5,6 +5,12 @@ Create jobs ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Instantiates an {anomaly-job}. [[ml-put-job-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/reset-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/reset-job.asciidoc index 9009d634a2e9c..2d9c4696cab4b 100644 --- a/docs/reference/ml/anomaly-detection/apis/reset-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/reset-job.asciidoc @@ -5,6 +5,12 @@ Reset jobs ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Resets an existing {anomaly-job}. [[ml-reset-job-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/revert-snapshot.asciidoc b/docs/reference/ml/anomaly-detection/apis/revert-snapshot.asciidoc index c8d7a27ee2047..792a102ff2953 100644 --- a/docs/reference/ml/anomaly-detection/apis/revert-snapshot.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/revert-snapshot.asciidoc @@ -5,6 +5,12 @@ Revert model snapshots ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Reverts to a specific snapshot. [[ml-revert-snapshot-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/start-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/start-datafeed.asciidoc index b54c80133d7dc..4281b3a02b826 100644 --- a/docs/reference/ml/anomaly-detection/apis/start-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/start-datafeed.asciidoc @@ -7,6 +7,12 @@ Start {dfeeds} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Starts one or more {dfeeds}. [[ml-start-datafeed-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/stop-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/stop-datafeed.asciidoc index bc15a1de8a056..91e5087715fb7 100644 --- a/docs/reference/ml/anomaly-detection/apis/stop-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/stop-datafeed.asciidoc @@ -7,6 +7,12 @@ Stop {dfeeds} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Stops one or more {dfeeds}. [[ml-stop-datafeed-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/update-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/update-datafeed.asciidoc index b3920d9d4f809..c03c921e7875a 100644 --- a/docs/reference/ml/anomaly-detection/apis/update-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/update-datafeed.asciidoc @@ -7,6 +7,12 @@ Update {dfeeds} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Updates certain properties of a {dfeed}. diff --git a/docs/reference/ml/anomaly-detection/apis/update-filter.asciidoc b/docs/reference/ml/anomaly-detection/apis/update-filter.asciidoc index a4221c37a438e..8fb5376f4acbc 100644 --- a/docs/reference/ml/anomaly-detection/apis/update-filter.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/update-filter.asciidoc @@ -5,6 +5,12 @@ Update filters ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Updates the description of a filter, adds items, or removes items. [[ml-update-filter-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc index ee13247fc8838..bed013c113c01 100644 --- a/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc @@ -5,6 +5,12 @@ Update jobs ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Updates certain properties of an {anomaly-job}. [[ml-update-job-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/update-snapshot.asciidoc b/docs/reference/ml/anomaly-detection/apis/update-snapshot.asciidoc index f8c0384860029..54276f62ff094 100644 --- a/docs/reference/ml/anomaly-detection/apis/update-snapshot.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/update-snapshot.asciidoc @@ -5,6 +5,12 @@ Update model snapshots ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Updates certain properties of a snapshot. [[ml-update-snapshot-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/upgrade-job-model-snapshot.asciidoc b/docs/reference/ml/anomaly-detection/apis/upgrade-job-model-snapshot.asciidoc index 3a74e3b2296df..2538b4958ada1 100644 --- a/docs/reference/ml/anomaly-detection/apis/upgrade-job-model-snapshot.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/upgrade-job-model-snapshot.asciidoc @@ -5,6 +5,12 @@ Upgrade model snapshots ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Upgrades an {anomaly-detect} model snapshot to the latest major version. NOTE: From {es} 8.10.0, a new version number is used to diff --git a/docs/reference/ml/anomaly-detection/apis/validate-detector.asciidoc b/docs/reference/ml/anomaly-detection/apis/validate-detector.asciidoc index c71673be7dc00..820845d0fd233 100644 --- a/docs/reference/ml/anomaly-detection/apis/validate-detector.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/validate-detector.asciidoc @@ -5,6 +5,12 @@ Validate detectors ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Validates detector configuration information. [[ml-valid-detector-request]] diff --git a/docs/reference/ml/anomaly-detection/apis/validate-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/validate-job.asciidoc index c77623566d4c8..382199e315695 100644 --- a/docs/reference/ml/anomaly-detection/apis/validate-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/validate-job.asciidoc @@ -5,6 +5,12 @@ Validate jobs ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-anomaly[{ml-cap} {anomaly-detect} APIs]. +-- + Validates {anomaly-job} configuration information. [[ml-valid-job-request]] diff --git a/docs/reference/ml/common/apis/get-ml-info.asciidoc b/docs/reference/ml/common/apis/get-ml-info.asciidoc index 104375bd641c8..62fae2a9a9184 100644 --- a/docs/reference/ml/common/apis/get-ml-info.asciidoc +++ b/docs/reference/ml/common/apis/get-ml-info.asciidoc @@ -7,6 +7,12 @@ Get {ml} info ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml[{ml-cap} APIs]. +-- + Returns defaults and limits used by machine learning. [[get-ml-info-request]] diff --git a/docs/reference/ml/common/apis/get-ml-memory.asciidoc b/docs/reference/ml/common/apis/get-ml-memory.asciidoc index 81e0f59a97e58..8c1cf620b70eb 100644 --- a/docs/reference/ml/common/apis/get-ml-memory.asciidoc +++ b/docs/reference/ml/common/apis/get-ml-memory.asciidoc @@ -7,6 +7,12 @@ Get {ml} memory stats ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml[{ml-cap} APIs]. +-- + Returns information on how {ml} is using memory. [[get-ml-memory-request]] diff --git a/docs/reference/ml/common/apis/ml-apis.asciidoc b/docs/reference/ml/common/apis/ml-apis.asciidoc index c4349f3eb7366..95f102ceecfa8 100644 --- a/docs/reference/ml/common/apis/ml-apis.asciidoc +++ b/docs/reference/ml/common/apis/ml-apis.asciidoc @@ -2,6 +2,12 @@ [[ml-apis]] = {ml-cap} APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml[{ml-cap} APIs]. +-- + You can use the following APIs to retrieve information related to the {stack-ml-features}: diff --git a/docs/reference/ml/common/apis/set-upgrade-mode.asciidoc b/docs/reference/ml/common/apis/set-upgrade-mode.asciidoc index 2469753382f9b..52adcac9bacdc 100644 --- a/docs/reference/ml/common/apis/set-upgrade-mode.asciidoc +++ b/docs/reference/ml/common/apis/set-upgrade-mode.asciidoc @@ -5,6 +5,12 @@ Set upgrade mode ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml[{ml-cap} APIs]. +-- + Sets a cluster wide upgrade_mode setting that prepares {ml} indices for an upgrade. diff --git a/docs/reference/ml/df-analytics/apis/delete-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/delete-dfanalytics.asciidoc index b505da570244f..a6b5058eab887 100644 --- a/docs/reference/ml/df-analytics/apis/delete-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/delete-dfanalytics.asciidoc @@ -6,6 +6,12 @@ Delete {dfanalytics-jobs} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-data-frame[{ml-cap} {dfanalytics} APIs]. +-- + Deletes an existing {dfanalytics-job}. diff --git a/docs/reference/ml/df-analytics/apis/evaluate-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/evaluate-dfanalytics.asciidoc index 8d31fcd0107d9..fb0b6da0bc805 100644 --- a/docs/reference/ml/df-analytics/apis/evaluate-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/evaluate-dfanalytics.asciidoc @@ -7,6 +7,12 @@ Evaluate {dfanalytics} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-data-frame[{ml-cap} {dfanalytics} APIs]. +-- + Evaluates the {dfanalytics} for an annotated index. diff --git a/docs/reference/ml/df-analytics/apis/explain-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/explain-dfanalytics.asciidoc index 0ee7ec5634582..566cfc3ba3596 100644 --- a/docs/reference/ml/df-analytics/apis/explain-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/explain-dfanalytics.asciidoc @@ -7,6 +7,12 @@ Explain {dfanalytics} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-data-frame[{ml-cap} {dfanalytics} APIs]. +-- + Explains a {dataframe-analytics-config}. diff --git a/docs/reference/ml/df-analytics/apis/get-dfanalytics-stats.asciidoc b/docs/reference/ml/df-analytics/apis/get-dfanalytics-stats.asciidoc index b193379084db1..e0cdfc40de4c0 100644 --- a/docs/reference/ml/df-analytics/apis/get-dfanalytics-stats.asciidoc +++ b/docs/reference/ml/df-analytics/apis/get-dfanalytics-stats.asciidoc @@ -6,8 +6,13 @@ Get {dfanalytics-jobs} stats ++++ -Retrieves usage information for {dfanalytics-jobs}. +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-data-frame[{ml-cap} {dfanalytics} APIs]. +-- +Retrieves usage information for {dfanalytics-jobs}. [[ml-get-dfanalytics-stats-request]] == {api-request-title} diff --git a/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc index c2a4caa981da1..6e65bdfe83a25 100644 --- a/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc @@ -5,6 +5,12 @@ Get {dfanalytics-jobs} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-data-frame[{ml-cap} {dfanalytics} APIs]. +-- + Retrieves configuration information for {dfanalytics-jobs}. diff --git a/docs/reference/ml/df-analytics/apis/ml-df-analytics-apis.asciidoc b/docs/reference/ml/df-analytics/apis/ml-df-analytics-apis.asciidoc index ba16f728e4b9a..a90f42b7b9757 100644 --- a/docs/reference/ml/df-analytics/apis/ml-df-analytics-apis.asciidoc +++ b/docs/reference/ml/df-analytics/apis/ml-df-analytics-apis.asciidoc @@ -2,6 +2,12 @@ [[ml-df-analytics-apis]] = {ml-cap} {dfanalytics} APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-data-frame[{ml-cap} {dfanalytics} APIs]. +-- + You can use the following APIs to perform {ml} {dfanalytics} activities: * <> diff --git a/docs/reference/ml/df-analytics/apis/preview-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/preview-dfanalytics.asciidoc index 2c61c3263992c..896cf5ca8eb8a 100644 --- a/docs/reference/ml/df-analytics/apis/preview-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/preview-dfanalytics.asciidoc @@ -7,6 +7,12 @@ Preview {dfanalytics} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-data-frame[{ml-cap} {dfanalytics} APIs]. +-- + Previews the features used by a {dataframe-analytics-config}. diff --git a/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc index 54cbe78b34452..8c2e95e0744ed 100644 --- a/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc @@ -6,6 +6,12 @@ Create {dfanalytics-jobs} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-data-frame[{ml-cap} {dfanalytics} APIs]. +-- + Instantiates a {dfanalytics-job}. diff --git a/docs/reference/ml/df-analytics/apis/start-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/start-dfanalytics.asciidoc index 70e996ef8dd0a..7409c443ff802 100644 --- a/docs/reference/ml/df-analytics/apis/start-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/start-dfanalytics.asciidoc @@ -7,6 +7,12 @@ Start {dfanalytics-jobs} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-data-frame[{ml-cap} {dfanalytics} APIs]. +-- + Starts a {dfanalytics-job}. diff --git a/docs/reference/ml/df-analytics/apis/stop-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/stop-dfanalytics.asciidoc index 2fa3bc4413d7a..95d39b98e3bbd 100644 --- a/docs/reference/ml/df-analytics/apis/stop-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/stop-dfanalytics.asciidoc @@ -7,6 +7,12 @@ Stop {dfanalytics-jobs} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-data-frame[{ml-cap} {dfanalytics} APIs]. +-- + Stops one or more {dfanalytics-jobs}. diff --git a/docs/reference/ml/df-analytics/apis/update-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/update-dfanalytics.asciidoc index 49cca176be69b..35e871761b21a 100644 --- a/docs/reference/ml/df-analytics/apis/update-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/update-dfanalytics.asciidoc @@ -6,6 +6,12 @@ Update {dfanalytics-jobs} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-data-frame[{ml-cap} {dfanalytics} APIs]. +-- + Updates an existing {dfanalytics-job}. diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc index 97122141d7558..4948db48664ed 100644 --- a/docs/reference/ml/ml-shared.asciidoc +++ b/docs/reference/ml/ml-shared.asciidoc @@ -3,7 +3,6 @@ Adaptive allocations configuration object. If enabled, the number of allocations of the model is set based on the current load the process gets. When the load is high, a new model allocation is automatically created (respecting the value of `max_number_of_allocations` if it's set). When the load is low, a model allocation is automatically removed (respecting the value of `min_number_of_allocations` if it's set). -The number of model allocations cannot be scaled down to less than `1` this way. If `adaptive_allocations` is enabled, do not set the number of allocations manually. end::adaptive-allocation[] @@ -19,7 +18,8 @@ end::adaptive-allocation-max-number[] tag::adaptive-allocation-min-number[] Specifies the minimum number of allocations to scale to. -If set, it must be greater than or equal to `1`. +If set, it must be greater than or equal to `0`. +If not defined, the deployment scales to `0`. end::adaptive-allocation-min-number[] tag::aggregations[] @@ -988,6 +988,7 @@ values are + -- * `bert`: Use for BERT-style models +* `deberta_v2`: Use for DeBERTa v2 and v3-style models * `mpnet`: Use for MPNet-style models * `roberta`: Use for RoBERTa-style and BART-style models * experimental:[] `xlm_roberta`: Use for XLMRoBERTa-style models @@ -1037,6 +1038,19 @@ sequence. Therefore, do not use `second` in this case. end::inference-config-nlp-tokenization-truncate[] +tag::inference-config-nlp-tokenization-truncate-deberta-v2[] +Indicates how tokens are truncated when they exceed `max_sequence_length`. +The default value is `first`. ++ +-- +* `balanced`: One or both of the first and second sequences may be truncated so as to balance the tokens included from both sequences. +* `none`: No truncation occurs; the inference request receives an error. +* `first`: Only the first sequence is truncated. +* `second`: Only the second sequence is truncated. If there is just one sequence, that sequence is truncated. +-- + +end::inference-config-nlp-tokenization-truncate-deberta-v2[] + tag::inference-config-nlp-tokenization-bert-with-special-tokens[] Tokenize with special tokens. The tokens typically included in BERT-style tokenization are: + @@ -1050,10 +1064,23 @@ tag::inference-config-nlp-tokenization-bert-ja-with-special-tokens[] Tokenize with special tokens if `true`. end::inference-config-nlp-tokenization-bert-ja-with-special-tokens[] +tag::inference-config-nlp-tokenization-deberta-v2[] +DeBERTa-style tokenization is to be performed with the enclosed settings. +end::inference-config-nlp-tokenization-deberta-v2[] + tag::inference-config-nlp-tokenization-max-sequence-length[] Specifies the maximum number of tokens allowed to be output by the tokenizer. end::inference-config-nlp-tokenization-max-sequence-length[] +tag::inference-config-nlp-tokenization-deberta-v2-with-special-tokens[] +Tokenize with special tokens. The tokens typically included in DeBERTa-style tokenization are: ++ +-- +* `[CLS]`: The first token of the sequence being classified. +* `[SEP]`: Indicates sequence separation and sequence end. +-- +end::inference-config-nlp-tokenization-deberta-v2-with-special-tokens[] + tag::inference-config-nlp-tokenization-roberta[] RoBERTa-style tokenization is to be performed with the enclosed settings. end::inference-config-nlp-tokenization-roberta[] diff --git a/docs/reference/ml/trained-models/apis/clear-trained-model-deployment-cache.asciidoc b/docs/reference/ml/trained-models/apis/clear-trained-model-deployment-cache.asciidoc index f24379705fc75..1e1639ab5eee2 100644 --- a/docs/reference/ml/trained-models/apis/clear-trained-model-deployment-cache.asciidoc +++ b/docs/reference/ml/trained-models/apis/clear-trained-model-deployment-cache.asciidoc @@ -6,6 +6,12 @@ Clear trained model deployment cache ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs]. +-- + Clears the {infer} cache on all nodes where the deployment is assigned. [[clear-trained-model-deployment-cache-request]] diff --git a/docs/reference/ml/trained-models/apis/delete-trained-models-aliases.asciidoc b/docs/reference/ml/trained-models/apis/delete-trained-models-aliases.asciidoc index 1b67a642f7b7a..22e0422df9fb7 100644 --- a/docs/reference/ml/trained-models/apis/delete-trained-models-aliases.asciidoc +++ b/docs/reference/ml/trained-models/apis/delete-trained-models-aliases.asciidoc @@ -6,6 +6,12 @@ Delete trained model aliases ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs]. +-- + Deletes a trained model alias. diff --git a/docs/reference/ml/trained-models/apis/delete-trained-models.asciidoc b/docs/reference/ml/trained-models/apis/delete-trained-models.asciidoc index 1b54343d1f1c9..d738c6630a78f 100644 --- a/docs/reference/ml/trained-models/apis/delete-trained-models.asciidoc +++ b/docs/reference/ml/trained-models/apis/delete-trained-models.asciidoc @@ -6,6 +6,12 @@ Delete trained models ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs]. +-- + Deletes an existing trained {infer} model. diff --git a/docs/reference/ml/trained-models/apis/get-trained-models-stats.asciidoc b/docs/reference/ml/trained-models/apis/get-trained-models-stats.asciidoc index beff87e6ec6e6..60c06a6124d40 100644 --- a/docs/reference/ml/trained-models/apis/get-trained-models-stats.asciidoc +++ b/docs/reference/ml/trained-models/apis/get-trained-models-stats.asciidoc @@ -6,6 +6,12 @@ Get trained models stats ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs]. +-- + Retrieves usage information for trained models. diff --git a/docs/reference/ml/trained-models/apis/get-trained-models.asciidoc b/docs/reference/ml/trained-models/apis/get-trained-models.asciidoc index f203578c6f1c7..3736766b6b063 100644 --- a/docs/reference/ml/trained-models/apis/get-trained-models.asciidoc +++ b/docs/reference/ml/trained-models/apis/get-trained-models.asciidoc @@ -6,8 +6,13 @@ Get trained models ++++ -Retrieves configuration information for a trained model. +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs]. +-- +Retrieves configuration information for a trained model. [[ml-get-trained-models-request]] == {api-request-title} diff --git a/docs/reference/ml/trained-models/apis/infer-trained-model-deployment.asciidoc b/docs/reference/ml/trained-models/apis/infer-trained-model-deployment.asciidoc index 83bc56d18df63..8bb48eeb5cbf2 100644 --- a/docs/reference/ml/trained-models/apis/infer-trained-model-deployment.asciidoc +++ b/docs/reference/ml/trained-models/apis/infer-trained-model-deployment.asciidoc @@ -6,6 +6,12 @@ Infer trained model deployment ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs]. +-- + Evaluates a trained model. deprecated::[8.3.0,Replaced by <>.] diff --git a/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc b/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc index 9aac913e7559f..93b8aa3422abc 100644 --- a/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc +++ b/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc @@ -6,6 +6,12 @@ Infer trained model ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs]. +-- + Evaluates a trained model. The model may be any supervised model either trained by {dfanalytics} or imported. @@ -137,6 +143,18 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio (Optional, string) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= + `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] @@ -213,6 +231,17 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio (Optional, string) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] @@ -289,6 +318,17 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio (Optional, string) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] @@ -385,6 +425,21 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio (Optional, string) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`span`:::: +(Optional, integer) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] + +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] @@ -505,6 +560,21 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio (Optional, string) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`span`:::: +(Optional, integer) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] + +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] @@ -596,6 +666,17 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio (Optional, string) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] @@ -675,6 +756,21 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio (Optional, integer) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +`with_special_tokens`:::: +(Optional, boolean) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-with-special-tokens] +======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`span`:::: +(Optional, integer) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] + `with_special_tokens`:::: (Optional, boolean) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-with-special-tokens] @@ -778,6 +874,17 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio (Optional, string) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] diff --git a/docs/reference/ml/trained-models/apis/ml-trained-models-apis.asciidoc b/docs/reference/ml/trained-models/apis/ml-trained-models-apis.asciidoc index 83ef3c49fb5ed..5b0fc2b83afa5 100644 --- a/docs/reference/ml/trained-models/apis/ml-trained-models-apis.asciidoc +++ b/docs/reference/ml/trained-models/apis/ml-trained-models-apis.asciidoc @@ -2,6 +2,12 @@ [[ml-df-trained-models-apis]] = {ml-cap} trained model APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs]. +-- + You can use the following APIs to perform model management operations: * <> diff --git a/docs/reference/ml/trained-models/apis/put-trained-model-definition-part.asciidoc b/docs/reference/ml/trained-models/apis/put-trained-model-definition-part.asciidoc index d1da29abffcd3..857b86892a6b9 100644 --- a/docs/reference/ml/trained-models/apis/put-trained-model-definition-part.asciidoc +++ b/docs/reference/ml/trained-models/apis/put-trained-model-definition-part.asciidoc @@ -6,6 +6,12 @@ Create part of a trained model ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs]. +-- + Creates part of a trained model definition. [[ml-put-trained-model-definition-part-request]] diff --git a/docs/reference/ml/trained-models/apis/put-trained-model-vocabulary.asciidoc b/docs/reference/ml/trained-models/apis/put-trained-model-vocabulary.asciidoc index 2fdf86259388f..39f93f882235a 100644 --- a/docs/reference/ml/trained-models/apis/put-trained-model-vocabulary.asciidoc +++ b/docs/reference/ml/trained-models/apis/put-trained-model-vocabulary.asciidoc @@ -6,6 +6,12 @@ Create trained model vocabulary ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs]. +-- + Creates a trained model vocabulary. This is supported only for natural language processing (NLP) models. diff --git a/docs/reference/ml/trained-models/apis/put-trained-models-aliases.asciidoc b/docs/reference/ml/trained-models/apis/put-trained-models-aliases.asciidoc index d8c00efe28015..3fcc7f5bb2da4 100644 --- a/docs/reference/ml/trained-models/apis/put-trained-models-aliases.asciidoc +++ b/docs/reference/ml/trained-models/apis/put-trained-models-aliases.asciidoc @@ -6,6 +6,11 @@ Create or update trained model aliases ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs]. +-- Creates or updates a trained model alias. diff --git a/docs/reference/ml/trained-models/apis/put-trained-models.asciidoc b/docs/reference/ml/trained-models/apis/put-trained-models.asciidoc index e29bc8823ab29..256c0d29f8d2f 100644 --- a/docs/reference/ml/trained-models/apis/put-trained-models.asciidoc +++ b/docs/reference/ml/trained-models/apis/put-trained-models.asciidoc @@ -3,11 +3,15 @@ = Create trained models API [subs="attributes"] ++++ - Create trained models - ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs]. +-- + Creates a trained model. WARNING: Models created in version 7.8.0 are not backwards compatible @@ -773,6 +777,37 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio (Optional, boolean) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-with-special-tokens] ==== +`deberta_v2`:: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +==== +`do_lower_case`::: +(Optional, boolean) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] ++ +-- +Defaults to `false`. +-- + +`max_sequence_length`::: +(Optional, integer) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] + +`span`::: +(Optional, integer) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] + +`truncate`::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] + +`with_special_tokens`::: +(Optional, boolean) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2-with-special-tokens] +==== `roberta`:: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] diff --git a/docs/reference/ml/trained-models/apis/start-trained-model-deployment.asciidoc b/docs/reference/ml/trained-models/apis/start-trained-model-deployment.asciidoc index 6f7e2a4d9f988..7eaa149976652 100644 --- a/docs/reference/ml/trained-models/apis/start-trained-model-deployment.asciidoc +++ b/docs/reference/ml/trained-models/apis/start-trained-model-deployment.asciidoc @@ -6,6 +6,12 @@ Start trained model deployment ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs]. +-- + Starts a new trained model deployment. [[start-trained-model-deployment-request]] @@ -138,8 +144,8 @@ normal priority deployments. Controls how many inference requests are allowed in the queue at a time. Every machine learning node in the cluster where the model can be allocated has a queue of this size; when the number of requests exceeds the total value, -new requests are rejected with a 429 error. Defaults to 1024. Max allowed value -is 1000000. +new requests are rejected with a 429 error. Defaults to 10000. Max allowed value +is 100000. `threads_per_allocation`:: (Optional, integer) @@ -173,7 +179,7 @@ The API returns the following results: "model_bytes": 265632637, "threads_per_allocation" : 1, "number_of_allocations" : 1, - "queue_capacity" : 1024, + "queue_capacity" : 10000, "priority": "normal" }, "routing_table": { @@ -229,4 +235,4 @@ POST _ml/trained_models/my_model/deployment/_start?deployment_id=my_model_for_se } } -------------------------------------------------- -// TEST[skip:TBD] \ No newline at end of file +// TEST[skip:TBD] diff --git a/docs/reference/ml/trained-models/apis/stop-trained-model-deployment.asciidoc b/docs/reference/ml/trained-models/apis/stop-trained-model-deployment.asciidoc index 622b440622cd3..2f179e2391b78 100644 --- a/docs/reference/ml/trained-models/apis/stop-trained-model-deployment.asciidoc +++ b/docs/reference/ml/trained-models/apis/stop-trained-model-deployment.asciidoc @@ -6,6 +6,12 @@ Stop trained model deployment ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs]. +-- + Stops a trained model deployment. [[stop-trained-model-deployment-request]] diff --git a/docs/reference/ml/trained-models/apis/update-trained-model-deployment.asciidoc b/docs/reference/ml/trained-models/apis/update-trained-model-deployment.asciidoc index d49ee3c6e872c..a986e412f1a8a 100644 --- a/docs/reference/ml/trained-models/apis/update-trained-model-deployment.asciidoc +++ b/docs/reference/ml/trained-models/apis/update-trained-model-deployment.asciidoc @@ -7,6 +7,12 @@ Update trained model deployment ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-ml-trained-model[{ml-cap} trained model APIs]. +-- + Updates certain properties of a trained model deployment. [[update-trained-model-deployment-request]] diff --git a/docs/reference/modules/cluster/remote-clusters-migration.asciidoc b/docs/reference/modules/cluster/remote-clusters-migration.asciidoc index e205d7cb141fe..e84304ce9ef94 100644 --- a/docs/reference/modules/cluster/remote-clusters-migration.asciidoc +++ b/docs/reference/modules/cluster/remote-clusters-migration.asciidoc @@ -66,10 +66,11 @@ indices that were created from the auto-follow pattern. On the local cluster: . Enhance any roles used by local cluster users with the required -<> for {ccr} and {ccs}. +<> or +<> for {ccr} and {ccs}. Refer to <>. Note: -** You only need to assign additional `remote_indices` privileges to existing +** You only need to assign additional `remote_indices` or `remote_cluster` privileges to existing roles used for cross-cluster operations. You should be able to copy these privileges from the original roles on the remote cluster, where they are defined under the certification based security model. @@ -197,7 +198,7 @@ authentication. Resume any persistent tasks that you stopped earlier. Tasks should be restarted by the same user or API key that created the task before the migration. Ensure the roles of this user or API key have been updated with the required -`remote_indices` privileges. For users, tasks capture the caller's credentials +`remote_indices` or `remote_cluster` privileges. For users, tasks capture the caller's credentials when started and run in that user's security context. For API keys, restarting a task will update the task with the updated API key. @@ -246,7 +247,7 @@ If you need to roll back, follow these steps on the local cluster: . Remove the remote cluster definition by setting the remote cluster settings to `null`. -. Remove the `remote_indices` privileges from any roles that were updated during +. Remove the `remote_indices` or 'remote_cluster' privileges from any roles that were updated during the migration. . On each node, remove the `remote_cluster_client.ssl.*` settings from diff --git a/docs/reference/modules/cluster/remote-clusters-troubleshooting.asciidoc b/docs/reference/modules/cluster/remote-clusters-troubleshooting.asciidoc index df3c54794dc06..e21f93d81afc7 100644 --- a/docs/reference/modules/cluster/remote-clusters-troubleshooting.asciidoc +++ b/docs/reference/modules/cluster/remote-clusters-troubleshooting.asciidoc @@ -399,7 +399,7 @@ This does not show up in any logs. ====== Resolution -. Check that the local user has the necessary `remote_indices` privileges. Grant sufficient `remote_indices` privileges if necessary. +. Check that the local user has the necessary `remote_indices` or `remote_cluster` privileges. Grant sufficient `remote_indices` or `remote_cluster` privileges if necessary. . If permission is not an issue locally, ask the remote cluster administrator to create and distribute a <>. Replace the diff --git a/docs/reference/modules/discovery/bootstrapping.asciidoc b/docs/reference/modules/discovery/bootstrapping.asciidoc index 81ac3f6cc4cdc..5120c1d17e69b 100644 --- a/docs/reference/modules/discovery/bootstrapping.asciidoc +++ b/docs/reference/modules/discovery/bootstrapping.asciidoc @@ -5,7 +5,7 @@ Starting an Elasticsearch cluster for the very first time requires the initial set of <> to be explicitly defined on one or more of the master-eligible nodes in the cluster. This is known as _cluster bootstrapping_. This is only required the first time a cluster starts up. -Freshly-started nodes that are joining a running cluster obtain this +Freshly-started nodes that are joining a running cluster obtain this information from the cluster's elected master. The initial set of master-eligible nodes is defined in the @@ -27,17 +27,20 @@ node: if it is not possible to use the `node.name` of the node and there are multiple nodes sharing a single IP address. +Do not set `cluster.initial_master_nodes` on master-ineligible nodes. + [IMPORTANT] ==== After the cluster has formed, remove the `cluster.initial_master_nodes` setting -from each node's configuration. It should not be set for master-ineligible -nodes, master-eligible nodes joining an existing cluster, or nodes which are -restarting. - -If you leave `cluster.initial_master_nodes` in place once the cluster has -formed then there is a risk that a future misconfiguration may result in -bootstrapping a new cluster alongside your existing cluster. It may not be -possible to recover from this situation without losing data. +from each node's configuration and never set it again for this cluster. Do not +configure this setting on nodes joining an existing cluster. Do not configure +this setting on nodes which are restarting. Do not configure this setting when +performing a full-cluster restart. + +If you leave `cluster.initial_master_nodes` in place once the cluster has formed +then there is a risk that a future misconfiguration may result in bootstrapping +a new cluster alongside your existing cluster. It may not be possible to recover +from this situation without losing data. ==== The simplest way to create a new cluster is for you to select one of your diff --git a/docs/reference/modules/discovery/discovery-settings.asciidoc b/docs/reference/modules/discovery/discovery-settings.asciidoc index 401d1e7206d35..daf84f0292c12 100644 --- a/docs/reference/modules/discovery/discovery-settings.asciidoc +++ b/docs/reference/modules/discovery/discovery-settings.asciidoc @@ -46,8 +46,11 @@ setting, see <>. Sets the initial set of master-eligible nodes in a brand-new cluster. By default this list is empty, meaning that this node expects to join a cluster that has already been bootstrapped. Remove this setting once the cluster has -formed. Do not use this setting when restarting nodes or when adding new nodes -to an existing cluster. See <>. +formed, and never set it again for this cluster. Do not configure this setting +on master-ineligible nodes. Do not configure this setting on nodes joining an +existing cluster. Do not configure this setting on nodes which are restarting. +Do not configure this setting when performing a full-cluster restart. See +<>. [discrete] ==== Expert settings diff --git a/docs/reference/modules/discovery/voting.asciidoc b/docs/reference/modules/discovery/voting.asciidoc index 9e483d5883017..04cae9d02ab66 100644 --- a/docs/reference/modules/discovery/voting.asciidoc +++ b/docs/reference/modules/discovery/voting.asciidoc @@ -63,8 +63,7 @@ departed nodes from the voting configuration manually. Use the of resilience. No matter how it is configured, Elasticsearch will not suffer from a -"{wikipedia}/Split-brain_(computing)[split-brain]" inconsistency. -The `cluster.auto_shrink_voting_configuration` +"split-brain" inconsistency. The `cluster.auto_shrink_voting_configuration` setting affects only its availability in the event of the failure of some of its nodes and the administrative tasks that must be performed as nodes join and leave the cluster. diff --git a/docs/reference/modules/indices/circuit_breaker.asciidoc b/docs/reference/modules/indices/circuit_breaker.asciidoc index a5a787e23d170..13d81821c4f33 100644 --- a/docs/reference/modules/indices/circuit_breaker.asciidoc +++ b/docs/reference/modules/indices/circuit_breaker.asciidoc @@ -2,7 +2,16 @@ === Circuit breaker settings [[circuit-breaker-description]] // tag::circuit-breaker-description-tag[] -{es} contains multiple circuit breakers used to prevent operations from causing an OutOfMemoryError. Each breaker specifies a limit for how much memory it can use. Additionally, there is a parent-level breaker that specifies the total amount of memory that can be used across all breakers. +{es} contains multiple circuit breakers used to prevent operations from using an excessive amount of memory. Each breaker tracks the memory +used by certain operations and specifies a limit for how much memory it may track. Additionally, there +is a parent-level breaker that specifies the total amount of memory that may be tracked across all breakers. + +When a circuit breaker reaches its limit, {es} will reject further operations. See <> for information about errors +raised by circuit breakers. + +Circuit breakers do not track all memory usage in {es} and therefore provide only incomplete protection against excessive memory usage. If +{es} uses too much memory then it may suffer from performance issues and nodes may even fail with an `OutOfMemoryError`. See +<> for help with troubleshooting high heap usage. Except where noted otherwise, these settings can be dynamically updated on a live cluster with the <> API. @@ -175,7 +184,8 @@ an `OutOfMemory` exception which would bring down the node. To prevent this from happening, a special <> is used, which limits the memory allocation during the execution of a <> query. When the breaker is triggered, an `org.elasticsearch.common.breaker.CircuitBreakingException` -is thrown and a descriptive error message is returned to the user. +is thrown and a descriptive error message including `circuit_breaking_exception` +is returned to the user. This <> can be configured using the following settings: diff --git a/docs/reference/modules/network.asciidoc b/docs/reference/modules/network.asciidoc index 8fdc9f2e4f9cb..1e4c5a21d386c 100644 --- a/docs/reference/modules/network.asciidoc +++ b/docs/reference/modules/network.asciidoc @@ -153,23 +153,34 @@ The only requirements are that each node must be: * Accessible at its transport publish address by all other nodes in its cluster, and by any remote clusters that will discover it using - <>. + <>. Each node must have its own distinct publish address. If you specify the transport publish address using a hostname then {es} will resolve this hostname to an IP address once during startup, and other nodes will use the resulting IP address instead of resolving the name again -themselves. To avoid confusion, use a hostname which resolves to the node's -address in all network locations. +themselves. You must use a hostname such that all of the addresses to which it +resolves are addresses at which the node is accessible from all other nodes. To +avoid confusion, it is simplest to use a hostname which resolves to a single +address. + +If you specify the transport publish address using a +<> then {es} will resolve this value to +a single IP address during startup, and other nodes will use the resulting IP +address instead of resolving the value again themselves. You must use a value +such that all of the addresses to which it resolves are addresses at which the +node is accessible from all other nodes. To avoid confusion, it is simplest to +use a value which resolves to a single address. It is usually a mistake to use +`0.0.0.0` as a publish address on hosts with more than one network interface. ===== Using a single address The most common configuration is for {es} to bind to a single address at which -it is accessible to clients and other nodes. In this configuration you should -just set `network.host` to that address. You should not separately set any bind -or publish addresses, nor should you separately configure the addresses for the -HTTP or transport interfaces. +it is accessible to clients and other nodes. To use this configuration, set +only `network.host` to the desired address. Do not separately set any bind or +publish addresses. Do not separately specify the addresses for the HTTP or +transport interfaces. ===== Using multiple addresses diff --git a/docs/reference/modules/shard-ops.asciidoc b/docs/reference/modules/shard-ops.asciidoc index c0e5ee6a220f0..66ceebcfa0319 100644 --- a/docs/reference/modules/shard-ops.asciidoc +++ b/docs/reference/modules/shard-ops.asciidoc @@ -1,5 +1,5 @@ [[shard-allocation-relocation-recovery]] -=== Shard allocation, relocation, and recovery +== Shard allocation, relocation, and recovery Each <> in Elasticsearch is divided into one or more <>. Each document in an index belongs to a single shard. @@ -12,14 +12,16 @@ Over the course of normal operation, Elasticsearch allocates shard copies to nod TIP: To learn about optimizing the number and size of shards in your cluster, refer to <>. To learn about how read and write operations are replicated across shards and shard copies, refer to <>. +[discrete] [[shard-allocation]] -==== Shard allocation +=== Shard allocation include::{es-ref-dir}/modules/shard-allocation-desc.asciidoc[] By default, the primary and replica shard copies for an index can be allocated to any node in the cluster, and may be relocated to rebalance the cluster. -===== Adjust shard allocation settings +[discrete] +==== Adjust shard allocation settings You can control how shard copies are allocated using the following settings: @@ -27,7 +29,8 @@ You can control how shard copies are allocated using the following settings: - <>: Use these settings to control how the shard copies for a specific index are allocated. For example, you might want to allocate an index to a node in a specific data tier, or to an node with specific attributes. -===== Monitor shard allocation +[discrete] +==== Monitor shard allocation If a shard copy is unassigned, it means that the shard copy is not allocated to any node in the cluster. This can happen if there are not enough nodes in the cluster to allocate the shard copy, or if the shard copy can't be allocated to any node that satisfies the shard allocation filtering rules. When a shard copy is unassigned, your cluster is considered unhealthy and returns a yellow or red cluster health status. @@ -39,12 +42,14 @@ You can use the following APIs to monitor shard allocation: <>. +[discrete] [[shard-recovery]] -==== Shard recovery +=== Shard recovery include::{es-ref-dir}/modules/shard-recovery-desc.asciidoc[] -===== Adjust shard recovery settings +[discrete] +==== Adjust shard recovery settings To control how shards are recovered, for example the resources that can be used by recovery operations, and which indices should be prioritized for recovery, you can adjust the following settings: @@ -54,21 +59,24 @@ To control how shards are recovered, for example the resources that can be used Shard recovery operations also respect general shard allocation settings. -===== Monitor shard recovery +[discrete] +==== Monitor shard recovery You can use the following APIs to monitor shard allocation: - View a list of in-progress and completed recoveries using the <> - View detailed information about a specific recovery using the <> +[discrete] [[shard-relocation]] -==== Shard relocation +=== Shard relocation Shard relocation is the process of moving shard copies from one node to another. This can happen when a node joins or leaves the cluster, or when the cluster is rebalancing. When a shard copy is relocated, it is created as a new shard copy on the target node. When the shard copy is fully allocated and recovered, the old shard copy is deleted. If the shard copy being relocated is a primary, then the new shard copy is marked as primary before the old shard copy is deleted. -===== Adjust shard relocation settings +[discrete] +==== Adjust shard relocation settings You can control how and when shard copies are relocated. For example, you can adjust the rebalancing settings that control when shard copies are relocated to balance the cluster, or the high watermark for disk-based shard allocation that can trigger relocation. These settings are part of the <>. diff --git a/docs/reference/modules/threadpool.asciidoc b/docs/reference/modules/threadpool.asciidoc index ed4becbfbb6d0..8ae8f59c22982 100644 --- a/docs/reference/modules/threadpool.asciidoc +++ b/docs/reference/modules/threadpool.asciidoc @@ -13,16 +13,10 @@ There are several thread pools, but the important ones include: [[search-threadpool]] `search`:: - For coordination of count/search operations at the shard level whose computation - is offloaded to the search_worker thread pool. Used also by fetch and other search + For count/search operations at the shard level. Used also by fetch and other search related operations Thread pool type is `fixed` with a size of `int((`<>`pass:[ * ]3) / 2) + 1`, and queue_size of `1000`. -`search_worker`:: - For the heavy workload of count/search operations that may be executed concurrently - across segments within the same shard when possible. Thread pool type is `fixed` - with a size of `int((`<>`pass:[ * ]3) / 2) + 1`, and unbounded queue_size . - [[search-throttled]]`search_throttled`:: For count/search/suggest/get operations on `search_throttled indices`. Thread pool type is `fixed` with a size of `1`, and queue_size of `100`. @@ -121,6 +115,11 @@ There are several thread pools, but the important ones include: `min(5 * (`<>`), 50)` and queue_size of `1000`. +[[modules-threadpool-esql]]`esql_worker`:: + Executes <> operations. Thread pool type is `fixed` with a + size of `int((`<> + `pass:[ * ]3) / 2) + 1`, and queue_size of `1000`. + Thread pool settings are <> and can be changed by editing `elasticsearch.yml`. Changing a specific thread pool can be done by setting its type-specific parameters; for example, changing the number of diff --git a/docs/reference/nodes-shards.asciidoc b/docs/reference/nodes-shards.asciidoc new file mode 100644 index 0000000000000..11095ed7b7eb3 --- /dev/null +++ b/docs/reference/nodes-shards.asciidoc @@ -0,0 +1,43 @@ +[[nodes-shards]] +== Nodes and shards + +[NOTE] +==== +Nodes and shards are what make {es} distributed and scalable. +These concepts aren't essential if you're just getting started. How you <> in production determines what you need to know: + +* *Self-managed {es}*: You are responsible for setting up and managing nodes, clusters, shards, and replicas. This includes managing the underlying infrastructure, scaling, and ensuring high availability through failover and backup strategies. +* *Elastic Cloud*: Elastic can autoscale resources in response to workload changes. Choose from different deployment types to apply sensible defaults for your use case. A basic understanding of nodes, shards, and replicas is still important. +* *Elastic Cloud Serverless*: You don't need to worry about nodes, shards, or replicas. These resources are 100% automated on the serverless platform, which is designed to scale with your workload. +==== + +You can add servers (_nodes_) to a cluster to increase capacity, and {es} automatically distributes your data and query load across all of the available nodes. + +Elastic is able to distribute your data across nodes by subdividing an index into _shards_. Each index in {es} is a grouping +of one or more physical shards, where each shard is a self-contained Lucene index containing a subset of the documents in +the index. By distributing the documents in an index across multiple shards, and distributing those shards across multiple +nodes, {es} increases indexing and query capacity. + +There are two types of shards: _primaries_ and _replicas_. Each document in an index belongs to one primary shard. A replica +shard is a copy of a primary shard. Replicas maintain redundant copies of your data across the nodes in your cluster. +This protects against hardware failure and increases capacity to serve read requests like searching or retrieving a document. + +[TIP] +==== +The number of primary shards in an index is fixed at the time that an index is created, but the number of replica shards can +be changed at any time, without interrupting indexing or query operations. +==== + +Shard copies in your cluster are automatically balanced across nodes to provide scale and high availability. All nodes are +aware of all the other nodes in the cluster and can forward client requests to the appropriate node. This allows {es} +to distribute indexing and query load across the cluster. + +If you're exploring {es} for the first time or working in a development environment, then you can use a cluster with a single node and create indices +with only one shard. However, in a production environment, you should build a cluster with multiple nodes and indices +with multiple shards to increase performance and resilience. + +// TODO - diagram + +* To learn about optimizing the number and size of shards in your cluster, refer to <>. +* To learn about how read and write operations are replicated across shards and shard copies, refer to <>. +* To adjust how shards are allocated and balanced across nodes, refer to <>. \ No newline at end of file diff --git a/docs/reference/query-dsl/intervals-query.asciidoc b/docs/reference/query-dsl/intervals-query.asciidoc index 1e3380389d861..069021dddb69f 100644 --- a/docs/reference/query-dsl/intervals-query.asciidoc +++ b/docs/reference/query-dsl/intervals-query.asciidoc @@ -73,7 +73,9 @@ Valid rules include: * <> * <> * <> +* <> * <> +* <> * <> * <> -- @@ -122,8 +124,9 @@ unstemmed ones. ==== `prefix` rule parameters The `prefix` rule matches terms that start with a specified set of characters. -This prefix can expand to match at most 128 terms. If the prefix matches more -than 128 terms, {es} returns an error. You can use the +This prefix can expand to match at most `indices.query.bool.max_clause_count` +<> terms. If the prefix matches more terms, +{es} returns an error. You can use the <> option in the field mapping to avoid this limit. @@ -149,7 +152,8 @@ separate `analyzer` is specified. ==== `wildcard` rule parameters The `wildcard` rule matches terms using a wildcard pattern. This pattern can -expand to match at most 128 terms. If the pattern matches more than 128 terms, +expand to match at most `indices.query.bool.max_clause_count` +<> terms. If the pattern matches more terms, {es} returns an error. `pattern`:: @@ -178,12 +182,44 @@ The `pattern` is normalized using the search analyzer from this field, unless `analyzer` is specified separately. -- +[[intervals-regexp]] +==== `regexp` rule parameters + +The `regexp` rule matches terms using a regular expression pattern. +This pattern can expand to match at most `indices.query.bool.max_clause_count` +<> terms. +If the pattern matches more terms,{es} returns an error. + +`pattern`:: +(Required, string) Regexp pattern used to find matching terms. +For a list of operators supported by the +`regexp` pattern, see <>. + +WARNING: Avoid using wildcard patterns, such as `.*` or `.*?+``. This can +increase the iterations needed to find matching terms and slow search +performance. +-- +`analyzer`:: +(Optional, string) <> used to normalize the `pattern`. +Defaults to the top-level ``'s analyzer. +-- +`use_field`:: ++ +-- +(Optional, string) If specified, match intervals from this field rather than the +top-level ``. + +The `pattern` is normalized using the search analyzer from this field, unless +`analyzer` is specified separately. +-- + [[intervals-fuzzy]] ==== `fuzzy` rule parameters The `fuzzy` rule matches terms that are similar to the provided term, within an edit distance defined by <>. If the fuzzy expansion matches more than -128 terms, {es} returns an error. +`indices.query.bool.max_clause_count` +<> terms, {es} returns an error. `term`:: (Required, string) The term to match @@ -214,6 +250,41 @@ The `term` is normalized using the search analyzer from this field, unless `analyzer` is specified separately. -- +[[intervals-range]] +==== `range` rule parameters + +The `range` rule matches terms contained within a provided range. +This range can expand to match at most `indices.query.bool.max_clause_count` +<> terms. +If the range matches more terms,{es} returns an error. + +`gt`:: +(Optional, string) Greater than: match terms greater than the provided term. + +`gte`:: +(Optional, string) Greater than or equal to: match terms greater than or +equal to the provided term. + +`lt`:: +(Optional, string) Less than: match terms less than the provided term. + +`lte`:: +(Optional, string) Less than or equal to: match terms less than or +equal to the provided term. + +NOTE: It is required to provide one of `gt` or `gte` params. +It is required to provide one of `lt` or `lte` params. + + +`analyzer`:: +(Optional, string) <> used to normalize the `pattern`. +Defaults to the top-level ``'s analyzer. + +`use_field`:: +(Optional, string) If specified, match intervals from this field rather than the +top-level ``. + + [[intervals-all_of]] ==== `all_of` rule parameters diff --git a/docs/reference/query-dsl/knn-query.asciidoc b/docs/reference/query-dsl/knn-query.asciidoc index 05a00b9949912..e42bd78d9f14a 100644 --- a/docs/reference/query-dsl/knn-query.asciidoc +++ b/docs/reference/query-dsl/knn-query.asciidoc @@ -100,7 +100,7 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=knn-query-vector-builde -- (Optional, integer) The number of nearest neighbors to return from each shard. {es} collects `k` results from each shard, then merges them to find the global top results. -This value must be less than or equal to `num_candidates`. Defaults to `num_candidates`. +This value must be less than or equal to `num_candidates`. Defaults to search request size. -- `num_candidates`:: @@ -241,7 +241,7 @@ to <>: * kNN search over nested dense_vectors diversifies the top results over the top-level document * `filter` over the top-level document metadata is supported and acts as a -post-filter +pre-filter * `filter` over `nested` field metadata is not supported A sample query can look like below: diff --git a/docs/reference/query-dsl/query_filter_context.asciidoc b/docs/reference/query-dsl/query_filter_context.asciidoc index 78e1549644aa6..1fd75ef0e841d 100644 --- a/docs/reference/query-dsl/query_filter_context.asciidoc +++ b/docs/reference/query-dsl/query_filter_context.asciidoc @@ -29,26 +29,45 @@ parameter, such as the `query` parameter in the [discrete] [[filter-context]] === Filter context -In a filter context, a query clause answers the question ``__Does this -document match this query clause?__'' The answer is a simple Yes or No -- no -scores are calculated. Filter context is mostly used for filtering structured -data, e.g. -* __Does this +timestamp+ fall into the range 2015 to 2016?__ -* __Is the +status+ field set to ++"published"++__? +A filter answers the binary question “Does this document match this query clause?”. The answer is simply "yes" or "no". +Filtering has several benefits: -Frequently used filters will be cached automatically by Elasticsearch, to -speed up performance. +. *Simple binary logic*: In a filter context, a query clause determines document matches based on a yes/no criterion, without score calculation. +. *Performance*: Because they don't compute relevance scores, filters execute faster than queries. +. *Caching*: {es} automatically caches frequently used filters, speeding up subsequent search performance. +. *Resource efficiency*: Filters consume less CPU resources compared to full-text queries. +. *Query combination*: Filters can be combined with scored queries to refine result sets efficiently. -Filter context is in effect whenever a query clause is passed to a `filter` -parameter, such as the `filter` or `must_not` parameters in the -<> query, the `filter` parameter in the -<> query, or the -<> aggregation. +Filters are particularly effective for querying structured data and implementing "must have" criteria in complex searches. + +Structured data refers to information that is highly organized and formatted in a predefined manner. In the context of Elasticsearch, this typically includes: + +* Numeric fields (integers, floating-point numbers) +* Dates and timestamps +* Boolean values +* Keyword fields (exact match strings) +* Geo-points and geo-shapes + +Unlike full-text fields, structured data has a consistent, predictable format, making it ideal for precise filtering operations. + +Common filter applications include: + +* Date range checks: for example is the `timestamp` field between 2015 and 2016 +* Specific field value checks: for example is the `status` field equal to "published" or is the `author` field equal to "John Doe" + +Filter context applies when a query clause is passed to a `filter` parameter, such as: + +* `filter` or `must_not` parameters in <> queries +* `filter` parameter in <> queries +* <> aggregations + +Filters optimize query performance and efficiency, especially for structured data queries and when combined with full-text searches. [discrete] [[query-filter-context-ex]] === Example of query and filter contexts + Below is an example of query clauses being used in query and filter context in the `search` API. This query will match documents where all of the following conditions are met: @@ -93,4 +112,4 @@ significand's precision will be converted to floats with loss of precision. TIP: Use query clauses in query context for conditions which should affect the score of matching documents (i.e. how well does the document match), and use -all other query clauses in filter context. \ No newline at end of file +all other query clauses in filter context. diff --git a/docs/reference/query-dsl/rule-query.asciidoc b/docs/reference/query-dsl/rule-query.asciidoc index dfedc2261bbde..43e79f656a55a 100644 --- a/docs/reference/query-dsl/rule-query.asciidoc +++ b/docs/reference/query-dsl/rule-query.asciidoc @@ -12,6 +12,12 @@ The old syntax using `rule_query` and `ruleset_id` is deprecated and will be removed in a future release, so it is strongly advised to migrate existing rule queries to the new API structure. ==== +[TIP] +==== +The rule query is not supported for use alongside reranking. +If you want to use query rules in conjunction with reranking, use the <> instead. +==== + Applies <> to the query before returning results. Query rules can be used to promote documents in the manner of a <> based on matching defined rules, or to identify specific documents to exclude from a contextual result set. If no matching query rules are defined, the "organic" matches for the query are returned. diff --git a/docs/reference/query-dsl/script-score-query.asciidoc b/docs/reference/query-dsl/script-score-query.asciidoc index 9291b8c15f0d1..051c9c6f9c32d 100644 --- a/docs/reference/query-dsl/script-score-query.asciidoc +++ b/docs/reference/query-dsl/script-score-query.asciidoc @@ -62,10 +62,17 @@ multiplied by `boost` to produce final documents' scores. Defaults to `1.0`. ===== Use relevance scores in a script Within a script, you can -{ref}/modules-scripting-fields.html#scripting-score[access] +{ref}/modules-scripting-fields.html#scripting-score[access] the `_score` variable which represents the current relevance score of a document. +[[script-score-access-term-statistics]] +===== Use term statistics in a script + +Within a script, you can +{ref}/modules-scripting-fields.html#scripting-term-statistics[access] +the `_termStats` variable which provides statistical information about the terms used in the child query of the `script_score` query. + [[script-score-predefined-functions]] ===== Predefined functions You can use any of the available {painless}/painless-contexts.html[painless @@ -147,7 +154,7 @@ updated since update operations also update the value of the `_seq_no` field. [[decay-functions-numeric-fields]] ====== Decay functions for numeric fields -You can read more about decay functions +You can read more about decay functions {ref}/query-dsl-function-score-query.html#function-decay[here]. * `double decayNumericLinear(double origin, double scale, double offset, double decay, double docValue)` @@ -233,7 +240,7 @@ The `script_score` query calculates the score for every matching document, or hit. There are faster alternative query types that can efficiently skip non-competitive hits: -* If you want to boost documents on some static fields, use the +* If you want to boost documents on some static fields, use the <> query. * If you want to boost documents closer to a date or geographic point, use the <> query. diff --git a/docs/reference/query-dsl/semantic-query.asciidoc b/docs/reference/query-dsl/semantic-query.asciidoc index 22b5e6c5e6aad..914f4429f7f9c 100644 --- a/docs/reference/query-dsl/semantic-query.asciidoc +++ b/docs/reference/query-dsl/semantic-query.asciidoc @@ -25,7 +25,7 @@ GET my-index-000001/_search } } ------------------------------------------------------------ -// TEST[skip:TBD] +// TEST[skip: Requires inference endpoints] [discrete] @@ -79,7 +79,7 @@ POST my-index/_search } } ------------------------------------------------------------ -// TEST[skip:TBD] +// TEST[skip: Requires inference endpoints] You can also use semantic_text as part of <> to make ranking relevant results easier: @@ -116,80 +116,4 @@ GET my-index/_search } } ------------------------------------------------------------ -// TEST[skip:TBD] - - -[discrete] -[[advanced-search]] -=== Advanced search on `semantic_text` fields - -The `semantic` query uses default settings for searching on `semantic_text` fields for ease of use. -If you want to fine-tune a search on a `semantic_text` field, you need to know the task type used by the `inference_id` configured in `semantic_text`. -You can find the task type using the <>, and check the `task_type` associated with the {infer} service. -Depending on the `task_type`, use either the <> or the <> query for greater flexibility and customization. - -NOTE: While it is possible to use the `sparse_vector` query or the `knn` query -on a `semantic_text` field, it is not supported to use the `semantic_query` on a -`sparse_vector` or `dense_vector` field type. - - -[discrete] -[[search-sparse-inference]] -==== Search with `sparse_embedding` inference - -When the {infer} endpoint uses a `sparse_embedding` model, you can use a <> on a <> field in the following way: - -[source,console] ------------------------------------------------------------- -GET test-index/_search -{ - "query": { - "nested": { - "path": "inference_field.inference.chunks", - "query": { - "sparse_vector": { - "field": "inference_field.inference.chunks.embeddings", - "inference_id": "my-inference-id", - "query": "mountain lake" - } - } - } - } -} ------------------------------------------------------------- -// TEST[skip:TBD] - -You can customize the `sparse_vector` query to include specific settings, like <>. - - -[discrete] -[[search-text-inferece]] -==== Search with `text_embedding` inference - -When the {infer} endpoint uses a `text_embedding` model, you can use a <> on a `semantic_text` field in the following way: - -[source,console] ------------------------------------------------------------- -GET test-index/_search -{ - "query": { - "nested": { - "path": "inference_field.inference.chunks", - "query": { - "knn": { - "field": "inference_field.inference.chunks.embeddings", - "query_vector_builder": { - "text_embedding": { - "model_id": "my_inference_id", - "model_text": "mountain lake" - } - } - } - } - } - } -} ------------------------------------------------------------- -// TEST[skip:TBD] - -You can customize the `knn` query to include specific settings, like `num_candidates` and `k`. +// TEST[skip: Requires inference endpoints] diff --git a/docs/reference/query-dsl/sparse-vector-query.asciidoc b/docs/reference/query-dsl/sparse-vector-query.asciidoc index 08dd7ab7f4470..399cf29d4dd12 100644 --- a/docs/reference/query-dsl/sparse-vector-query.asciidoc +++ b/docs/reference/query-dsl/sparse-vector-query.asciidoc @@ -104,7 +104,7 @@ Default: `5`. `tokens_weight_threshold`:: (Optional, float) preview:[] -Tokens whose weight is less than `tokens_weight_threshold` are considered nonsignificant and pruned. +Tokens whose weight is less than `tokens_weight_threshold` are considered insignificant and pruned. This value must be between 0 and 1. Default: `0.4`. diff --git a/docs/reference/query-dsl/terms-set-query.asciidoc b/docs/reference/query-dsl/terms-set-query.asciidoc index 2abfe54d53976..27717af3ac171 100644 --- a/docs/reference/query-dsl/terms-set-query.asciidoc +++ b/docs/reference/query-dsl/terms-set-query.asciidoc @@ -159,12 +159,22 @@ GET /job-candidates/_search `terms`:: + -- -(Required, array of strings) Array of terms you wish to find in the provided +(Required, array) Array of terms you wish to find in the provided ``. To return a document, a required number of terms must exactly match the field values, including whitespace and capitalization. -The required number of matching terms is defined in the -`minimum_should_match_field` or `minimum_should_match_script` parameter. +The required number of matching terms is defined in the `minimum_should_match`, +`minimum_should_match_field` or `minimum_should_match_script` parameters. Exactly +one of these parameters must be provided. +-- + +`minimum_should_match`:: ++ +-- +(Optional) Specification for the number of matching terms required to return +a document. + +For valid values, see <>. -- `minimum_should_match_field`:: diff --git a/docs/reference/query-dsl/text-expansion-query.asciidoc b/docs/reference/query-dsl/text-expansion-query.asciidoc index 8faecad1dbdb9..5c7bce8c3fcf0 100644 --- a/docs/reference/query-dsl/text-expansion-query.asciidoc +++ b/docs/reference/query-dsl/text-expansion-query.asciidoc @@ -7,6 +7,13 @@ deprecated[8.15.0, This query has been replaced by <>.] +.Deprecation usage note +**** +You can continue using `rank_features` fields with `text_expansion` queries in the current version. +However, if you plan to upgrade, we recommend updating mappings to use the `sparse_vector` field type and <>. +This will allow you to take advantage of the new capabilities and improvements available in newer versions. +**** + The text expansion query uses a {nlp} model to convert the query text into a list of token-weight pairs which are then used in a query against a <> or <> field. @@ -68,7 +75,7 @@ Default: `5`. `tokens_weight_threshold`:: (Optional, float) preview:[] -Tokens whose weight is less than `tokens_weight_threshold` are considered nonsignificant and pruned. +Tokens whose weight is less than `tokens_weight_threshold` are considered insignificant and pruned. This value must be between 0 and 1. Default: `0.4`. diff --git a/docs/reference/query-dsl/weighted-tokens-query.asciidoc b/docs/reference/query-dsl/weighted-tokens-query.asciidoc index d4318665a9778..fb051f4229df6 100644 --- a/docs/reference/query-dsl/weighted-tokens-query.asciidoc +++ b/docs/reference/query-dsl/weighted-tokens-query.asciidoc @@ -58,7 +58,7 @@ This value must between 1 and 100. Default: `5`. `tokens_weight_threshold`:: -(Optional, float) Tokens whose weight is less than `tokens_weight_threshold` are considered nonsignificant and pruned. +(Optional, float) Tokens whose weight is less than `tokens_weight_threshold` are considered insignificant and pruned. This value must be between 0 and 1. Default: `0.4`. diff --git a/docs/reference/query-rules/apis/delete-query-rule.asciidoc b/docs/reference/query-rules/apis/delete-query-rule.asciidoc index 01b73033aa361..4d91092eaf8af 100644 --- a/docs/reference/query-rules/apis/delete-query-rule.asciidoc +++ b/docs/reference/query-rules/apis/delete-query-rule.asciidoc @@ -6,6 +6,12 @@ Delete query rule ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-query_rules[Query rules APIs]. +-- + Removes an individual query rule within an existing query ruleset. This is a destructive action that is only recoverable by re-adding the same rule via the <> API. diff --git a/docs/reference/query-rules/apis/delete-query-ruleset.asciidoc b/docs/reference/query-rules/apis/delete-query-ruleset.asciidoc index 31507dce3d12d..168310dcd4078 100644 --- a/docs/reference/query-rules/apis/delete-query-ruleset.asciidoc +++ b/docs/reference/query-rules/apis/delete-query-ruleset.asciidoc @@ -6,6 +6,12 @@ Delete query ruleset ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-query_rules[Query rules APIs]. +-- + Removes a query ruleset and its associated data. This is a destructive action that is not recoverable. diff --git a/docs/reference/query-rules/apis/get-query-rule.asciidoc b/docs/reference/query-rules/apis/get-query-rule.asciidoc index 56713965d7bdc..742982e5897e1 100644 --- a/docs/reference/query-rules/apis/get-query-rule.asciidoc +++ b/docs/reference/query-rules/apis/get-query-rule.asciidoc @@ -6,6 +6,12 @@ Get query rule ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-query_rules[Query rules APIs]. +-- + Retrieves information about an individual query rule within a query ruleset. [[get-query-rule-request]] diff --git a/docs/reference/query-rules/apis/get-query-ruleset.asciidoc b/docs/reference/query-rules/apis/get-query-ruleset.asciidoc index 6bbcd157ea9e1..55574fb7c67e9 100644 --- a/docs/reference/query-rules/apis/get-query-ruleset.asciidoc +++ b/docs/reference/query-rules/apis/get-query-ruleset.asciidoc @@ -6,6 +6,12 @@ Get query ruleset ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-query_rules[Query rules APIs]. +-- + Retrieves information about a query ruleset. [[get-query-ruleset-request]] diff --git a/docs/reference/query-rules/apis/index.asciidoc b/docs/reference/query-rules/apis/index.asciidoc index 53d5fc3dc4eee..7dcdf0e8f4e1c 100644 --- a/docs/reference/query-rules/apis/index.asciidoc +++ b/docs/reference/query-rules/apis/index.asciidoc @@ -7,6 +7,12 @@ --- +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-query_rules[Query rules APIs]. +-- + <> allow you to configure per-query rules that are applied at query time to queries that match the specific rule. Query rules are organized into _rulesets_, collections of query rules that are matched against incoming queries. Query rules are applied using the <>. @@ -23,6 +29,7 @@ Use the following APIs to manage query rulesets: * <> * <> * <> +* preview:[] <> include::put-query-ruleset.asciidoc[] include::get-query-ruleset.asciidoc[] @@ -31,4 +38,5 @@ include::delete-query-ruleset.asciidoc[] include::put-query-rule.asciidoc[] include::get-query-rule.asciidoc[] include::delete-query-rule.asciidoc[] +include::test-query-ruleset.asciidoc[] diff --git a/docs/reference/query-rules/apis/list-query-rulesets.asciidoc b/docs/reference/query-rules/apis/list-query-rulesets.asciidoc index 6832934f6985c..5f61bcb98b085 100644 --- a/docs/reference/query-rules/apis/list-query-rulesets.asciidoc +++ b/docs/reference/query-rules/apis/list-query-rulesets.asciidoc @@ -6,6 +6,12 @@ List query rulesets ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-query_rules[Query rules APIs]. +-- + Returns information about all stored query rulesets. Summary information on the number of rules per ruleset will be returned, and full details can be returned with the <> command. @@ -124,7 +130,7 @@ PUT _query_rules/ruleset-3 }, { "rule_id": "rule-3", - "type": "pinned", + "type": "exclude", "criteria": [ { "type": "fuzzy", @@ -178,6 +184,9 @@ A sample response: "rule_total_count": 1, "rule_criteria_types_counts": { "exact": 1 + }, + "rule_type_counts": { + "pinned": 1 } }, { @@ -186,6 +195,9 @@ A sample response: "rule_criteria_types_counts": { "exact": 1, "fuzzy": 1 + }, + "rule_type_counts": { + "pinned": 2 } }, { @@ -194,6 +206,10 @@ A sample response: "rule_criteria_types_counts": { "exact": 1, "fuzzy": 2 + }, + "rule_type_counts": { + "pinned": 2, + "exclude": 1 } } ] diff --git a/docs/reference/query-rules/apis/put-query-rule.asciidoc b/docs/reference/query-rules/apis/put-query-rule.asciidoc index 714ed9b096d1d..df33d22b39029 100644 --- a/docs/reference/query-rules/apis/put-query-rule.asciidoc +++ b/docs/reference/query-rules/apis/put-query-rule.asciidoc @@ -6,6 +6,12 @@ Create or update query rule ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-query_rules[Query rules APIs]. +-- + Creates or updates an individual query rule within a query ruleset. [[put-query-rule-request]] diff --git a/docs/reference/query-rules/apis/put-query-ruleset.asciidoc b/docs/reference/query-rules/apis/put-query-ruleset.asciidoc index df7ec100db076..ea689dc0bf305 100644 --- a/docs/reference/query-rules/apis/put-query-ruleset.asciidoc +++ b/docs/reference/query-rules/apis/put-query-ruleset.asciidoc @@ -6,6 +6,12 @@ Create or update query ruleset ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-query_rules[Query rules APIs]. +-- + Creates or updates a query ruleset. [[put-query-ruleset-request]] diff --git a/docs/reference/query-rules/apis/test-query-ruleset.asciidoc b/docs/reference/query-rules/apis/test-query-ruleset.asciidoc new file mode 100644 index 0000000000000..4a47754f572e5 --- /dev/null +++ b/docs/reference/query-rules/apis/test-query-ruleset.asciidoc @@ -0,0 +1,139 @@ +[role="xpack"] +[[test-query-ruleset]] +=== Test query ruleset + +++++ +Tests query ruleset +++++ + +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-query_rules[Query rules APIs]. +-- + +Evaluates match criteria against a query ruleset to identify the rules that would match that criteria. + +preview::[] + +[[test-query-ruleset-request]] +==== {api-request-title} + +`POST _query_rules//_test` + +[[test-query-ruleset-prereq]] +==== {api-prereq-title} + +Requires the `manage_search_query_rules` privilege. + +[[test-query-ruleset-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) + +[[test-query-rule-request-body]] +==== {api-request-body-title} + +`match_criteria`:: +(Required, object) Defines the match criteria to apply to rules in the given query ruleset. +Match criteria should match the keys defined in the `criteria.metadata` field of the rule. + +[[test-query-ruleset-response-codes]] +==== {api-response-codes-title} + +`400`:: +The `ruleset_id` or `match_criteria` were not provided. + +`404` (Missing resources):: +No query ruleset matching `ruleset_id` could be found. + +[[test-query-ruleset-example]] +==== {api-examples-title} + +To test a ruleset, provide the match criteria that you want to test against: + +//// + +[source,console] +-------------------------------------------------- +PUT _query_rules/my-ruleset +{ + "rules": [ + { + "rule_id": "my-rule1", + "type": "pinned", + "criteria": [ + { + "type": "contains", + "metadata": "query_string", + "values": [ "pugs", "puggles" ] + } + ], + "actions": { + "ids": [ + "id1", + "id2" + ] + } + }, + { + "rule_id": "my-rule2", + "type": "pinned", + "criteria": [ + { + "type": "fuzzy", + "metadata": "query_string", + "values": [ "rescue dogs" ] + } + ], + "actions": { + "docs": [ + { + "_index": "index1", + "_id": "id3" + }, + { + "_index": "index2", + "_id": "id4" + } + ] + } + } + ] +} +-------------------------------------------------- +// TESTSETUP + +[source,console] +-------------------------------------------------- +DELETE _query_rules/my-ruleset +-------------------------------------------------- +// TEARDOWN + +//// + +[source,console] +---- +POST _query_rules/my-ruleset/_test +{ + "match_criteria": { + "query_string": "puggles" + } +} +---- + +A sample response: + +[source,console-result] +---- +{ + "total_matched_rules": 1, + "matched_rules": [ + { + "ruleset_id": "my-ruleset", + "rule_id": "my-rule1" + } + ] +} +---- diff --git a/docs/reference/quickstart/aggs-tutorial.asciidoc b/docs/reference/quickstart/aggs-tutorial.asciidoc new file mode 100644 index 0000000000000..0a8494c3eb75d --- /dev/null +++ b/docs/reference/quickstart/aggs-tutorial.asciidoc @@ -0,0 +1,2184 @@ +[[aggregations-tutorial]] +== Analyze eCommerce data with aggregations using Query DSL +++++ +Basics: Analyze eCommerce data with aggregations +++++ + +This hands-on tutorial shows you how to analyze eCommerce data using {es} <> with the `_search` API and Query DSL. + +You'll learn how to: + +* Calculate key business metrics such as average order value +* Analyze sales patterns over time +* Compare performance across product categories +* Track moving averages and cumulative totals + +[discrete] +[[aggregations-tutorial-requirements]] +=== Requirements + +You'll need: + +. A running instance of <>, either on {serverless-full} or together with {kib} on Elastic Cloud Hosted/Self Managed deployments. +** If you don't have a deployment, you can run the following command in your terminal to set up a <>: ++ +[source,sh] +---- +curl -fsSL https://elastic.co/start-local | sh +---- +// NOTCONSOLE +. The {kibana-ref}/get-started.html#gs-get-data-into-kibana[sample eCommerce data] loaded into {es}. To load sample data follow these steps in your UI: +* Open the *Integrations* pages by searching in the global search field. +* Search for `sample data` in the **Integrations** search field. +* Open the *Sample data* page. +* Select the *Other sample data sets* collapsible. +* Add the *Sample eCommerce orders* data set. +This will create and populate an index called `kibana_sample_data_ecommerce`. + +[discrete] +[[aggregations-tutorial-inspect-data]] +=== Inspect index structure + +Before we start analyzing the data, let's examine the structure of the documents in our sample eCommerce index. Run this command to see the field <>: + +[source,console] +---- +GET kibana_sample_data_ecommerce/_mapping +---- +// TEST[skip:Using Kibana sample data] + +The response shows the field mappings for the `kibana_sample_data_ecommerce` index. + +.Example response +[%collapsible] +============== +[source,console-response] +---- +{ + "kibana_sample_data_ecommerce": { + "mappings": { + "properties": { + "category": { + "type": "text", + "fields": { <1> + "keyword": { + "type": "keyword" + } + } + }, + "currency": { + "type": "keyword" + }, + "customer_birth_date": { + "type": "date" + }, + "customer_first_name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "customer_full_name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "customer_gender": { + "type": "keyword" + }, + "customer_id": { + "type": "keyword" + }, + "customer_last_name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "customer_phone": { + "type": "keyword" + }, + "day_of_week": { + "type": "keyword" + }, + "day_of_week_i": { + "type": "integer" + }, + "email": { + "type": "keyword" + }, + "event": { + "properties": { + "dataset": { + "type": "keyword" + } + } + }, + "geoip": { + "properties": { <2> + "city_name": { + "type": "keyword" + }, + "continent_name": { + "type": "keyword" + }, + "country_iso_code": { + "type": "keyword" + }, + "location": { + "type": "geo_point" <3> + }, + "region_name": { + "type": "keyword" + } + } + }, + "manufacturer": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "order_date": { + "type": "date" + }, + "order_id": { + "type": "keyword" + }, + "products": { + "properties": { <4> + "_id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "base_price": { + "type": "half_float" + }, + "base_unit_price": { + "type": "half_float" + }, + "category": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "created_on": { + "type": "date" + }, + "discount_amount": { + "type": "half_float" + }, + "discount_percentage": { + "type": "half_float" + }, + "manufacturer": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "min_price": { + "type": "half_float" + }, + "price": { + "type": "half_float" + }, + "product_id": { + "type": "long" + }, + "product_name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + }, + "analyzer": "english" + }, + "quantity": { + "type": "integer" + }, + "sku": { + "type": "keyword" + }, + "tax_amount": { + "type": "half_float" + }, + "taxful_price": { + "type": "half_float" + }, + "taxless_price": { + "type": "half_float" + }, + "unit_discount_amount": { + "type": "half_float" + } + } + }, + "sku": { + "type": "keyword" + }, + "taxful_total_price": { + "type": "half_float" + }, + "taxless_total_price": { + "type": "half_float" + }, + "total_quantity": { + "type": "integer" + }, + "total_unique_products": { + "type": "integer" + }, + "type": { + "type": "keyword" + }, + "user": { + "type": "keyword" + } + } + } + } +} +---- +<1> `fields`: Multi-field mapping that allows both full text and exact matching +<2> `geoip.properties`: Object type field containing location-related properties +<3> `geoip.location`: Geographic coordinates stored as geo_point for location-based queries +<4> `products.properties`: Nested structure containing details about items in each order +============== + +The sample data includes the following <>: + +* <> and <> for text fields +** Most `text` fields have a `.keyword` subfield for exact matching using <> +* <> for date fields +* 3 <> types: +** `integer` for whole numbers +** `long` for large whole numbers +** `half_float` for floating-point numbers +* <> for geographic coordinates +* <> for nested structures such as `products`, `geoip`, `event` + +Now that we understand the structure of our sample data, let's start analyzing it. + +[discrete] +[[aggregations-tutorial-basic-metrics]] +=== Get key business metrics + +Let's start by calculating important metrics about orders and customers. + +[discrete] +[[aggregations-tutorial-order-value]] +==== Get average order size + +Calculate the average order value across all orders in the dataset using the <> aggregation. + +[source,console] +---- +GET kibana_sample_data_ecommerce/_search +{ + "size": 0, <1> + "aggs": { + "avg_order_value": { <2> + "avg": { <3> + "field": "taxful_total_price" + } + } + } +} +---- +// TEST[skip:Using Kibana sample data] +<1> Set `size` to 0 to avoid returning matched documents in the response and return only the aggregation results +<2> A meaningful name that describes what this metric represents +<3> Configures an `avg` aggregation, which calculates a simple arithmetic mean + +.Example response +[%collapsible] +============== +[source,console-result] +---- +{ + "took": 0, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 4675, <1> + "relation": "eq" + }, + "max_score": null, + "hits": [] <2> + }, + "aggregations": { + "avg_order_value": { <3> + "value": 75.05542864304813 <4> + } + } +} +---- +// TEST[skip:Using Kibana sample data] +<1> Total number of orders in the dataset +<2> `hits` is empty because we set `size` to 0 +<3> Results appear under the name we specified in the request +<4> The average order value is calculated dynamically from all the orders in the dataset +============== + +[discrete] +[[aggregations-tutorial-order-stats]] +==== Get multiple order statistics at once + +Calculate multiple statistics about orders in one request using the <> aggregation. + +[source,console] +---- +GET kibana_sample_data_ecommerce/_search +{ + "size": 0, + "aggs": { + "order_stats": { <1> + "stats": { <2> + "field": "taxful_total_price" + } + } + } +} +---- +// TEST[skip:Using Kibana sample data] +<1> A descriptive name for this set of statistics +<2> `stats` returns count, min, max, avg, and sum at once + +.Example response +[%collapsible] +============== +[source,console-result] +---- +{ + "aggregations": { + "order_stats": { + "count": 4675, <1> + "min": 6.98828125, <2> + "max": 2250, <3> + "avg": 75.05542864304813, <4> + "sum": 350884.12890625 <5> + } + } +} +---- +// TEST[skip:Using Kibana sample data] +<1> `"count"`: Total number of orders in the dataset +<2> `"min"`: Lowest individual order value in the dataset +<3> `"max"`: Highest individual order value in the dataset +<4> `"avg"`: Average value per order across all orders +<5> `"sum"`: Total revenue from all orders combined +============== + +[TIP] +==== +The <> is more efficient than running individual min, max, avg, and sum aggregations. +==== + +[discrete] +[[aggregations-tutorial-sales-patterns]] +=== Analyze sales patterns + +Let's group orders in different ways to understand sales patterns. + +[discrete] +[[aggregations-tutorial-category-breakdown]] +==== Break down sales by category + +Group orders by category to see which product categories are most popular, using the <> aggregation. + +[source,console] +---- +GET kibana_sample_data_ecommerce/_search +{ + "size": 0, + "aggs": { + "sales_by_category": { <1> + "terms": { <2> + "field": "category.keyword", <3> + "size": 5, <4> + "order": { "_count": "desc" } <5> + } + } + } +} +---- +// TEST[skip:Using Kibana sample data] +<1> Name reflecting the business purpose of this breakdown +<2> `terms` aggregation groups documents by field values +<3> Use <> field for exact matching on text fields +<4> Limit to top 5 categories +<5> Order by number of orders (descending) + +.Example response +[%collapsible] +============== +[source,console-result] +---- +{ + "took": 4, + "timed_out": false, + "_shards": { + "total": 5, + "successful": 5, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 4675, + "relation": "eq" + }, + "max_score": null, + "hits": [] + }, + "aggregations": { + "sales_by_category": { + "doc_count_error_upper_bound": 0, <1> + "sum_other_doc_count": 572, <2> + "buckets": [ <3> + { + "key": "Men's Clothing", <4> + "doc_count": 2024 <5> + }, + { + "key": "Women's Clothing", + "doc_count": 1903 + }, + { + "key": "Women's Shoes", + "doc_count": 1136 + }, + { + "key": "Men's Shoes", + "doc_count": 944 + }, + { + "key": "Women's Accessories", + "doc_count": 830 + } + ] + } + } +} +---- +// TEST[skip:Using Kibana sample data] +<1> Due to Elasticsearch's distributed architecture, when <> run across multiple shards, the doc counts may have a small margin of error. This value indicates the maximum possible error in the counts. +<2> Count of documents in categories beyond the requested size. +<3> Array of category buckets, ordered by count. +<4> Category name. +<5> Number of orders in this category. +============== + +[discrete] +[[aggregations-tutorial-daily-sales]] +==== Track daily sales patterns + +Group orders by day to track daily sales patterns using the <> aggregation. + +[source,console] +---- +GET kibana_sample_data_ecommerce/_search +{ + "size": 0, + "aggs": { + "daily_orders": { <1> + "date_histogram": { <2> + "field": "order_date", + "calendar_interval": "day", <3> + "format": "yyyy-MM-dd", <4> + "min_doc_count": 0 <5> + } + } + } +} +---- +// TEST[skip:Using Kibana sample data] +<1> Descriptive name for the time-series aggregation results. +<2> The `date_histogram` aggregration groups documents into time-based buckets, similar to terms aggregation but for dates. +<3> Uses <> to handle months with different lengths. `"day"` ensures consistent daily grouping regardless of timezone. +<4> Formats dates in response using <> (e.g. "yyyy-MM-dd"). Refer to <> for additional options. +<5> When `min_doc_count` is 0, returns buckets for days with no orders, useful for continuous time series visualization. + +.Example response +[%collapsible] +============== +[source,console-result] +---- +{ + "took": 2, + "timed_out": false, + "_shards": { + "total": 5, + "successful": 5, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 4675, + "relation": "eq" + }, + "max_score": null, + "hits": [] + }, + "aggregations": { + "daily_orders": { <1> + "buckets": [ <2> + { + "key_as_string": "2024-11-28", <3> + "key": 1732752000000, <4> + "doc_count": 146 <5> + }, + { + "key_as_string": "2024-11-29", + "key": 1732838400000, + "doc_count": 153 + }, + { + "key_as_string": "2024-11-30", + "key": 1732924800000, + "doc_count": 143 + }, + { + "key_as_string": "2024-12-01", + "key": 1733011200000, + "doc_count": 140 + }, + { + "key_as_string": "2024-12-02", + "key": 1733097600000, + "doc_count": 139 + }, + { + "key_as_string": "2024-12-03", + "key": 1733184000000, + "doc_count": 157 + }, + { + "key_as_string": "2024-12-04", + "key": 1733270400000, + "doc_count": 145 + }, + { + "key_as_string": "2024-12-05", + "key": 1733356800000, + "doc_count": 152 + }, + { + "key_as_string": "2024-12-06", + "key": 1733443200000, + "doc_count": 163 + }, + { + "key_as_string": "2024-12-07", + "key": 1733529600000, + "doc_count": 141 + }, + { + "key_as_string": "2024-12-08", + "key": 1733616000000, + "doc_count": 151 + }, + { + "key_as_string": "2024-12-09", + "key": 1733702400000, + "doc_count": 143 + }, + { + "key_as_string": "2024-12-10", + "key": 1733788800000, + "doc_count": 143 + }, + { + "key_as_string": "2024-12-11", + "key": 1733875200000, + "doc_count": 142 + }, + { + "key_as_string": "2024-12-12", + "key": 1733961600000, + "doc_count": 161 + }, + { + "key_as_string": "2024-12-13", + "key": 1734048000000, + "doc_count": 144 + }, + { + "key_as_string": "2024-12-14", + "key": 1734134400000, + "doc_count": 157 + }, + { + "key_as_string": "2024-12-15", + "key": 1734220800000, + "doc_count": 158 + }, + { + "key_as_string": "2024-12-16", + "key": 1734307200000, + "doc_count": 144 + }, + { + "key_as_string": "2024-12-17", + "key": 1734393600000, + "doc_count": 151 + }, + { + "key_as_string": "2024-12-18", + "key": 1734480000000, + "doc_count": 145 + }, + { + "key_as_string": "2024-12-19", + "key": 1734566400000, + "doc_count": 157 + }, + { + "key_as_string": "2024-12-20", + "key": 1734652800000, + "doc_count": 158 + }, + { + "key_as_string": "2024-12-21", + "key": 1734739200000, + "doc_count": 153 + }, + { + "key_as_string": "2024-12-22", + "key": 1734825600000, + "doc_count": 165 + }, + { + "key_as_string": "2024-12-23", + "key": 1734912000000, + "doc_count": 153 + }, + { + "key_as_string": "2024-12-24", + "key": 1734998400000, + "doc_count": 158 + }, + { + "key_as_string": "2024-12-25", + "key": 1735084800000, + "doc_count": 160 + }, + { + "key_as_string": "2024-12-26", + "key": 1735171200000, + "doc_count": 159 + }, + { + "key_as_string": "2024-12-27", + "key": 1735257600000, + "doc_count": 152 + }, + { + "key_as_string": "2024-12-28", + "key": 1735344000000, + "doc_count": 142 + } + ] + } + } +} +---- +// TEST[skip:Using Kibana sample data] +<1> Results of our named aggregation "daily_orders" +<2> Time-based buckets from date_histogram aggregation +<3> `key_as_string` is the human-readable date for this bucket +<4> `key` is the same date represented as the Unix timestamp for this bucket +<5> `doc_count` counts the number of documents that fall into this time bucket +============== + +[discrete] +[[aggregations-tutorial-combined-analysis]] +=== Combine metrics with groupings + +Now let's calculate <> within each group to get deeper insights. + +[discrete] +[[aggregations-tutorial-category-metrics]] +==== Compare category performance + +Calculate metrics within each category to compare performance across categories. + +[source,console] +---- +GET kibana_sample_data_ecommerce/_search +{ + "size": 0, + "aggs": { + "categories": { + "terms": { + "field": "category.keyword", + "size": 5, + "order": { "total_revenue": "desc" } <1> + }, + "aggs": { <2> + "total_revenue": { <3> + "sum": { + "field": "taxful_total_price" + } + }, + "avg_order_value": { <4> + "avg": { + "field": "taxful_total_price" + } + }, + "total_items": { <5> + "sum": { + "field": "total_quantity" + } + } + } + } + } +} +---- +// TEST[skip:Using Kibana sample data] +<1> Order categories by their total revenue instead of count +<2> Define metrics to calculate within each category +<3> Total revenue for the category +<4> Average order value in the category +<5> Total number of items sold + +.Example response +[%collapsible] +============== +[source,console-result] +---- +{ + "aggregations": { + "categories": { + "buckets": [ + { + "key": "Men's Clothing", <1> + "doc_count": 2179, <2> + "total_revenue": { <3> + "value": 156729.453125 + }, + "avg_order_value": { <4> + "value": 71.92726898715927 + }, + "total_items": { <5> + "value": 8716 + } + }, + { + "key": "Women's Clothing", + "doc_count": 2262, + ... + } + ] + } + } +} +---- +// TEST[skip:Using Kibana sample data] +<1> Category name +<2> Number of orders +<3> Total revenue for this category +<4> Average order value for this category +<5> Total quantity of items sold +============== + +[discrete] +[[aggregations-tutorial-daily-metrics]] +==== Analyze daily sales performance + +Let's combine metrics to track daily trends: daily revenue, unique customers, and average basket size. + +[source,console] +---- +GET kibana_sample_data_ecommerce/_search +{ + "size": 0, + "aggs": { + "daily_sales": { + "date_histogram": { + "field": "order_date", + "calendar_interval": "day", + "format": "yyyy-MM-dd" + }, + "aggs": { + "revenue": { <1> + "sum": { + "field": "taxful_total_price" + } + }, + "unique_customers": { <2> + "cardinality": { + "field": "customer_id" + } + }, + "avg_basket_size": { <3> + "avg": { + "field": "total_quantity" + } + } + } + } + } +} +---- +// TEST[skip:Using Kibana sample data] +<1> Daily revenue +<2> Uses the <> aggregation to count unique customers per day +<3> Average number of items per order + +.Example response +[%collapsible] +============== +[source,console-result] +---- +{ + "took": 119, + "timed_out": false, + "_shards": { + "total": 5, + "successful": 5, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 4675, + "relation": "eq" + }, + "max_score": null, + "hits": [] + }, + "aggregations": { + "daily_sales": { + "buckets": [ + { + "key_as_string": "2024-11-14", + "key": 1731542400000, + "doc_count": 146, + "unique_customers": { <1> + "value": 42 + }, + "revenue": { <2> + "value": 10578.53125 + }, + "avg_basket_size": { <3> + "value": 2.1780821917808217 + } + }, + { + "key_as_string": "2024-11-15", + "key": 1731628800000, + "doc_count": 153, + "unique_customers": { + "value": 44 + }, + "revenue": { + "value": 10448 + }, + "avg_basket_size": { + "value": 2.183006535947712 + } + }, + { + "key_as_string": "2024-11-16", + "key": 1731715200000, + "doc_count": 143, + "unique_customers": { + "value": 45 + }, + "revenue": { + "value": 10283.484375 + }, + "avg_basket_size": { + "value": 2.111888111888112 + } + }, + { + "key_as_string": "2024-11-17", + "key": 1731801600000, + "doc_count": 140, + "unique_customers": { + "value": 42 + }, + "revenue": { + "value": 10145.5234375 + }, + "avg_basket_size": { + "value": 2.142857142857143 + } + }, + { + "key_as_string": "2024-11-18", + "key": 1731888000000, + "doc_count": 139, + "unique_customers": { + "value": 42 + }, + "revenue": { + "value": 12012.609375 + }, + "avg_basket_size": { + "value": 2.158273381294964 + } + }, + { + "key_as_string": "2024-11-19", + "key": 1731974400000, + "doc_count": 157, + "unique_customers": { + "value": 43 + }, + "revenue": { + "value": 11009.45703125 + }, + "avg_basket_size": { + "value": 2.0955414012738856 + } + }, + { + "key_as_string": "2024-11-20", + "key": 1732060800000, + "doc_count": 145, + "unique_customers": { + "value": 44 + }, + "revenue": { + "value": 10720.59375 + }, + "avg_basket_size": { + "value": 2.179310344827586 + } + }, + { + "key_as_string": "2024-11-21", + "key": 1732147200000, + "doc_count": 152, + "unique_customers": { + "value": 43 + }, + "revenue": { + "value": 11185.3671875 + }, + "avg_basket_size": { + "value": 2.1710526315789473 + } + }, + { + "key_as_string": "2024-11-22", + "key": 1732233600000, + "doc_count": 163, + "unique_customers": { + "value": 44 + }, + "revenue": { + "value": 13560.140625 + }, + "avg_basket_size": { + "value": 2.2576687116564416 + } + }, + { + "key_as_string": "2024-11-23", + "key": 1732320000000, + "doc_count": 141, + "unique_customers": { + "value": 45 + }, + "revenue": { + "value": 9884.78125 + }, + "avg_basket_size": { + "value": 2.099290780141844 + } + }, + { + "key_as_string": "2024-11-24", + "key": 1732406400000, + "doc_count": 151, + "unique_customers": { + "value": 44 + }, + "revenue": { + "value": 11075.65625 + }, + "avg_basket_size": { + "value": 2.0927152317880795 + } + }, + { + "key_as_string": "2024-11-25", + "key": 1732492800000, + "doc_count": 143, + "unique_customers": { + "value": 41 + }, + "revenue": { + "value": 10323.8515625 + }, + "avg_basket_size": { + "value": 2.167832167832168 + } + }, + { + "key_as_string": "2024-11-26", + "key": 1732579200000, + "doc_count": 143, + "unique_customers": { + "value": 44 + }, + "revenue": { + "value": 10369.546875 + }, + "avg_basket_size": { + "value": 2.167832167832168 + } + }, + { + "key_as_string": "2024-11-27", + "key": 1732665600000, + "doc_count": 142, + "unique_customers": { + "value": 46 + }, + "revenue": { + "value": 11711.890625 + }, + "avg_basket_size": { + "value": 2.1971830985915495 + } + }, + { + "key_as_string": "2024-11-28", + "key": 1732752000000, + "doc_count": 161, + "unique_customers": { + "value": 43 + }, + "revenue": { + "value": 12612.6640625 + }, + "avg_basket_size": { + "value": 2.1180124223602483 + } + }, + { + "key_as_string": "2024-11-29", + "key": 1732838400000, + "doc_count": 144, + "unique_customers": { + "value": 42 + }, + "revenue": { + "value": 10176.87890625 + }, + "avg_basket_size": { + "value": 2.0347222222222223 + } + }, + { + "key_as_string": "2024-11-30", + "key": 1732924800000, + "doc_count": 157, + "unique_customers": { + "value": 43 + }, + "revenue": { + "value": 11480.33203125 + }, + "avg_basket_size": { + "value": 2.159235668789809 + } + }, + { + "key_as_string": "2024-12-01", + "key": 1733011200000, + "doc_count": 158, + "unique_customers": { + "value": 42 + }, + "revenue": { + "value": 11533.265625 + }, + "avg_basket_size": { + "value": 2.0822784810126582 + } + }, + { + "key_as_string": "2024-12-02", + "key": 1733097600000, + "doc_count": 144, + "unique_customers": { + "value": 43 + }, + "revenue": { + "value": 10499.8125 + }, + "avg_basket_size": { + "value": 2.201388888888889 + } + }, + { + "key_as_string": "2024-12-03", + "key": 1733184000000, + "doc_count": 151, + "unique_customers": { + "value": 40 + }, + "revenue": { + "value": 12111.6875 + }, + "avg_basket_size": { + "value": 2.172185430463576 + } + }, + { + "key_as_string": "2024-12-04", + "key": 1733270400000, + "doc_count": 145, + "unique_customers": { + "value": 40 + }, + "revenue": { + "value": 10530.765625 + }, + "avg_basket_size": { + "value": 2.0965517241379312 + } + }, + { + "key_as_string": "2024-12-05", + "key": 1733356800000, + "doc_count": 157, + "unique_customers": { + "value": 43 + }, + "revenue": { + "value": 11872.5625 + }, + "avg_basket_size": { + "value": 2.1464968152866244 + } + }, + { + "key_as_string": "2024-12-06", + "key": 1733443200000, + "doc_count": 158, + "unique_customers": { + "value": 42 + }, + "revenue": { + "value": 12109.453125 + }, + "avg_basket_size": { + "value": 2.151898734177215 + } + }, + { + "key_as_string": "2024-12-07", + "key": 1733529600000, + "doc_count": 153, + "unique_customers": { + "value": 42 + }, + "revenue": { + "value": 11057.40625 + }, + "avg_basket_size": { + "value": 2.111111111111111 + } + }, + { + "key_as_string": "2024-12-08", + "key": 1733616000000, + "doc_count": 165, + "unique_customers": { + "value": 42 + }, + "revenue": { + "value": 13095.609375 + }, + "avg_basket_size": { + "value": 2.1818181818181817 + } + }, + { + "key_as_string": "2024-12-09", + "key": 1733702400000, + "doc_count": 153, + "unique_customers": { + "value": 41 + }, + "revenue": { + "value": 12574.015625 + }, + "avg_basket_size": { + "value": 2.2287581699346406 + } + }, + { + "key_as_string": "2024-12-10", + "key": 1733788800000, + "doc_count": 158, + "unique_customers": { + "value": 42 + }, + "revenue": { + "value": 11188.1875 + }, + "avg_basket_size": { + "value": 2.151898734177215 + } + }, + { + "key_as_string": "2024-12-11", + "key": 1733875200000, + "doc_count": 160, + "unique_customers": { + "value": 42 + }, + "revenue": { + "value": 12117.65625 + }, + "avg_basket_size": { + "value": 2.20625 + } + }, + { + "key_as_string": "2024-12-12", + "key": 1733961600000, + "doc_count": 159, + "unique_customers": { + "value": 45 + }, + "revenue": { + "value": 11558.25 + }, + "avg_basket_size": { + "value": 2.1823899371069184 + } + }, + { + "key_as_string": "2024-12-13", + "key": 1734048000000, + "doc_count": 152, + "unique_customers": { + "value": 45 + }, + "revenue": { + "value": 11921.1171875 + }, + "avg_basket_size": { + "value": 2.289473684210526 + } + }, + { + "key_as_string": "2024-12-14", + "key": 1734134400000, + "doc_count": 142, + "unique_customers": { + "value": 45 + }, + "revenue": { + "value": 11135.03125 + }, + "avg_basket_size": { + "value": 2.183098591549296 + } + } + ] + } + } +} +---- +// TEST[skip:Using Kibana sample data] +============== + +[discrete] +[[aggregations-tutorial-trends]] +=== Track trends and patterns + +You can use <> on the results of other aggregations. +Let's analyze how metrics change over time. + +[discrete] +[[aggregations-tutorial-moving-average]] +==== Smooth out daily fluctuations + +Moving averages help identify trends by reducing day-to-day noise in the data. +Let's observe sales trends more clearly by smoothing daily revenue variations, using the <> aggregation. + +[source,console] +---- +GET kibana_sample_data_ecommerce/_search +{ + "size": 0, + "aggs": { + "daily_sales": { + "date_histogram": { + "field": "order_date", + "calendar_interval": "day" + }, + "aggs": { + "daily_revenue": { <1> + "sum": { + "field": "taxful_total_price" + } + }, + "smoothed_revenue": { <2> + "moving_fn": { <3> + "buckets_path": "daily_revenue", <4> + "window": 3, <5> + "script": "MovingFunctions.unweightedAvg(values)" <6> + } + } + } + } + } +} +---- +// TEST[skip:Using Kibana sample data] +<1> Calculate daily revenue first. +<2> Create a smoothed version of the daily revenue. +<3> Use `moving_fn` for moving window calculations. +<4> Reference the revenue from our date histogram. +<5> Use a 3-day window — use different window sizes to see trends at different time scales. +<6> Use the built-in unweighted average function in the `moving_fn` aggregation. + +.Example response +[%collapsible] +============== +[source,console-result] +---- +{ + "took": 13, + "timed_out": false, + "_shards": { + "total": 5, + "successful": 5, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 4675, + "relation": "eq" + }, + "max_score": null, + "hits": [] + }, + "aggregations": { + "daily_sales": { + "buckets": [ + { + "key_as_string": "2024-11-14T00:00:00.000Z", <1> + "key": 1731542400000, + "doc_count": 146, <2> + "daily_revenue": { <3> + "value": 10578.53125 + }, + "smoothed_revenue": { <4> + "value": null + } + }, + { + "key_as_string": "2024-11-15T00:00:00.000Z", + "key": 1731628800000, + "doc_count": 153, + "daily_revenue": { + "value": 10448 + }, + "smoothed_revenue": { <5> + "value": 10578.53125 + } + }, + { + "key_as_string": "2024-11-16T00:00:00.000Z", + "key": 1731715200000, + "doc_count": 143, + "daily_revenue": { + "value": 10283.484375 + }, + "smoothed_revenue": { + "value": 10513.265625 + } + }, + { + "key_as_string": "2024-11-17T00:00:00.000Z", + "key": 1731801600000, + "doc_count": 140, + "daily_revenue": { + "value": 10145.5234375 + }, + "smoothed_revenue": { + "value": 10436.671875 + } + }, + { + "key_as_string": "2024-11-18T00:00:00.000Z", + "key": 1731888000000, + "doc_count": 139, + "daily_revenue": { + "value": 12012.609375 + }, + "smoothed_revenue": { + "value": 10292.3359375 + } + }, + { + "key_as_string": "2024-11-19T00:00:00.000Z", + "key": 1731974400000, + "doc_count": 157, + "daily_revenue": { + "value": 11009.45703125 + }, + "smoothed_revenue": { + "value": 10813.872395833334 + } + }, + { + "key_as_string": "2024-11-20T00:00:00.000Z", + "key": 1732060800000, + "doc_count": 145, + "daily_revenue": { + "value": 10720.59375 + }, + "smoothed_revenue": { + "value": 11055.86328125 + } + }, + { + "key_as_string": "2024-11-21T00:00:00.000Z", + "key": 1732147200000, + "doc_count": 152, + "daily_revenue": { + "value": 11185.3671875 + }, + "smoothed_revenue": { + "value": 11247.553385416666 + } + }, + { + "key_as_string": "2024-11-22T00:00:00.000Z", + "key": 1732233600000, + "doc_count": 163, + "daily_revenue": { + "value": 13560.140625 + }, + "smoothed_revenue": { + "value": 10971.805989583334 + } + }, + { + "key_as_string": "2024-11-23T00:00:00.000Z", + "key": 1732320000000, + "doc_count": 141, + "daily_revenue": { + "value": 9884.78125 + }, + "smoothed_revenue": { + "value": 11822.033854166666 + } + }, + { + "key_as_string": "2024-11-24T00:00:00.000Z", + "key": 1732406400000, + "doc_count": 151, + "daily_revenue": { + "value": 11075.65625 + }, + "smoothed_revenue": { + "value": 11543.4296875 + } + }, + { + "key_as_string": "2024-11-25T00:00:00.000Z", + "key": 1732492800000, + "doc_count": 143, + "daily_revenue": { + "value": 10323.8515625 + }, + "smoothed_revenue": { + "value": 11506.859375 + } + }, + { + "key_as_string": "2024-11-26T00:00:00.000Z", + "key": 1732579200000, + "doc_count": 143, + "daily_revenue": { + "value": 10369.546875 + }, + "smoothed_revenue": { + "value": 10428.096354166666 + } + }, + { + "key_as_string": "2024-11-27T00:00:00.000Z", + "key": 1732665600000, + "doc_count": 142, + "daily_revenue": { + "value": 11711.890625 + }, + "smoothed_revenue": { + "value": 10589.684895833334 + } + }, + { + "key_as_string": "2024-11-28T00:00:00.000Z", + "key": 1732752000000, + "doc_count": 161, + "daily_revenue": { + "value": 12612.6640625 + }, + "smoothed_revenue": { + "value": 10801.763020833334 + } + }, + { + "key_as_string": "2024-11-29T00:00:00.000Z", + "key": 1732838400000, + "doc_count": 144, + "daily_revenue": { + "value": 10176.87890625 + }, + "smoothed_revenue": { + "value": 11564.700520833334 + } + }, + { + "key_as_string": "2024-11-30T00:00:00.000Z", + "key": 1732924800000, + "doc_count": 157, + "daily_revenue": { + "value": 11480.33203125 + }, + "smoothed_revenue": { + "value": 11500.477864583334 + } + }, + { + "key_as_string": "2024-12-01T00:00:00.000Z", + "key": 1733011200000, + "doc_count": 158, + "daily_revenue": { + "value": 11533.265625 + }, + "smoothed_revenue": { + "value": 11423.291666666666 + } + }, + { + "key_as_string": "2024-12-02T00:00:00.000Z", + "key": 1733097600000, + "doc_count": 144, + "daily_revenue": { + "value": 10499.8125 + }, + "smoothed_revenue": { + "value": 11063.4921875 + } + }, + { + "key_as_string": "2024-12-03T00:00:00.000Z", + "key": 1733184000000, + "doc_count": 151, + "daily_revenue": { + "value": 12111.6875 + }, + "smoothed_revenue": { + "value": 11171.13671875 + } + }, + { + "key_as_string": "2024-12-04T00:00:00.000Z", + "key": 1733270400000, + "doc_count": 145, + "daily_revenue": { + "value": 10530.765625 + }, + "smoothed_revenue": { + "value": 11381.588541666666 + } + }, + { + "key_as_string": "2024-12-05T00:00:00.000Z", + "key": 1733356800000, + "doc_count": 157, + "daily_revenue": { + "value": 11872.5625 + }, + "smoothed_revenue": { + "value": 11047.421875 + } + }, + { + "key_as_string": "2024-12-06T00:00:00.000Z", + "key": 1733443200000, + "doc_count": 158, + "daily_revenue": { + "value": 12109.453125 + }, + "smoothed_revenue": { + "value": 11505.005208333334 + } + }, + { + "key_as_string": "2024-12-07T00:00:00.000Z", + "key": 1733529600000, + "doc_count": 153, + "daily_revenue": { + "value": 11057.40625 + }, + "smoothed_revenue": { + "value": 11504.260416666666 + } + }, + { + "key_as_string": "2024-12-08T00:00:00.000Z", + "key": 1733616000000, + "doc_count": 165, + "daily_revenue": { + "value": 13095.609375 + }, + "smoothed_revenue": { + "value": 11679.807291666666 + } + }, + { + "key_as_string": "2024-12-09T00:00:00.000Z", + "key": 1733702400000, + "doc_count": 153, + "daily_revenue": { + "value": 12574.015625 + }, + "smoothed_revenue": { + "value": 12087.489583333334 + } + }, + { + "key_as_string": "2024-12-10T00:00:00.000Z", + "key": 1733788800000, + "doc_count": 158, + "daily_revenue": { + "value": 11188.1875 + }, + "smoothed_revenue": { + "value": 12242.34375 + } + }, + { + "key_as_string": "2024-12-11T00:00:00.000Z", + "key": 1733875200000, + "doc_count": 160, + "daily_revenue": { + "value": 12117.65625 + }, + "smoothed_revenue": { + "value": 12285.9375 + } + }, + { + "key_as_string": "2024-12-12T00:00:00.000Z", + "key": 1733961600000, + "doc_count": 159, + "daily_revenue": { + "value": 11558.25 + }, + "smoothed_revenue": { + "value": 11959.953125 + } + }, + { + "key_as_string": "2024-12-13T00:00:00.000Z", + "key": 1734048000000, + "doc_count": 152, + "daily_revenue": { + "value": 11921.1171875 + }, + "smoothed_revenue": { + "value": 11621.364583333334 + } + }, + { + "key_as_string": "2024-12-14T00:00:00.000Z", + "key": 1734134400000, + "doc_count": 142, + "daily_revenue": { + "value": 11135.03125 + }, + "smoothed_revenue": { + "value": 11865.674479166666 + } + } + ] + } + } +} +---- +// TEST[skip:Using Kibana sample data] +<1> Date of the bucket is in default ISO format because we didn't specify a format +<2> Number of orders for this day +<3> Raw daily revenue before smoothing +<4> First day has no smoothed value as it needs previous days for the calculation +<5> Moving average starts from second day, using a 3-day window +============== + +[TIP] +==== +Notice how the smoothed values lag behind the actual values - this is because they need previous days' data to calculate. The first day will always be null when using moving averages. +==== + +[discrete] +[[aggregations-tutorial-cumulative]] +==== Track running totals + +Track running totals over time using the <> aggregation. + +[source,console] +---- +GET kibana_sample_data_ecommerce/_search +{ + "size": 0, + "aggs": { + "daily_sales": { + "date_histogram": { + "field": "order_date", + "calendar_interval": "day" + }, + "aggs": { + "revenue": { + "sum": { + "field": "taxful_total_price" + } + }, + "cumulative_revenue": { <1> + "cumulative_sum": { <2> + "buckets_path": "revenue" <3> + } + } + } + } + } +} +---- +// TEST[skip:Using Kibana sample data] +<1> Name for our running total +<2> `cumulative_sum` adds up values across buckets +<3> Reference the revenue we want to accumulate + +.Example response +[%collapsible] +============== +[source,console-result] +---- +{ + "took": 4, + "timed_out": false, + "_shards": { + "total": 5, + "successful": 5, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 4675, + "relation": "eq" + }, + "max_score": null, + "hits": [] + }, + "aggregations": { + "daily_sales": { <1> + "buckets": [ <2> + { + "key_as_string": "2024-11-14T00:00:00.000Z", <3> + "key": 1731542400000, + "doc_count": 146, + "revenue": { <4> + "value": 10578.53125 + }, + "cumulative_revenue": { <5> + "value": 10578.53125 + } + }, + { + "key_as_string": "2024-11-15T00:00:00.000Z", + "key": 1731628800000, + "doc_count": 153, + "revenue": { + "value": 10448 + }, + "cumulative_revenue": { + "value": 21026.53125 + } + }, + { + "key_as_string": "2024-11-16T00:00:00.000Z", + "key": 1731715200000, + "doc_count": 143, + "revenue": { + "value": 10283.484375 + }, + "cumulative_revenue": { + "value": 31310.015625 + } + }, + { + "key_as_string": "2024-11-17T00:00:00.000Z", + "key": 1731801600000, + "doc_count": 140, + "revenue": { + "value": 10145.5234375 + }, + "cumulative_revenue": { + "value": 41455.5390625 + } + }, + { + "key_as_string": "2024-11-18T00:00:00.000Z", + "key": 1731888000000, + "doc_count": 139, + "revenue": { + "value": 12012.609375 + }, + "cumulative_revenue": { + "value": 53468.1484375 + } + }, + { + "key_as_string": "2024-11-19T00:00:00.000Z", + "key": 1731974400000, + "doc_count": 157, + "revenue": { + "value": 11009.45703125 + }, + "cumulative_revenue": { + "value": 64477.60546875 + } + }, + { + "key_as_string": "2024-11-20T00:00:00.000Z", + "key": 1732060800000, + "doc_count": 145, + "revenue": { + "value": 10720.59375 + }, + "cumulative_revenue": { + "value": 75198.19921875 + } + }, + { + "key_as_string": "2024-11-21T00:00:00.000Z", + "key": 1732147200000, + "doc_count": 152, + "revenue": { + "value": 11185.3671875 + }, + "cumulative_revenue": { + "value": 86383.56640625 + } + }, + { + "key_as_string": "2024-11-22T00:00:00.000Z", + "key": 1732233600000, + "doc_count": 163, + "revenue": { + "value": 13560.140625 + }, + "cumulative_revenue": { + "value": 99943.70703125 + } + }, + { + "key_as_string": "2024-11-23T00:00:00.000Z", + "key": 1732320000000, + "doc_count": 141, + "revenue": { + "value": 9884.78125 + }, + "cumulative_revenue": { + "value": 109828.48828125 + } + }, + { + "key_as_string": "2024-11-24T00:00:00.000Z", + "key": 1732406400000, + "doc_count": 151, + "revenue": { + "value": 11075.65625 + }, + "cumulative_revenue": { + "value": 120904.14453125 + } + }, + { + "key_as_string": "2024-11-25T00:00:00.000Z", + "key": 1732492800000, + "doc_count": 143, + "revenue": { + "value": 10323.8515625 + }, + "cumulative_revenue": { + "value": 131227.99609375 + } + }, + { + "key_as_string": "2024-11-26T00:00:00.000Z", + "key": 1732579200000, + "doc_count": 143, + "revenue": { + "value": 10369.546875 + }, + "cumulative_revenue": { + "value": 141597.54296875 + } + }, + { + "key_as_string": "2024-11-27T00:00:00.000Z", + "key": 1732665600000, + "doc_count": 142, + "revenue": { + "value": 11711.890625 + }, + "cumulative_revenue": { + "value": 153309.43359375 + } + }, + { + "key_as_string": "2024-11-28T00:00:00.000Z", + "key": 1732752000000, + "doc_count": 161, + "revenue": { + "value": 12612.6640625 + }, + "cumulative_revenue": { + "value": 165922.09765625 + } + }, + { + "key_as_string": "2024-11-29T00:00:00.000Z", + "key": 1732838400000, + "doc_count": 144, + "revenue": { + "value": 10176.87890625 + }, + "cumulative_revenue": { + "value": 176098.9765625 + } + }, + { + "key_as_string": "2024-11-30T00:00:00.000Z", + "key": 1732924800000, + "doc_count": 157, + "revenue": { + "value": 11480.33203125 + }, + "cumulative_revenue": { + "value": 187579.30859375 + } + }, + { + "key_as_string": "2024-12-01T00:00:00.000Z", + "key": 1733011200000, + "doc_count": 158, + "revenue": { + "value": 11533.265625 + }, + "cumulative_revenue": { + "value": 199112.57421875 + } + }, + { + "key_as_string": "2024-12-02T00:00:00.000Z", + "key": 1733097600000, + "doc_count": 144, + "revenue": { + "value": 10499.8125 + }, + "cumulative_revenue": { + "value": 209612.38671875 + } + }, + { + "key_as_string": "2024-12-03T00:00:00.000Z", + "key": 1733184000000, + "doc_count": 151, + "revenue": { + "value": 12111.6875 + }, + "cumulative_revenue": { + "value": 221724.07421875 + } + }, + { + "key_as_string": "2024-12-04T00:00:00.000Z", + "key": 1733270400000, + "doc_count": 145, + "revenue": { + "value": 10530.765625 + }, + "cumulative_revenue": { + "value": 232254.83984375 + } + }, + { + "key_as_string": "2024-12-05T00:00:00.000Z", + "key": 1733356800000, + "doc_count": 157, + "revenue": { + "value": 11872.5625 + }, + "cumulative_revenue": { + "value": 244127.40234375 + } + }, + { + "key_as_string": "2024-12-06T00:00:00.000Z", + "key": 1733443200000, + "doc_count": 158, + "revenue": { + "value": 12109.453125 + }, + "cumulative_revenue": { + "value": 256236.85546875 + } + }, + { + "key_as_string": "2024-12-07T00:00:00.000Z", + "key": 1733529600000, + "doc_count": 153, + "revenue": { + "value": 11057.40625 + }, + "cumulative_revenue": { + "value": 267294.26171875 + } + }, + { + "key_as_string": "2024-12-08T00:00:00.000Z", + "key": 1733616000000, + "doc_count": 165, + "revenue": { + "value": 13095.609375 + }, + "cumulative_revenue": { + "value": 280389.87109375 + } + }, + { + "key_as_string": "2024-12-09T00:00:00.000Z", + "key": 1733702400000, + "doc_count": 153, + "revenue": { + "value": 12574.015625 + }, + "cumulative_revenue": { + "value": 292963.88671875 + } + }, + { + "key_as_string": "2024-12-10T00:00:00.000Z", + "key": 1733788800000, + "doc_count": 158, + "revenue": { + "value": 11188.1875 + }, + "cumulative_revenue": { + "value": 304152.07421875 + } + }, + { + "key_as_string": "2024-12-11T00:00:00.000Z", + "key": 1733875200000, + "doc_count": 160, + "revenue": { + "value": 12117.65625 + }, + "cumulative_revenue": { + "value": 316269.73046875 + } + }, + { + "key_as_string": "2024-12-12T00:00:00.000Z", + "key": 1733961600000, + "doc_count": 159, + "revenue": { + "value": 11558.25 + }, + "cumulative_revenue": { + "value": 327827.98046875 + } + }, + { + "key_as_string": "2024-12-13T00:00:00.000Z", + "key": 1734048000000, + "doc_count": 152, + "revenue": { + "value": 11921.1171875 + }, + "cumulative_revenue": { + "value": 339749.09765625 + } + }, + { + "key_as_string": "2024-12-14T00:00:00.000Z", + "key": 1734134400000, + "doc_count": 142, + "revenue": { + "value": 11135.03125 + }, + "cumulative_revenue": { + "value": 350884.12890625 + } + } + ] + } + } +} +---- +// TEST[skip:Using Kibana sample data] +<1> `daily_sales`: Results from our daily sales date histogram +<2> `buckets`: Array of time-based buckets +<3> `key_as_string`: Date for this bucket (in ISO format since no format specified) +<4> `revenue`: Daily revenue for this date +<5> `cumulative_revenue`: Running total of revenue up to this date +============== + +[discrete] +[[aggregations-tutorial-next-steps]] +=== Next steps + +Refer to the <> for more details on all available aggregation types. \ No newline at end of file diff --git a/docs/reference/quickstart/full-text-filtering-tutorial.asciidoc b/docs/reference/quickstart/full-text-filtering-tutorial.asciidoc new file mode 100644 index 0000000000000..b602ee5076434 --- /dev/null +++ b/docs/reference/quickstart/full-text-filtering-tutorial.asciidoc @@ -0,0 +1,640 @@ +[[full-text-filter-tutorial]] +== Basic full-text search and filtering in {es} +++++ +Basics: Full-text search and filtering +++++ + +This is a hands-on introduction to the basics of <> with {es}, also known as _lexical search_, using the <> and <>. +You'll also learn how to filter data, to narrow down search results based on exact criteria. + +In this scenario, we're implementing a search function for a cooking blog. +The blog contains recipes with various attributes including textual content, categorical data, and numerical ratings. + +The goal is to create search queries that enable users to: + +* Find recipes based on ingredients they want to use or avoid +* Discover dishes suitable for their dietary needs +* Find highly-rated recipes in specific categories +* Find recent recipes from their favorite authors + +To achieve these goals we'll use different Elasticsearch queries to perform full-text search, apply filters, and combine multiple search criteria. + +[discrete] +[[full-text-filter-tutorial-requirements]] +=== Requirements + +You'll need a running {es} cluster, together with {kib} to use the Dev Tools API Console. +Run the following command in your terminal to set up a <>: + +[source,sh] +---- +curl -fsSL https://elastic.co/start-local | sh +---- +// NOTCONSOLE + +[discrete] +[[full-text-filter-tutorial-create-index]] +=== Step 1: Create an index + +Create the `cooking_blog` index to get started: + +[source,console] +---- +PUT /cooking_blog +---- +// TESTSETUP + +Now define the mappings for the index: + +[source,console] +---- +PUT /cooking_blog/_mapping +{ + "properties": { + "title": { + "type": "text", + "analyzer": "standard", <1> + "fields": { <2> + "keyword": { + "type": "keyword", + "ignore_above": 256 <3> + } + } + }, + "description": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "author": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "date": { + "type": "date", + "format": "yyyy-MM-dd" + }, + "category": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "tags": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "rating": { + "type": "float" + } + } +} +---- +// TEST +<1> The `standard` analyzer is used by default for `text` fields if an `analyzer` isn't specified. It's included here for demonstration purposes. +<2> <> are used here to index `text` fields as both `text` and `keyword` <>. This enables both full-text search and exact matching/filtering on the same field. +Note that if you used <>, these multi-fields would be created automatically. +<3> The <> prevents indexing values longer than 256 characters in the `keyword` field. Again this is the default value, but it's included here for for demonstration purposes. +It helps to save disk space and avoid potential issues with Lucene's term byte-length limit. + +[TIP] +==== +Full-text search is powered by <>. +Text analysis normalizes and standardizes text data so it can be efficiently stored in an inverted index and searched in near real-time. +Analysis happens at both <>. +This tutorial won't cover analysis in detail, but it's important to understand how text is processed to create effective search queries. +==== + +[discrete] +[[full-text-filter-tutorial-index-data]] +=== Step 2: Add sample blog posts to your index + +Now you'll need to index some example blog posts using the <>. +Note that `text` fields are analyzed and multi-fields are generated at index time. + +[source,console] +---- +POST /cooking_blog/_bulk?refresh=wait_for +{"index":{"_id":"1"}} +{"title":"Perfect Pancakes: A Fluffy Breakfast Delight","description":"Learn the secrets to making the fluffiest pancakes, so amazing you won't believe your tastebuds. This recipe uses buttermilk and a special folding technique to create light, airy pancakes that are perfect for lazy Sunday mornings.","author":"Maria Rodriguez","date":"2023-05-01","category":"Breakfast","tags":["pancakes","breakfast","easy recipes"],"rating":4.8} +{"index":{"_id":"2"}} +{"title":"Spicy Thai Green Curry: A Vegetarian Adventure","description":"Dive into the flavors of Thailand with this vibrant green curry. Packed with vegetables and aromatic herbs, this dish is both healthy and satisfying. Don't worry about the heat - you can easily adjust the spice level to your liking.","author":"Liam Chen","date":"2023-05-05","category":"Main Course","tags":["thai","vegetarian","curry","spicy"],"rating":4.6} +{"index":{"_id":"3"}} +{"title":"Classic Beef Stroganoff: A Creamy Comfort Food","description":"Indulge in this rich and creamy beef stroganoff. Tender strips of beef in a savory mushroom sauce, served over a bed of egg noodles. It's the ultimate comfort food for chilly evenings.","author":"Emma Watson","date":"2023-05-10","category":"Main Course","tags":["beef","pasta","comfort food"],"rating":4.7} +{"index":{"_id":"4"}} +{"title":"Vegan Chocolate Avocado Mousse","description":"Discover the magic of avocado in this rich, vegan chocolate mousse. Creamy, indulgent, and secretly healthy, it's the perfect guilt-free dessert for chocolate lovers.","author":"Alex Green","date":"2023-05-15","category":"Dessert","tags":["vegan","chocolate","avocado","healthy dessert"],"rating":4.5} +{"index":{"_id":"5"}} +{"title":"Crispy Oven-Fried Chicken","description":"Get that perfect crunch without the deep fryer! This oven-fried chicken recipe delivers crispy, juicy results every time. A healthier take on the classic comfort food.","author":"Maria Rodriguez","date":"2023-05-20","category":"Main Course","tags":["chicken","oven-fried","healthy"],"rating":4.9} +---- +// TEST[continued] + +[discrete] +[[full-text-filter-tutorial-match-query]] +=== Step 3: Perform basic full-text searches + +Full-text search involves executing text-based queries across one or more document fields. +These queries calculate a relevance score for each matching document, based on how closely the document's content aligns with the search terms. +{es} offers various query types, each with its own method for matching text and <>. + +[discrete] +==== `match` query + +The <> query is the standard query for full-text, or "lexical", search. +The query text will be analyzed according to the analyzer configuration specified on each field (or at query time). + +First, search the `description` field for "fluffy pancakes": + +[source,console] +---- +GET /cooking_blog/_search +{ + "query": { + "match": { + "description": { + "query": "fluffy pancakes" <1> + } + } + } +} +---- +// TEST[continued] +<1> By default, the `match` query uses `OR` logic between the resulting tokens. This means it will match documents that contain either "fluffy" or "pancakes", or both, in the description field. + +At search time, {es} defaults to the analyzer defined in the field mapping. In this example, we're using the `standard` analyzer. Using a different analyzer at search time is an <>. + +.Example response +[%collapsible] +============== +[source,console-result] +---- +{ + "took": 0, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { <1> + "total": { + "value": 1, + "relation": "eq" + }, + "max_score": 1.8378843, <2> + "hits": [ + { + "_index": "cooking_blog", + "_id": "1", + "_score": 1.8378843, <3> + "_source": { + "title": "Perfect Pancakes: A Fluffy Breakfast Delight", <4> + "description": "Learn the secrets to making the fluffiest pancakes, so amazing you won't believe your tastebuds. This recipe uses buttermilk and a special folding technique to create light, airy pancakes that are perfect for lazy Sunday mornings.", <5> + "author": "Maria Rodriguez", + "date": "2023-05-01", + "category": "Breakfast", + "tags": [ + "pancakes", + "breakfast", + "easy recipes" + ], + "rating": 4.8 + } + } + ] + } +} +---- +// TESTRESPONSE[s/"took": 0/"took": "$body.took"/] +// TESTRESPONSE[s/"total": 1/"total": $body._shards.total/] +// TESTRESPONSE[s/"successful": 1/"successful": $body._shards.successful/] +// TESTRESPONSE[s/"value": 1/"value": $body.hits.total.value/] +// TESTRESPONSE[s/"max_score": 1.8378843/"max_score": $body.hits.max_score/] +// TESTRESPONSE[s/"_score": 1.8378843/"_score": $body.hits.hits.0._score/] +<1> The `hits` object contains the total number of matching documents and their relation to the total. Refer to <> for more details about the `hits` object. +<2> `max_score` is the highest relevance score among all matching documents. In this example, we only have one matching document. +<3> `_score` is the relevance score for a specific document, indicating how well it matches the query. Higher scores indicate better matches. In this example the `max_score` is the same as the `_score`, as there is only one matching document. +<4> The title contains both "Fluffy" and "Pancakes", matching our search terms exactly. +<5> The description includes "fluffiest" and "pancakes", further contributing to the document's relevance due to the analysis process. +============== + +[discrete] +==== Require all terms in a match query + +Specify the `and` operator to require both terms in the `description` field. +This stricter search returns _zero hits_ on our sample data, as no document contains both "fluffy" and "pancakes" in the description. + +[source,console] +---- +GET /cooking_blog/_search +{ + "query": { + "match": { + "description": { + "query": "fluffy pancakes", + "operator": "and" + } + } + } +} +---- +// TEST[continued] + +.Example response +[%collapsible] +============== +[source,console-result] +---- +{ + "took": 0, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 0, + "relation": "eq" + }, + "max_score": null, + "hits": [] + } +} +---- +// TESTRESPONSE[s/"took": 0/"took": "$body.took"/] +============== + +[discrete] +==== Specify a minimum number of terms to match + +Use the <> parameter to specify the minimum number of terms a document should have to be included in the search results. + +Search the title field to match at least 2 of the 3 terms: "fluffy", "pancakes", or "breakfast". +This is useful for improving relevance while allowing some flexibility. + +[source,console] +---- +GET /cooking_blog/_search +{ + "query": { + "match": { + "title": { + "query": "fluffy pancakes breakfast", + "minimum_should_match": 2 + } + } + } +} +---- +// TEST[continued] + +[discrete] +[[full-text-filter-tutorial-multi-match]] +=== Step 4: Search across multiple fields at once + +When users enter a search query, they often don't know (or care) whether their search terms appear in a specific field. +A <> query allows searching across multiple fields simultaneously. + +Let's start with a basic `multi_match` query: + +[source,console] +---- +GET /cooking_blog/_search +{ + "query": { + "multi_match": { + "query": "vegetarian curry", + "fields": ["title", "description", "tags"] + } + } +} +---- +// TEST[continued] + +This query searches for "vegetarian curry" across the title, description, and tags fields. Each field is treated with equal importance. + +However, in many cases, matches in certain fields (like the title) might be more relevant than others. We can adjust the importance of each field using field boosting: + +[source,console] +---- +GET /cooking_blog/_search +{ + "query": { + "multi_match": { + "query": "vegetarian curry", + "fields": ["title^3", "description^2", "tags"] <1> + } + } +} +---- +// TEST[continued] +<1> The `^` syntax applies a boost to specific fields: ++ +* `title^3`: The title field is 3 times more important than an unboosted field +* `description^2`: The description is 2 times more important +* `tags`: No boost applied (equivalent to `^1`) ++ +These boosts help tune relevance, prioritizing matches in the title over the description, and matches in the description over tags. + +Learn more about fields and per-field boosting in the <> reference. + +.Example response +[%collapsible] +============== +[source,console-result] +---- +{ + "took": 0, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 1, + "relation": "eq" + }, + "max_score": 7.546015, + "hits": [ + { + "_index": "cooking_blog", + "_id": "2", + "_score": 7.546015, + "_source": { + "title": "Spicy Thai Green Curry: A Vegetarian Adventure", <1> + "description": "Dive into the flavors of Thailand with this vibrant green curry. Packed with vegetables and aromatic herbs, this dish is both healthy and satisfying. Don't worry about the heat - you can easily adjust the spice level to your liking.", <2> + "author": "Liam Chen", + "date": "2023-05-05", + "category": "Main Course", + "tags": [ + "thai", + "vegetarian", + "curry", + "spicy" + ], <3> + "rating": 4.6 + } + } + ] + } +} +---- +// TESTRESPONSE[s/"took": 0/"took": "$body.took"/] +// TESTRESPONSE[s/"_score": 7.546015/"_score": $body.hits.hits.0._score/] +// TESTRESPONSE[s/"max_score": 7.546015/"max_score": $body.hits.max_score/] +<1> The title contains "Vegetarian" and "Curry", which matches our search terms. The title field has the highest boost (^3), contributing significantly to this document's relevance score. +<2> The description contains "curry" and related terms like "vegetables", further increasing the document's relevance. +<3> The tags include both "vegetarian" and "curry", providing an exact match for our search terms, albeit with no boost. + +This result demonstrates how the `multi_match` query with field boosts helps users find relevant recipes across multiple fields. +Even though the exact phrase "vegetarian curry" doesn't appear in any single field, the combination of matches across fields produces a highly relevant result. +============== + +[TIP] +==== +The `multi_match` query is often recommended over a single `match` query for most text search use cases, as it provides more flexibility and better matches user expectations. +==== + +[discrete] +[[full-text-filter-tutorial-filtering]] +=== Step 5: Filter and find exact matches + +<> allows you to narrow down your search results based on exact criteria. +Unlike full-text searches, filters are binary (yes/no) and do not affect the relevance score. +Filters execute faster than queries because excluded results don't need to be scored. + +This <> query will return only blog posts in the "Breakfast" category. + +[source,console] +---- +GET /cooking_blog/_search +{ + "query": { + "bool": { + "filter": [ + { "term": { "category.keyword": "Breakfast" } } <1> + ] + } + } +} +---- +// TEST[continued] +<1> Note the use of `category.keyword` here. This refers to the <> multi-field of the `category` field, ensuring an exact, case-sensitive match. + +[TIP] +==== +The `.keyword` suffix accesses the unanalyzed version of a field, enabling exact, case-sensitive matching. This works in two scenarios: + +1. *When using dynamic mapping for text fields*. Elasticsearch automatically creates a `.keyword` sub-field. +2. *When text fields are explicitly mapped with a `.keyword` sub-field*. For example, we explicitly mapped the `category` field in <> of this tutorial. +==== + +[discrete] +[[full-text-filter-tutorial-range-query]] +==== Search for posts within a date range + +Often users want to find content published within a specific time frame. +A <> query finds documents that fall within numeric or date ranges. + +[source,console] +---- +GET /cooking_blog/_search +{ + "query": { + "range": { + "date": { + "gte": "2023-05-01", <1> + "lte": "2023-05-31" <2> + } + } + } +} +---- +// TEST[continued] +<1> Greater than or equal to May 1, 2023. +<2> Less than or equal to May 31, 2023. + +[discrete] +[[full-text-filter-tutorial-term-query]] +==== Find exact matches + +Sometimes users want to search for exact terms to eliminate ambiguity in their search results. +A <> query searches for an exact term in a field without analyzing it. +Exact, case-sensitive matches on specific terms are often referred to as "keyword" searches. + +Here you'll search for the author "Maria Rodriguez" in the `author.keyword` field. + +[source,console] +---- +GET /cooking_blog/_search +{ + "query": { + "term": { + "author.keyword": "Maria Rodriguez" <1> + } + } +} +---- +// TEST[continued] +<1> The `term` query has zero flexibility. For example, here the queries `maria` or `maria rodriguez` would have zero hits, due to case sensitivity. + +[TIP] +==== +Avoid using the `term` query for <> because they are transformed by the analysis process. +==== + +[discrete] +[[full-text-filter-tutorial-complex-bool]] +=== Step 6: Combine multiple search criteria + +A <> query allows you to combine multiple query clauses to create sophisticated searches. +In this tutorial scenario it's useful for when users have complex requirements for finding recipes. + +Let's create a query that addresses the following user needs: + +* Must be a vegetarian recipe +* Should contain "curry" or "spicy" in the title or description +* Should be a main course +* Must not be a dessert +* Must have a rating of at least 4.5 +* Should prefer recipes published in the last month + +[source,console] +---- +GET /cooking_blog/_search +{ + "query": { + "bool": { + "must": [ + { "term": { "tags": "vegetarian" } }, + { + "range": { + "rating": { + "gte": 4.5 + } + } + } + ], + "should": [ + { + "term": { + "category": "Main Course" + } + }, + { + "multi_match": { + "query": "curry spicy", + "fields": [ + "title^2", + "description" + ] + } + }, + { + "range": { + "date": { + "gte": "now-1M/d" + } + } + } + ], + "must_not": [ <1> + { + "term": { + "category.keyword": "Dessert" + } + } + ] + } + } +} +---- +// TEST[continued] +<1> The `must_not` clause excludes documents that match the specified criteria. This is a powerful tool for filtering out unwanted results. + +.Example response +[%collapsible] +============== +[source,console-result] +---- +{ + "took": 1, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 1, + "relation": "eq" + }, + "max_score": 7.444513, + "hits": [ + { + "_index": "cooking_blog", + "_id": "2", + "_score": 7.444513, + "_source": { + "title": "Spicy Thai Green Curry: A Vegetarian Adventure", <1> + "description": "Dive into the flavors of Thailand with this vibrant green curry. Packed with vegetables and aromatic herbs, this dish is both healthy and satisfying. Don't worry about the heat - you can easily adjust the spice level to your liking.", <2> + "author": "Liam Chen", + "date": "2023-05-05", + "category": "Main Course", <3> + "tags": [ <4> + "thai", + "vegetarian", <5> + "curry", + "spicy" + ], + "rating": 4.6 <6> + } + } + ] + } +} +---- +// TESTRESPONSE[s/"took": 1/"took": "$body.took"/] +<1> The title contains "Spicy" and "Curry", matching our should condition. With the default <> behavior, this field contributes most to the relevance score. +<2> While the description also contains matching terms, only the best matching field's score is used by default. +<3> The recipe was published within the last month, satisfying our recency preference. +<4> The "Main Course" category satisfies another `should` condition. +<5> The "vegetarian" tag satisfies a `must` condition, while "curry" and "spicy" tags align with our `should` preferences. +<6> The rating of 4.6 meets our minimum rating requirement of 4.5. +============== + +[discrete] +[[full-text-filter-tutorial-learn-more]] +=== Learn more + +This tutorial introduced the basics of full-text search and filtering in {es}. +Building a real-world search experience requires understanding many more advanced concepts and techniques. +Here are some resources once you're ready to dive deeper: + +* <>: Learn about the core components of full-text search in {es}. +* <>: Understand all your options for searching and analyzing data in {es}. +* <>: Understand how text is processed for full-text search. +* <>: Learn about more advanced search techniques using the `_search` API, including semantic search. + + diff --git a/docs/reference/quickstart/getting-started.asciidoc b/docs/reference/quickstart/getting-started.asciidoc index 6b3095e07f9d4..03bfb62548b25 100644 --- a/docs/reference/quickstart/getting-started.asciidoc +++ b/docs/reference/quickstart/getting-started.asciidoc @@ -1,103 +1,155 @@ [[getting-started]] -== Quick start guide +== Index and search data using {es} APIs +++++ +Basics: Index and search using APIs +++++ -This guide helps you learn how to: +This quick start guide is a hands-on introduction to the fundamental concepts of Elasticsearch: <>. -* Run {es} and {kib} (using {ecloud} or in a local Docker dev environment), -* add simple (non-timestamped) dataset to {es}, -* run basic searches. +You'll learn how to create an index, add data as documents, work with dynamic and explicit mappings, and perform your first basic searches. [TIP] ==== -If you're interested in using {es} with Python, check out Elastic Search Labs. This is the best place to explore AI-powered search use cases, such as working with embeddings, vector search, and retrieval augmented generation (RAG). - -* https://www.elastic.co/search-labs/tutorials/search-tutorial/welcome[Tutorial]: this walks you through building a complete search solution with {es}, from the ground up. -* https://github.com/elastic/elasticsearch-labs[`elasticsearch-labs` repository]: it contains a range of Python https://github.com/elastic/elasticsearch-labs/tree/main/notebooks[notebooks] and https://github.com/elastic/elasticsearch-labs/tree/main/example-apps[example apps]. +The code examples in this tutorial are in {kibana-ref}/console-kibana.html[Console] syntax by default. +You can {kibana-ref}/console-kibana.html#import-export-console-requests[convert into other programming languages] in the Console UI. ==== [discrete] -[[run-elasticsearch]] -=== Run {es} +[[getting-started-requirements]] +=== Requirements -The simplest way to set up {es} is to create a managed deployment with {ess} on -{ecloud}. If you prefer to manage your own test environment, install and -run {es} using Docker. +You'll need a running {es} cluster, together with {kib} to use the Dev Tools API Console. +Run the following command in your terminal to set up a <>: -include::{es-ref-dir}/tab-widgets/code.asciidoc[] -include::{es-ref-dir}/tab-widgets/quick-start-install-widget.asciidoc[] +[source,sh] +---- +curl -fsSL https://elastic.co/start-local | sh +---- +// NOTCONSOLE + +//// +[source,console] +---- +PUT books +PUT my-explicit-mappings-books +---- +// TESTSETUP + +[source,console] +-------------------------------------------------- +DELETE books +DELETE my-explicit-mappings-books +-------------------------------------------------- +// TEARDOWN + +//// [discrete] -[[send-requests-to-elasticsearch]] -=== Send requests to {es} +[[getting-started-index-creation]] +=== Step 1: Create an index + +Create a new index named `books`: + +[source,console] +---- +PUT /books +---- +// TEST[skip: index already setup] -You send data and other requests to {es} using REST APIs. This lets you interact -with {es} using any client that sends HTTP requests, such as -https://curl.se[curl]. You can also use {kib}'s Console to send requests to -{es}. +The following response indicates the index was created successfully. -include::{es-ref-dir}/tab-widgets/api-call-widget.asciidoc[] +.Example response +[%collapsible] +=============== +[source,console-result] +---- +{ + "acknowledged": true, + "shards_acknowledged": true, + "index": "books" +} +---- +// TEST[skip: index already setup] +=============== [discrete] -[[add-data]] -=== Add data +[[getting-started-add-documents]] +=== Step 2: Add data to your index -You add data to {es} as JSON objects called documents. {es} stores these +[TIP] +==== +This tutorial uses {es} APIs, but there are many other ways to +<>. +==== + +You add data to {es} as JSON objects called documents. +{es} stores these documents in searchable indices. [discrete] -[[add-single-document]] +[[getting-started-add-single-document]] ==== Add a single document Submit the following indexing request to add a single document to the `books` index. -The request automatically creates the index. -//// -[source,console] ----- -PUT books ----- -// TESTSETUP -//// +[TIP] +==== +If the index didn't already exist, this request would automatically create it. +==== [source,console] ---- POST books/_doc -{"name": "Snow Crash", "author": "Neal Stephenson", "release_date": "1992-06-01", "page_count": 470} +{ + "name": "Snow Crash", + "author": "Neal Stephenson", + "release_date": "1992-06-01", + "page_count": 470 +} ---- -// TEST[s/_doc/_doc?refresh=wait_for/] +// TEST[continued] -The response includes metadata that {es} generates for the document including a unique `_id` for the document within the index. +The response includes metadata that {es} generates for the document, including a unique `_id` for the document within the index. -.Expand to see example response +.Example response [%collapsible] =============== [source,console-result] ---- { - "_index": "books", - "_id": "O0lG2IsBaSa7VYx_rEia", - "_version": 1, - "result": "created", - "_shards": { - "total": 2, - "successful": 2, - "failed": 0 + "_index": "books", <1> + "_id": "O0lG2IsBaSa7VYx_rEia", <2> + "_version": 1, <3> + "result": "created", <4> + "_shards": { <5> + "total": 2, <6> + "successful": 2, <7> + "failed": 0 <8> }, - "_seq_no": 0, - "_primary_term": 1 + "_seq_no": 0, <9> + "_primary_term": 1 <10> } ---- -// TEST[skip:TODO] +// TEST[s/O0lG2IsBaSa7VYx_rEia/*/] +<1> The `_index` field indicates the index the document was added to. +<2> The `_id` field is the unique identifier for the document. +<3> The `_version` field indicates the version of the document. +<4> The `result` field indicates the result of the indexing operation. +<5> The `_shards` field contains information about the number of <> that the indexing operation was executed on and the number that succeeded. +<6> The `total` field indicates the total number of shards for the index. +<7> The `successful` field indicates the number of shards that the indexing operation was executed on. +<8> The `failed` field indicates the number of shards that failed during the indexing operation. '0' indicates no failures. +<9> The `_seq_no` field holds a monotonically increasing number incremented for each indexing operation on a shard. +<10> The `_primary_term` field is a monotonically increasing number incremented each time a primary shard is assigned to a different node. =============== [discrete] -[[add-multiple-documents]] +[[getting-started-add-multiple-documents]] ==== Add multiple documents -Use the `_bulk` endpoint to add multiple documents in one request. Bulk data -must be newline-delimited JSON (NDJSON). Each line must end in a newline -character (`\n`), including the last line. +Use the <> to add multiple documents in one request. Bulk data +must be formatted as newline-delimited JSON (NDJSON). [source,console] ---- @@ -117,7 +169,7 @@ POST /_bulk You should receive a response indicating there were no errors. -.Expand to see example response +.Example response [%collapsible] =============== [source,console-result] @@ -213,33 +265,221 @@ You should receive a response indicating there were no errors. =============== [discrete] -[[qs-search-data]] -=== Search data +[[getting-started-mappings-and-data-types]] +=== Step 3: Define mappings and data types -Indexed documents are available for search in near real-time. +<> define how data is stored and indexed in {es}, like a schema in a relational database. [discrete] -[[search-all-documents]] +[[getting-started-dynamic-mapping]] +==== Use dynamic mapping + +When using dynamic mapping, {es} automatically creates mappings for new fields by default. +The documents we've added so far have used dynamic mapping, because we didn't specify a mapping when creating the index. + +To see how dynamic mapping works, add a new document to the `books` index with a field that doesn't appear in the existing documents. + +[source,console] +---- +POST /books/_doc +{ + "name": "The Great Gatsby", + "author": "F. Scott Fitzgerald", + "release_date": "1925-04-10", + "page_count": 180, + "language": "EN" <1> +} +---- +// TEST[continued] +<1> The new field. + +View the mapping for the `books` index with the <>. The new field `new_field` has been added to the mapping with a `text` data type. + +[source,console] +---- +GET /books/_mapping +---- +// TEST[continued] + +.Example response +[%collapsible] +=============== +[source,console-result] +---- +{ + "books": { + "mappings": { + "properties": { + "author": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "new_field": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "page_count": { + "type": "long" + }, + "release_date": { + "type": "date" + } + } + } + } +} +---- +// TEST[continued] +=============== + +[discrete] +[[getting-started-explicit-mapping]] +==== Define explicit mapping + +Create an index named `my-explicit-mappings-books` with explicit mappings. +Pass each field's properties as a JSON object. This object should contain the <> and any additional <>. + +[source,console] +---- +PUT /my-explicit-mappings-books +{ + "mappings": { + "dynamic": false, <1> + "properties": { <2> + "name": { "type": "text" }, + "author": { "type": "text" }, + "release_date": { "type": "date", "format": "yyyy-MM-dd" }, + "page_count": { "type": "integer" } + } + } +} +---- +// TEST[continued] +<1> Disables dynamic mapping for the index. Documents containing fields not defined in the mapping will be rejected. +<2> The `properties` object defines the fields and their data types for documents in this index. + +.Example response +[%collapsible] +=============== +[source,console-result] +---- +{ + "acknowledged": true, + "shards_acknowledged": true, + "index": "my-explicit-mappings-books" +} +---- +// TEST[skip:already created in setup] +=============== + +[discrete] +[[getting-started-combined-mapping]] +==== Combine dynamic and explicit mappings + +Explicit mappings are defined at index creation, and documents must conform to these mappings. +You can also use the <>. +When an index has the `dynamic` flag set to `true`, you can add new fields to documents without updating the mapping. + +This allows you to combine explicit and dynamic mappings. +Learn more about <>. + +[discrete] +[[getting-started-search-data]] +=== Step 4: Search your index + +Indexed documents are available for search in near real-time, using the <>. +// TODO: You'll find more detailed quick start guides in TODO + +[discrete] +[[getting-started-search-all-documents]] ==== Search all documents Run the following command to search the `books` index for all documents: + [source,console] ---- GET books/_search ---- // TEST[continued] -The `_source` of each hit contains the original -JSON object submitted during indexing. +.Example response +[%collapsible] +=============== +[source,console-result] +---- +{ + "took": 2, <1> + "timed_out": false, <2> + "_shards": { <3> + "total": 5, + "successful": 5, + "skipped": 0, + "failed": 0 + }, + "hits": { <4> + "total": { <5> + "value": 7, + "relation": "eq" + }, + "max_score": 1, <6> + "hits": [ + { + "_index": "books", <7> + "_id": "CwICQpIBO6vvGGiC_3Ls", <8> + "_score": 1, <9> + "_source": { <10> + "name": "Brave New World", + "author": "Aldous Huxley", + "release_date": "1932-06-01", + "page_count": 268 + } + }, + ... (truncated) + ] + } +} +---- +// TEST[continued] +<1> The `took` field indicates the time in milliseconds for {es} to execute the search +<2> The `timed_out` field indicates whether the search timed out +<3> The `_shards` field contains information about the number of <> that the search was executed on and the number that succeeded +<4> The `hits` object contains the search results +<5> The `total` object provides information about the total number of matching documents +<6> The `max_score` field indicates the highest relevance score among all matching documents +<7> The `_index` field indicates the index the document belongs to +<8> The `_id` field is the document's unique identifier +<9> The `_score` field indicates the relevance score of the document +<10> The `_source` field contains the original JSON object submitted during indexing +=============== [discrete] -[[qs-match-query]] +[[getting-started-match-query]] ==== `match` query -You can use the `match` query to search for documents that contain a specific value in a specific field. -This is the standard query for performing full-text search, including fuzzy matching and phrase searches. +You can use the <> to search for documents that contain a specific value in a specific field. +This is the standard query for full-text searches. Run the following command to search the `books` index for documents containing `brave` in the `name` field: + [source,console] ---- GET books/_search @@ -253,32 +493,63 @@ GET books/_search ---- // TEST[continued] -[discrete] -[[whats-next]] -=== Next steps - -Now that {es} is up and running and you've learned the basics, you'll probably want to test out larger datasets, or index your own data. +.Example response +[%collapsible] +=============== +[source,console-result] +---- +{ + "took": 9, + "timed_out": false, + "_shards": { + "total": 5, + "successful": 5, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 1, + "relation": "eq" + }, + "max_score": 0.6931471, <1> + "hits": [ + { + "_index": "books", + "_id": "CwICQpIBO6vvGGiC_3Ls", + "_score": 0.6931471, + "_source": { + "name": "Brave New World", + "author": "Aldous Huxley", + "release_date": "1932-06-01", + "page_count": 268 + } + } + ] + } +} +---- +// TEST[continued] +<1> The `max_score` is the score of the highest-scoring document in the results. In this case, there is only one matching document, so the `max_score` is the score of that document. +=============== [discrete] -[[whats-next-search-learn-more]] -==== Learn more about search queries +[[getting-started-delete-indices]] +=== Step 5: Delete your indices (optional) -* <>. Jump here to learn about exact value search, full-text search, vector search, and more, using the <>. +When following along with examples, you might want to delete an index to start from scratch. +You can delete indices using the <>. -[discrete] -[[whats-next-more-data]] -==== Add more data +For example, run the following command to delete the indices created in this tutorial: -* Learn how to {kibana-ref}/sample-data.html[install sample data] using {kib}. This is a quick way to test out {es} on larger workloads. -* Learn how to use the {kibana-ref}/connect-to-elasticsearch.html#upload-data-kibana[upload data UI] in {kib} to add your own CSV, TSV, or JSON files. -* Use the https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html[bulk API] to ingest your own datasets to {es}. +[source,console] +---- +DELETE /books +DELETE /my-explicit-mappings-books +---- +// TEST[skip:handled by setup/teardown] -[discrete] -[[whats-next-client-libraries]] -==== {es} programming language clients - -* Check out our https://www.elastic.co/guide/en/elasticsearch/client/index.html[client library] to work with your {es} instance in your preferred programming language. -* If you're using Python, check out https://www.elastic.co/search-labs[Elastic Search Labs] for a range of examples that use the {es} Python client. This is the best place to explore AI-powered search use cases, such as working with embeddings, vector search, and retrieval augmented generation (RAG). -** This extensive, hands-on https://www.elastic.co/search-labs/tutorials/search-tutorial/welcome[tutorial] -walks you through building a complete search solution with {es}, from the ground up. -** https://github.com/elastic/elasticsearch-labs[`elasticsearch-labs`] contains a range of executable Python https://github.com/elastic/elasticsearch-labs/tree/main/notebooks[notebooks] and https://github.com/elastic/elasticsearch-labs/tree/main/example-apps[example apps]. \ No newline at end of file +[CAUTION] +==== +Deleting an index permanently deletes its documents, shards, and metadata. +==== diff --git a/docs/reference/quickstart/index.asciidoc b/docs/reference/quickstart/index.asciidoc index e517d039e620b..330582956c457 100644 --- a/docs/reference/quickstart/index.asciidoc +++ b/docs/reference/quickstart/index.asciidoc @@ -1,10 +1,45 @@ [[quickstart]] -= Quickstart += Quick starts -Get started quickly with {es}. +Use these quick starts to get hands-on experience with the {es} APIs. +Unless otherwise noted, these examples will use queries written in <> syntax. -* Learn how to run {es} (and {kib}) for <>. -* Follow our <> to add data to {es} and query it. +[discrete] +[[quickstart-requirements]] +== Requirements + +You'll need a running {es} cluster, together with {kib} to use the Dev Tools API Console. +Run the following command in your terminal to set up a <>: + +[source,sh] +---- +curl -fsSL https://elastic.co/start-local | sh +---- +// NOTCONSOLE + +Alternatively, refer to our <>. + +[discrete] +[[quickstart-list]] +== Hands-on quick starts + +* <>. Learn about indices, documents, and mappings, and perform a basic search using the Query DSL. +* <>. Learn about different options for querying data, including full-text search and filtering, using the Query DSL. +* <>: Learn how to query and aggregate your data using {esql}. +* <>. Learn how to analyze data using different types of aggregations, including metrics, buckets, and pipelines. +* <>: Learn how to create embeddings for your data with `semantic_text` and query using the `semantic` query. +** <>: Learn how to combine semantic search with full-text search. +* <>: Learn how to ingest dense vector embeddings into {es}. + +.Working in Python +****************** + +If you're interested in using {es} with Python, check out Elastic Search Labs: + +* https://github.com/elastic/elasticsearch-labs[`elasticsearch-labs` repository]: Contains a range of Python https://github.com/elastic/elasticsearch-labs/tree/main/notebooks[notebooks] and https://github.com/elastic/elasticsearch-labs/tree/main/example-apps[example apps]. +* https://www.elastic.co/search-labs/tutorials/search-tutorial/welcome[Tutorial]: This walks you through building a complete search solution with {es} from the ground up using Flask. +****************** -include::run-elasticsearch-locally.asciidoc[] include::getting-started.asciidoc[] +include::full-text-filtering-tutorial.asciidoc[] +include::aggs-tutorial.asciidoc[] diff --git a/docs/reference/quickstart/run-elasticsearch-locally.asciidoc b/docs/reference/quickstart/run-elasticsearch-locally.asciidoc deleted file mode 100644 index 24e0f3f22350e..0000000000000 --- a/docs/reference/quickstart/run-elasticsearch-locally.asciidoc +++ /dev/null @@ -1,183 +0,0 @@ -[[run-elasticsearch-locally]] -== Run {es} locally in Docker (without security) -++++ -Local dev setup (Docker) -++++ - -[WARNING] -==== -*DO NOT USE THESE INSTRUCTIONS FOR PRODUCTION DEPLOYMENTS* - -The instructions on this page are for *local development only*. Do not use these instructions for production deployments, because they are not secure. -While this approach is convenient for experimenting and learning, you should never run the service in this way in a production environment. -==== - -The following commands help you very quickly spin up a single-node {es} cluster, together with {kib} in Docker. -Note that if you don't need the {kib} UI, you can skip those instructions. - -[discrete] -[[local-dev-why]] -=== When would I use this setup? - -Use this setup if you want to quickly spin up {es} (and {kib}) for local development or testing. - -For example you might: - -* Want to run a quick test to see how a feature works. -* Follow a tutorial or guide that requires an {es} cluster, like our <>. -* Experiment with the {es} APIs using different tools, like the Dev Tools Console, cURL, or an Elastic programming language client. -* Quickly spin up an {es} cluster to test an executable https://github.com/elastic/elasticsearch-labs/tree/main/notebooks#readme[Python notebook] locally. - -[discrete] -[[local-dev-prerequisites]] -=== Prerequisites - -If you don't have Docker installed, https://www.docker.com/products/docker-desktop[download and install Docker Desktop] for your operating system. - -[discrete] -[[local-dev-env-vars]] -=== Set environment variables - -Configure the following environment variables. - -[source,sh] ----- -export ELASTIC_PASSWORD="" # password for "elastic" username -export KIBANA_PASSWORD="" # Used _internally_ by Kibana, must be at least 6 characters long ----- - -[discrete] -[[local-dev-create-docker-network]] -=== Create a Docker network - -To run both {es} and {kib}, you'll need to create a Docker network: - -[source,sh] ----- -docker network create elastic-net ----- - -[discrete] -[[local-dev-run-es]] -=== Run {es} - -Start the {es} container with the following command: - -ifeval::["{release-state}"=="unreleased"] -WARNING: Version {version} has not yet been released. -No Docker image is currently available for {es} {version}. -endif::[] - -[source,sh,subs="attributes"] ----- -docker run -p 127.0.0.1:9200:9200 -d --name elasticsearch --network elastic-net \ - -e ELASTIC_PASSWORD=$ELASTIC_PASSWORD \ - -e "discovery.type=single-node" \ - -e "xpack.security.http.ssl.enabled=false" \ - -e "xpack.license.self_generated.type=trial" \ - {docker-image} ----- - -[discrete] -[[local-dev-run-kib]] -=== Run {kib} (optional) - -To run {kib}, you must first set the `kibana_system` password in the {es} container. - -[source,sh,subs="attributes"] ----- -# configure the Kibana password in the ES container -curl -u elastic:$ELASTIC_PASSWORD \ - -X POST \ - http://localhost:9200/_security/user/kibana_system/_password \ - -d '{"password":"'"$KIBANA_PASSWORD"'"}' \ - -H 'Content-Type: application/json' ----- -// NOTCONSOLE - -Start the {kib} container with the following command: - -ifeval::["{release-state}"=="unreleased"] -WARNING: Version {version} has not yet been released. -No Docker image is currently available for {es} {version}. -endif::[] - -[source,sh,subs="attributes"] ----- -docker run -p 127.0.0.1:5601:5601 -d --name kibana --network elastic-net \ - -e ELASTICSEARCH_URL=http://elasticsearch:9200 \ - -e ELASTICSEARCH_HOSTS=http://elasticsearch:9200 \ - -e ELASTICSEARCH_USERNAME=kibana_system \ - -e ELASTICSEARCH_PASSWORD=$KIBANA_PASSWORD \ - -e "xpack.security.enabled=false" \ - -e "xpack.license.self_generated.type=trial" \ - {kib-docker-image} ----- - -When you access {kib}, use `elastic` as the username and the password you set earlier for the `ELASTIC_PASSWORD` environment variable. - -[NOTE] -==== -The service is started with a trial license. The trial license enables all features of Elasticsearch for a trial period of 30 days. After the trial period expires, the license is downgraded to a basic license, which is free forever. If you prefer to skip the trial and use the basic license, set the value of the `xpack.license.self_generated.type` variable to basic instead. For a detailed feature comparison between the different licenses, refer to our https://www.elastic.co/subscriptions[subscriptions page]. -==== - -[discrete] -[[local-dev-connecting-clients]] -== Connecting to {es} with language clients - -To connect to the {es} cluster from a language client, you can use basic authentication with the `elastic` username and the password you set in the environment variable. - -.*Expand* for details -[%collapsible] -============== - -You'll use the following connection details: - -* **{es} endpoint**: `http://localhost:9200` -* **Username**: `elastic` -* **Password**: `$ELASTIC_PASSWORD` (Value you set in the environment variable) - -For example, to connect with the Python `elasticsearch` client: - -[source,python] ----- -import os -from elasticsearch import Elasticsearch - -username = 'elastic' -password = os.getenv('ELASTIC_PASSWORD') # Value you set in the environment variable - -client = Elasticsearch( - "http://localhost:9200", - basic_auth=(username, password) -) - -print(client.info()) ----- - -Here's an example curl command using basic authentication: - -[source,sh,subs="attributes"] ----- -curl -u elastic:$ELASTIC_PASSWORD \ - -X PUT \ - http://localhost:9200/my-new-index \ - -H 'Content-Type: application/json' ----- -// NOTCONSOLE - -============== - -[discrete] -[[local-dev-next-steps]] -=== Next steps - -Use our <> to learn the basics of {es}: how to add data and query it. - -[discrete] -[[local-dev-production]] -=== Moving to production - -This setup is not suitable for production use. For production deployments, we recommend using our managed service on Elastic Cloud. https://cloud.elastic.co/registration[Sign up for a free trial] (no credit card required). - -Otherwise, refer to https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html[Install {es}] to learn about the various options for installing {es} in a self-managed production environment, including using Docker. diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc index e0568f500f268..506dff7891ad2 100644 --- a/docs/reference/redirects.asciidoc +++ b/docs/reference/redirects.asciidoc @@ -1926,4 +1926,19 @@ Refer to <>. [role="exclude",id="remote-clusters-privileges"] === Configure roles and users for remote clusters -Refer to <>. \ No newline at end of file +Refer to <>. + +[role="exclude",id="put-geoip-database-api"] +=== Create or update geoip database configuration API + +Refer to <>. + +[role="exclude",id="get-geoip-database-api"] +=== Get geoip database configuration + +Refer to <>. + +[role="exclude",id="delete-geoip-database-api"] +=== Delete geoip database configuration API + +Refer to <>. diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc index 6a03ed03f2610..c0b1377512f1e 100644 --- a/docs/reference/release-notes.asciidoc +++ b/docs/reference/release-notes.asciidoc @@ -6,7 +6,15 @@ This section summarizes the changes in each release. +* <> +* <> +* <> * <> +* <> +* <> +* <> +* <> +* <> * <> * <> * <> @@ -71,7 +79,15 @@ This section summarizes the changes in each release. -- +include::release-notes/8.18.0.asciidoc[] +include::release-notes/8.17.0.asciidoc[] +include::release-notes/8.16.1.asciidoc[] include::release-notes/8.16.0.asciidoc[] +include::release-notes/8.15.5.asciidoc[] +include::release-notes/8.15.4.asciidoc[] +include::release-notes/8.15.3.asciidoc[] +include::release-notes/8.15.2.asciidoc[] +include::release-notes/8.15.1.asciidoc[] include::release-notes/8.15.0.asciidoc[] include::release-notes/8.14.3.asciidoc[] include::release-notes/8.14.2.asciidoc[] diff --git a/docs/reference/release-notes/8.12.0.asciidoc b/docs/reference/release-notes/8.12.0.asciidoc index bfa99401f41a2..bd0ae032ef0b9 100644 --- a/docs/reference/release-notes/8.12.0.asciidoc +++ b/docs/reference/release-notes/8.12.0.asciidoc @@ -11,7 +11,7 @@ Also see <>. + When using `int8_hnsw` and the default `confidence_interval` (or any `confidence_interval` less than `1.0`) and when there are deleted documents in the segments, quantiles may fail to build and prevent merging. - ++ This issue is fixed in 8.12.1. * When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes, diff --git a/docs/reference/release-notes/8.15.0.asciidoc b/docs/reference/release-notes/8.15.0.asciidoc index bed1912fc1b84..80c86c7079f0c 100644 --- a/docs/reference/release-notes/8.15.0.asciidoc +++ b/docs/reference/release-notes/8.15.0.asciidoc @@ -30,6 +30,28 @@ signed integer) may encounter errors (issue: {es-issue}111854[#111854]) `xpack.security.authc.realms.*.files.role_mapping` configuration option. As a workaround, custom role mappings can be configured using the https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-put-role-mapping.html[REST API] (issue: {es-issue}112503[#112503]) +* ES|QL queries can lead to node crashes due to Out Of Memory errors when: +** Multiple indices match the query pattern +** These indices have many conflicting field mappings +** Many of those fields are included in the request +These issues deplete heap memory, increasing the likelihood of OOM errors. (issue: {es-issue}111964[#111964], {es-issue}111358[#111358]). +In Kibana, you might indirectly execute these queries when using Discover, or adding a Field Statistics panel to a dashboard. ++ +To work around this issue, you have a number of options: +** Downgrade to an earlier version +** Upgrade to 8.15.2 upon release +** Follow the instructions to +<> +** Change the default data view in Discover to a smaller set of indices and/or one with fewer mapping conflicts. + +* Synthetic source bug. Synthetic source may fail generating the _source at runtime, causing failures in get APIs or +partial failures in the search APIs. The result is that for the affected documents the _source can't be retrieved. +There is no workaround and the only option to is to upgrade to 8.15.2 when released. ++ +If you use synthetic source then you may be affected by this bug if the following is true: +** If you have more fields then the `index.mapping.total_fields.limit` setting allows. +** If you use dynamic mappings and the `index.mapping.total_fields.ignore_dynamic_beyond_limit` setting is enabled. + [[breaking-8.15.0]] [float] === Breaking changes diff --git a/docs/reference/release-notes/8.15.1.asciidoc b/docs/reference/release-notes/8.15.1.asciidoc new file mode 100644 index 0000000000000..7c48f457e3b4e --- /dev/null +++ b/docs/reference/release-notes/8.15.1.asciidoc @@ -0,0 +1,103 @@ +[[release-notes-8.15.1]] +== {es} version 8.15.1 + +Also see <>. + +[[known-issues-8.15.1]] +[float] +=== Known issues +* Elasticsearch will not start if custom role mappings are configured using the +`xpack.security.authc.realms.*.files.role_mapping` configuration option. As a workaround, custom role mappings +can be configured using the https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-put-role-mapping.html[REST API] (issue: {es-issue}112503[#112503]) + +* ES|QL queries can lead to node crashes due to Out Of Memory errors when: +** Multiple indices match the query pattern +** These indices have many conflicting field mappings +** Many of those fields are included in the request +These issues deplete heap memory, increasing the likelihood of OOM errors. (issue: {es-issue}111964[#111964], {es-issue}111358[#111358]). +In Kibana, you might indirectly execute these queries when using Discover, or adding a Field Statistics panel to a dashboard. ++ +To work around this issue, you have a number of options: +** Downgrade to an earlier version +** Upgrade to 8.15.2 upon release +** Follow the instructions to +<> +** Change the default data view in Discover to a smaller set of indices and/or one with fewer mapping conflicts. + +* Index Stats, Node Stats and Cluster Stats API can return a null pointer exception if an index contains a `dense_vector` field +but there is an index segment that does not contain any documents with a dense vector field ({es-pull}112720[#112720]). Workarounds: +** If the affected index already contains documents with a dense vector field, force merge the index to a single segment. +** If the affected index does not already contain documents with a dense vector field, index a document with a dense vector field +and then force merge to a single segment. +** If the affected index's `dense_vector` fields are unused, reindex without the `dense_vector` fields. + +* Synthetic source bug. Synthetic source may fail generating the _source at runtime, causing failures in get APIs or +partial failures in the search APIs. The result is that for the affected documents the _source can't be retrieved. +There is no workaround and the only option to is to upgrade to 8.15.2 when released. ++ +If you use synthetic source then you may be affected by this bug if the following is true: +** If you have more fields then the `index.mapping.total_fields.limit` setting allows. +** If you use dynamic mappings and the `index.mapping.total_fields.ignore_dynamic_beyond_limit` setting is enabled. + +[[bug-8.15.1]] +[float] +=== Bug fixes + +Aggregations:: +* Revert "Avoid bucket copies in Aggs" {es-pull}111758[#111758] (issue: {es-issue}111679[#111679]) + +Authorization:: +* Fix DLS over Runtime Fields {es-pull}112260[#112260] (issue: {es-issue}111637[#111637]) + +ES|QL:: +* Avoid losing error message in failure collector {es-pull}111983[#111983] (issue: {es-issue}111894[#111894]) +* Avoid wrapping rejection exception in exchange {es-pull}112178[#112178] (issue: {es-issue}112106[#112106]) +* ESQL: Fix for overzealous validation in case of invalid mapped fields {es-pull}111475[#111475] (issue: {es-issue}111452[#111452]) + +Geo:: +* Add maximum nested depth check to WKT parser {es-pull}111843[#111843] +* Always check `crsType` when folding spatial functions {es-pull}112090[#112090] (issue: {es-issue}112089[#112089]) +* Fix NPE when executing doc value queries over shape geometries with empty segments {es-pull}112139[#112139] + +Indices APIs:: +* Fix template alias parsing livelock {es-pull}112217[#112217] + +Infra/Core:: +* Fix windows memory locking {es-pull}111866[#111866] (issue: {es-issue}111847[#111847]) + +Ingest Node:: +* Fixing incorrect bulk request took time {es-pull}111863[#111863] (issue: {es-issue}111854[#111854]) +* Improve performance of grok pattern cycle detection {es-pull}111947[#111947] + +Logs:: +* Merge multiple ignored source entires for the same field {es-pull}111994[#111994] (issue: {es-issue}111694[#111694]) + +Machine Learning:: +* [Inference API] Move Delete inference checks to threadpool worker {es-pull}111646[#111646] + +Mapping:: +* Check for valid `parentDoc` before retrieving its previous {es-pull}112005[#112005] (issue: {es-issue}111990[#111990]) +* Fix calculation of parent offset for ignored source in some cases {es-pull}112046[#112046] +* Fix synthetic source for empty nested objects {es-pull}111943[#111943] (issue: {es-issue}111811[#111811]) +* No error when `store_array_source` is used without synthetic source {es-pull}111966[#111966] +* Prevent synthetic field loaders accessing stored fields from using stale data {es-pull}112173[#112173] (issue: {es-issue}112156[#112156]) + +Ranking:: +* Properly handle filters on `TextSimilarityRank` retriever {es-pull}111673[#111673] + +Relevance:: +* Semantic reranking should fail whenever inference ID does not exist {es-pull}112038[#112038] (issue: {es-issue}111934[#111934]) +* [Bugfix] Add `accessDeclaredMembers` permission to allow search application templates to parse floats {es-pull}111285[#111285] + +Search:: +* Explain Function Score Query {es-pull}111807[#111807] + +Security:: +* Fix "unexpected field [remote_cluster]" for CCS (RCS 1.0) when using API key that references `remote_cluster` {es-pull}112226[#112226] +* Fix connection timeout for `OpenIdConnectAuthenticator` get Userinfo {es-pull}112230[#112230] + +Vector Search:: +* Fix `NullPointerException` when doing knn search on empty index without dims {es-pull}111756[#111756] (issue: {es-issue}111733[#111733]) +* Speed up dense/sparse vector stats {es-pull}111729[#111729] (issue: {es-issue}111715[#111715]) + + diff --git a/docs/reference/release-notes/8.15.2.asciidoc b/docs/reference/release-notes/8.15.2.asciidoc new file mode 100644 index 0000000000000..d2853215098cd --- /dev/null +++ b/docs/reference/release-notes/8.15.2.asciidoc @@ -0,0 +1,44 @@ +[[release-notes-8.15.2]] +== {es} version 8.15.2 + +Also see <>. + +[[bug-8.15.2]] +[float] +=== Bug fixes + +Authorization:: +* Fix remote cluster credential secure settings reload {es-pull}111535[#111535] + +ES|QL:: +* ESQL: Don't mutate the `BoolQueryBuilder` in plan {es-pull}111519[#111519] +* ES|QL: Fix `ResolvedEnrichPolicy` serialization (bwc) in v 8.15 {es-pull}112985[#112985] (issue: {es-issue}112968[#112968]) +* Fix union-types where one index is missing the field {es-pull}111932[#111932] (issue: {es-issue}111912[#111912]) +* Support widening of numeric types in union-types {es-pull}112610[#112610] (issue: {es-issue}111277[#111277]) +* Shorten error messages for UnsupportedAttributes {es-pull}112819[#112819] +* Reduce memory footprint of serialized query execution plans {es-pull}112865[#112865] + +Infra/Core:: +* JSON parse failures should be 4xx codes {es-pull}112703[#112703] +* Json parsing exceptions should not cause 500 errors {es-pull}111548[#111548] (issue: {es-issue}111542[#111542]) +* Make sure file accesses in `DnRoleMapper` are done in stack frames with permissions {es-pull}112400[#112400] + +Ingest Node:: +* Fix missing header in `put_geoip_database` JSON spec {es-pull}112581[#112581] + +Logs:: +* Fix encoding of dynamic arrays in ignored source {es-pull}112713[#112713] + +Mapping:: +* Full coverage of ECS by ecs@mappings when `date_detection` is disabled {es-pull}112444[#112444] (issue: {es-issue}112398[#112398]) + +Search:: +* Fix parsing error in `_terms_enum` API {es-pull}112872[#112872] (issue: {es-issue}94378[#94378]) + +Security:: +* Allowlist `tracestate` header on remote server port {es-pull}112649[#112649] + +Vector Search:: +* Fix NPE in `dense_vector` stats {es-pull}112720[#112720] + + diff --git a/docs/reference/release-notes/8.15.3.asciidoc b/docs/reference/release-notes/8.15.3.asciidoc new file mode 100644 index 0000000000000..029da1af740e5 --- /dev/null +++ b/docs/reference/release-notes/8.15.3.asciidoc @@ -0,0 +1,47 @@ +[[release-notes-8.15.3]] +== {es} version 8.15.3 + +Also see <>. + +[[bug-8.15.3]] +[float] +=== Bug fixes + +Aggregations:: +* Don't validate internal stats if they are empty {es-pull}113846[#113846] (issue: {es-issue}113811[#113811]) +* Fix `needsScore` computation in `GlobalOrdCardinalityAggregator` {es-pull}113129[#113129] (issue: {es-issue}112975[#112975]) + +Authentication:: +* Enables cluster state role mapper, to include ECK operator-defined role mappings in role resolution {es-pull}114337[#114337] + +ES|QL:: +* ES|QL: Ensure minimum capacity for `PlanStreamInput` caches {es-pull}114116[#114116] +* ES|QL: Skip CASE function from `InferIsNotNull` rule checks {es-pull}113123[#113123] (issue: {es-issue}112704[#112704]) +* [ESQL] Fix init value in max float aggregation {es-pull}113699[#113699] +* [ESQL] Support datetime data type in Least and Greatest functions {es-pull}113961[#113961] + +Machine Learning:: +* Fix check on E5 model platform compatibility {es-pull}113437[#113437] (issue: {es-issue}113577[#113577]) +* Handle parsing ingest processors where definition is not a object {es-pull}113697[#113697] (issue: {es-issue}113615[#113615]) +* [ML][backport] Warn for model load failures if they have a status code <500 {es-pull}113410[#113410] +* [M] Fix error message formatting {es-pull}113266[#113266] + +Search:: +* Fix analyzed wildcard query in simple_query_string when disjunctions is empty {es-pull}114264[#114264] (issue: {es-issue}114185[#114185]) +* Fix collapse interaction with stored fields {es-pull}112761[#112761] (issue: {es-issue}112646[#112646]) + +[[enhancement-8.15.3]] +[float] +=== Enhancements + +Machine Learning:: +* Write downloaded model parts async {es-pull}111684[#111684] + +[[upgrade-8.15.3]] +[float] +=== Upgrades + +Snapshot/Restore:: +* Upgrade protobufer to 3.25.5 {es-pull}113869[#113869] + + diff --git a/docs/reference/release-notes/8.15.4.asciidoc b/docs/reference/release-notes/8.15.4.asciidoc new file mode 100644 index 0000000000000..61a54363f3c6d --- /dev/null +++ b/docs/reference/release-notes/8.15.4.asciidoc @@ -0,0 +1,33 @@ +[[release-notes-8.15.4]] +== {es} version 8.15.4 + +Also see <>. + +[[bug-8.15.4]] +[float] +=== Bug fixes + +Aggregations:: +* Always check the parent breaker with zero bytes in `PreallocatedCircuitBreakerService` {es-pull}115181[#115181] + +ES|QL:: +* ESQL: Disable pushdown of WHERE past STATS {es-pull}115308[#115308] (issue: {es-issue}115281[#115281]) + +Geo:: +* Try to simplify geometries that fail with `TopologyException` {es-pull}115834[#115834] + +Infra/Core:: +* Guard blob store local directory creation with `doPrivileged` {es-pull}115459[#115459] + +License:: +* Fix lingering license warning header in IP filter {es-pull}115510[#115510] (issue: {es-issue}114865[#114865]) + +Search:: +* Do not exclude empty arrays or empty objects in source filtering {es-pull}112250[#112250] (issue: {es-issue}109668[#109668]) +* Fix synthetic source handling for `bit` type in `dense_vector` field {es-pull}114407[#114407] (issue: {es-issue}114402[#114402]) + +Vector Search:: +* Fix dim validation for bit `element_type` {es-pull}114533[#114533] +* Support semantic_text in object fields {es-pull}114601[#114601] (issue: {es-issue}114401[#114401]) + + diff --git a/docs/reference/release-notes/8.15.5.asciidoc b/docs/reference/release-notes/8.15.5.asciidoc new file mode 100644 index 0000000000000..4590b809b4329 --- /dev/null +++ b/docs/reference/release-notes/8.15.5.asciidoc @@ -0,0 +1,29 @@ +[[release-notes-8.15.5]] +== {es} version 8.15.5 + +Also see <>. + +[[bug-8.15.5]] +[float] +=== Bug fixes + +Machine Learning:: +* Fix bug in ML autoscaling when some node info is unavailable {es-pull}116650[#116650] + +Mapping:: +* Change synthetic source logic for `constant_keyword` {es-pull}117182[#117182] (issue: {es-issue}117083[#117083]) + +Relevance:: +* Fix handling of bulk requests with semantic text fields and delete ops {es-pull}116942[#116942] + +Vector Search:: +* Update Semantic Query To Handle Zero Size Responses {es-pull}116277[#116277] (issue: {es-issue}116083[#116083]) + +[[enhancement-8.15.5]] +[float] +=== Enhancements + +Data streams:: +* Apm-data: disable date_detection for all apm data streams {es-pull}116995[#116995] + + diff --git a/docs/reference/release-notes/8.16.0.asciidoc b/docs/reference/release-notes/8.16.0.asciidoc index 7b2e7459be968..fd7ef963d9ff7 100644 --- a/docs/reference/release-notes/8.16.0.asciidoc +++ b/docs/reference/release-notes/8.16.0.asciidoc @@ -1,8 +1,513 @@ [[release-notes-8.16.0]] == {es} version 8.16.0 -coming[8.16.0] - Also see <>. +[[breaking-8.16.0]] +[float] +=== Breaking changes + +Analysis:: +* Set lenient to true by default when using updateable synonyms {es-pull}110901[#110901] + +Data streams:: +* Update data stream lifecycle telemetry to track global retention {es-pull}112451[#112451] + +ES|QL:: +* Entirely remove META FUNCTIONS {es-pull}113967[#113967] + +Mapping:: +* JDK locale database change {es-pull}113975[#113975] + +Search:: +* Adding breaking change entry for retrievers {es-pull}115399[#115399] + +[[bug-8.16.0]] +[float] +=== Bug fixes + +Aggregations:: +* Always check the parent breaker with zero bytes in `PreallocatedCircuitBreakerService` {es-pull}115181[#115181] +* Force using the last centroid during merging {es-pull}111644[#111644] (issue: {es-issue}111065[#111065]) + +Authentication:: +* Check for disabling own user in Put User API {es-pull}112262[#112262] (issue: {es-issue}90205[#90205]) +* Expose cluster-state role mappings in APIs {es-pull}114951[#114951] + +Authorization:: +* Fix DLS & FLS sometimes being enforced when it is disabled {es-pull}111915[#111915] (issue: {es-issue}94709[#94709]) +* Fix DLS using runtime fields and synthetic source {es-pull}112341[#112341] + +CRUD:: +* Don't fail retention lease sync actions due to capacity constraints {es-pull}109414[#109414] (issue: {es-issue}105926[#105926]) + +Cluster Coordination:: +* Ensure clean thread context in `MasterService` {es-pull}114512[#114512] + +Data streams:: +* Adding support for data streams with a match-all template {es-pull}111311[#111311] (issue: {es-issue}111204[#111204]) +* Exclude internal data streams from global retention {es-pull}112100[#112100] +* Fix verbose get data stream API not requiring extra privileges {es-pull}112973[#112973] +* OTel mappings: avoid metrics to be rejected when attributes are malformed {es-pull}114856[#114856] +* Resolve pipelines from template on lazy rollover write {es-pull}116031[#116031] (issue: {es-issue}112781[#112781]) +* [apm-data] Apply lazy rollover on index template creation {es-pull}116219[#116219] (issue: {es-issue}116230[#116230]) +* [otel-data] Add more kubernetes aliases {es-pull}115429[#115429] +* logs-apm.error-*: define log.level field as keyword {es-pull}112440[#112440] + +Distributed:: +* Handle `InternalSendException` inline for non-forking handlers {es-pull}114375[#114375] + +EQL:: +* Fix validation of TEXT fields with case insensitive comparison {es-pull}111238[#111238] (issue: {es-issue}111235[#111235]) + +ES|QL:: +* Add Values aggregation tests, fix `ConstantBytesRefBlock` memory handling {es-pull}111367[#111367] +* Align year diffing to the rest of the units in DATE_DIFF: chronological {es-pull}113103[#113103] (issue: {es-issue}112482[#112482]) +* Disable pushdown of WHERE past STATS {es-pull}115308[#115308] (issue: {es-issue}115281[#115281]) +* Fix CASE when conditions are multivalued {es-pull}112401[#112401] (issue: {es-issue}112359[#112359]) +* Fix DEBUG log of filter {es-pull}116086[#116086] (issue: {es-issue}116055[#116055]) +* Fix Double operations returning infinite {es-pull}111064[#111064] (issue: {es-issue}111026[#111026]) +* Fix `REVERSE` with backspace character {es-pull}115245[#115245] (issues: {es-issue}114372[#114372], {es-issue}115227[#115227], {es-issue}115228[#115228]) +* Fix a bug in VALUES agg {es-pull}115952[#115952] +* Fix a bug in `MV_PERCENTILE` {es-pull}112218[#112218] (issues: {es-issue}112193[#112193], {es-issue}112180[#112180], {es-issue}112187[#112187], {es-issue}112188[#112188]) +* Fix filtered grouping on ords {es-pull}115312[#115312] (issue: {es-issue}114897[#114897]) +* Fix grammar changes around per agg filtering {es-pull}114848[#114848] +* Fix serialization during `can_match` {es-pull}111779[#111779] (issues: {es-issue}111701[#111701], {es-issue}111726[#111726]) +* Fix synthetic attribute pruning {es-pull}111413[#111413] (issue: {es-issue}105821[#105821]) +* Don't lose the original casting error message {es-pull}111968[#111968] (issue: {es-issue}111967[#111967]) +* Fix for missing indices error message {es-pull}111797[#111797] (issue: {es-issue}111712[#111712]) +* Restrict sorting for `_source` and counter field types {es-pull}114638[#114638] (issues: {es-issue}114423[#114423], {es-issue}111976[#111976]) +* Better validation for GROK patterns {es-pull}110574[#110574] (issue: {es-issue}110533[#110533]) +* Better validation for RLIKE patterns {es-pull}112489[#112489] (issue: {es-issue}112485[#112485]) +* Better validation of GROK patterns {es-pull}112200[#112200] (issue: {es-issue}112111[#112111]) +* Fix LIMIT pushdown past MV_EXPAND {es-pull}115624[#115624] (issues: {es-issue}102084[#102084], {es-issue}102061[#102061]) +* Fix ST_CENTROID_AGG when no records are aggregated {es-pull}114888[#114888] (issue: {es-issue}106025[#106025]) +* Spatial search functions support multi-valued fields in compute engine {es-pull}112063[#112063] (issues: {es-issue}112102[#112102], {es-issue}112505[#112505], {es-issue}110830[#110830]) +* Check expression resolved before checking its data type in `ImplicitCasting` {es-pull}113314[#113314] (issue: {es-issue}113242[#113242]) +* Simplify patterns for subfields {es-pull}111118[#111118] +* Simplify syntax of named parameter for identifier and pattern {es-pull}115061[#115061] +* Skip validating remote cluster index names in parser {es-pull}114271[#114271] +* Use `RangeQuery` and String in `BinaryComparison` on datetime fields {es-pull}110669[#110669] (issue: {es-issue}107900[#107900]) +* Verify aggregation filter's type is boolean to avoid `class_cast_exception` {es-pull}116274[#116274] +* Add tests for stats by constant {es-pull}110593[#110593] (issue: {es-issue}105383[#105383]) +* Make named parameter for identifier and pattern snapshot {es-pull}114784[#114784] +* Validate `mv_sort` order {es-pull}110021[#110021] (issue: {es-issue}109910[#109910]) + +Geo:: +* Fix cases of collections with one point {es-pull}111193[#111193] (issue: {es-issue}110982[#110982]) +* Try to simplify geometries that fail with `TopologyException` {es-pull}115834[#115834] + +Health:: +* Set `replica_unassigned_buffer_time` in constructor {es-pull}112612[#112612] + +ILM+SLM:: +* Make `SnapshotLifecycleStats` immutable so `SnapshotLifecycleMetadata.EMPTY` isn't changed as side-effect {es-pull}111215[#111215] + +Indices APIs:: +* Align dot prefix validation with Serverless {es-pull}116266[#116266] +* Revert "Add `ResolvedExpression` wrapper" {es-pull}115317[#115317] + +Infra/Core:: +* Fix max file size check to use `getMaxFileSize` {es-pull}113723[#113723] (issue: {es-issue}113705[#113705]) +* Guard blob store local directory creation with `doPrivileged` {es-pull}115459[#115459] +* Handle `BigInteger` in xcontent copy {es-pull}111937[#111937] (issue: {es-issue}111812[#111812]) +* Report JVM stats for all memory pools (97046) {es-pull}115117[#115117] (issue: {es-issue}97046[#97046]) +* `ByteArrayStreamInput:` Return -1 when there are no more bytes to read {es-pull}112214[#112214] + +Infra/Logging:: +* Only emit product origin in deprecation log if present {es-pull}111683[#111683] (issue: {es-issue}81757[#81757]) + +Infra/Settings:: +* GET _cluster/settings with include_defaults returns the expected fallback value if defined in elasticsearch.yml {es-pull}110816[#110816] (issue: {es-issue}110815[#110815]) + +Ingest Node:: +* Fix IPinfo geolocation schema {es-pull}115147[#115147] +* Fix `getDatabaseType` for unusual MMDBs {es-pull}112888[#112888] + +License:: +* Fix Start Trial API output acknowledgement header for features {es-pull}111740[#111740] (issue: {es-issue}111739[#111739]) +* Fix `TokenService` always appearing used in Feature Usage {es-pull}112263[#112263] (issue: {es-issue}61956[#61956]) +* Fix lingering license warning header in IP filter {es-pull}115510[#115510] (issue: {es-issue}114865[#114865]) + +Logs:: +* Do not expand dots when storing objects in ignored source {es-pull}113910[#113910] +* Fix `ignore_above` handling in synthetic source when index level setting is used {es-pull}113570[#113570] (issue: {es-issue}113538[#113538]) +* Fix synthetic source for flattened field when used with `ignore_above` {es-pull}113499[#113499] (issue: {es-issue}112044[#112044]) +* Prohibit changes to index mode, source, and sort settings during restore {es-pull}115811[#115811] + +Machine Learning:: +* Avoid `ModelAssignment` deadlock {es-pull}109684[#109684] +* Avoid `catch (Throwable t)` in `AmazonBedrockStreamingChatProcessor` {es-pull}115715[#115715] +* Allow for `pytorch_inference` results to include zero-dimensional tensors +* Empty percentile results no longer throw no_such_element_exception in Anomaly Detection jobs {es-pull}116015[#116015] (issue: {es-issue}116013[#116013]) +* Fix NPE in Get Deployment Stats {es-pull}115404[#115404] +* Fix bug in ML serverless autoscaling which prevented trained model updates from triggering a scale up {es-pull}110734[#110734] +* Fix stream support for `TaskType.ANY` {es-pull}115656[#115656] +* Fix parameter initialization for large forecasting models {ml-pull}2759[#2759] +* Forward bedrock connection errors to user {es-pull}115868[#115868] +* Ignore unrecognized openai sse fields {es-pull}114715[#114715] +* Prevent NPE if model assignment is removed while waiting to start {es-pull}115430[#115430] +* Send mid-stream errors to users {es-pull}114549[#114549] +* Temporarily return both `modelId` and `inferenceId` for GET /_inference until we migrate clients to only `inferenceId` {es-pull}111490[#111490] +* Warn for model load failures if they have a status code <500 {es-pull}113280[#113280] +* [Inference API] Remove unused Cohere rerank service settings fields in a BWC way {es-pull}110427[#110427] +* [ML] Create Inference API will no longer return model_id and now only return inference_id {es-pull}112508[#112508] + +Mapping:: +* Fix `MapperBuilderContext#isDataStream` when used in dynamic mappers {es-pull}110554[#110554] +* Fix synthetic source field names for multi-fields {es-pull}112850[#112850] +* Retrieve the source for objects and arrays in a separate parsing phase {es-pull}113027[#113027] (issue: {es-issue}112374[#112374]) +* Two empty mappings now are created equally {es-pull}107936[#107936] (issue: {es-issue}107031[#107031]) + +Ranking:: +* Fix MLTQuery handling of custom term frequencies {es-pull}110846[#110846] +* Fix RRF validation for `rank_constant` < 1 {es-pull}112058[#112058] +* Fix score count validation in reranker response {es-pull}111212[#111212] (issue: {es-issue}111202[#111202]) + +Search:: +* Allow for querries on `_tier` to skip shards in the `can_match` phase {es-pull}114990[#114990] (issue: {es-issue}114910[#114910]) +* Allow out of range term queries for numeric types {es-pull}112916[#112916] +* Do not exclude empty arrays or empty objects in source filtering {es-pull}112250[#112250] (issue: {es-issue}109668[#109668]) +* Fix synthetic source handling for `bit` type in `dense_vector` field {es-pull}114407[#114407] (issue: {es-issue}114402[#114402]) +* Improve DateTime error handling and add some bad date tests {es-pull}112723[#112723] (issue: {es-issue}112190[#112190]) +* Improve date expression/remote handling in index names {es-pull}112405[#112405] (issue: {es-issue}112243[#112243]) +* Make "too many clauses" throw IllegalArgumentException to avoid 500s {es-pull}112678[#112678] (issue: {es-issue}112177[#112177]) +* Make empty string searches be consistent with case (in)sensitivity {es-pull}110833[#110833] +* Prevent flattening of ordered and unordered interval sources {es-pull}114234[#114234] +* Remove needless forking to GENERIC in `TransportMultiSearchAction` {es-pull}110796[#110796] +* Search/Mapping: KnnVectorQueryBuilder support for allowUnmappedFields {es-pull}107047[#107047] (issue: {es-issue}106846[#106846]) +* Span term query to convert to match no docs when unmapped field is targeted {es-pull}113251[#113251] +* Speedup `CanMatchPreFilterSearchPhase` constructor {es-pull}110860[#110860] +* Update `BlobCacheBufferedIndexInput::readVLong` to correctly handle negative long values {es-pull}115594[#115594] +* [8.x] Limit the number of tasks that a single search can submit {es-pull}115932[#115932] + +Security:: +* Add ECK Role Mapping Cleanup {es-pull}115823[#115823] +* Updated the transport CA name in Security Auto-Configuration. {es-pull}106520[#106520] (issue: {es-issue}106455[#106455]) + +Snapshot/Restore:: +* Handle status code 0 in S3 CMU response {es-pull}116212[#116212] (issue: {es-issue}102294[#102294]) + +TSDB:: +* Implement `parseBytesRef` for `TimeSeriesRoutingHashFieldType` {es-pull}113373[#113373] (issue: {es-issue}112399[#112399]) + +Task Management:: +* Improve handling of failure to create persistent task {es-pull}114386[#114386] + +Transform:: +* Allow task canceling of validate API calls {es-pull}110951[#110951] +* Include reason when no nodes are found {es-pull}112409[#112409] (issue: {es-issue}112404[#112404]) + +Vector Search:: +* Fix dim validation for bit `element_type` {es-pull}114533[#114533] +* Support semantic_text in object fields {es-pull}114601[#114601] (issue: {es-issue}114401[#114401]) + +Watcher:: +* Truncating watcher history if it is too large {es-pull}111245[#111245] (issue: {es-issue}94745[#94745]) + +[[deprecation-8.16.0]] +[float] +=== Deprecations + +Analysis:: +* Deprecate dutch_kp and lovins stemmer as they are removed in Lucene 10 {es-pull}113143[#113143] +* deprecate `edge_ngram` side parameter {es-pull}110829[#110829] + +CRUD:: +* Deprecate dot-prefixed indices and composable template index patterns {es-pull}112571[#112571] + +Search:: +* Adding deprecation warnings for rrf using rank and `sub_searches` {es-pull}114854[#114854] +* Deprecate legacy params from range query {es-pull}113286[#113286] + +[[enhancement-8.16.0]] +[float] +=== Enhancements + +Aggregations:: +* Account for `DelayedBucket` before reduction {es-pull}113013[#113013] +* Add protection for OOM during aggregations partial reduction {es-pull}110520[#110520] +* Deduplicate `BucketOrder` when deserializing {es-pull}112707[#112707] +* Lower the memory footprint when creating `DelayedBucket` {es-pull}112519[#112519] +* Reduce heap usage for `AggregatorsReducer` {es-pull}112874[#112874] +* Remove reduce and `reduceContext` from `DelayedBucket` {es-pull}112547[#112547] + +Allocation:: +* Add link to flood-stage watermark exception message {es-pull}111315[#111315] +* Always allow rebalancing by default {es-pull}111015[#111015] + +Application:: +* [Profiling] add `container.id` field to event index template {es-pull}111969[#111969] + +Authorization:: +* Add manage roles privilege {es-pull}110633[#110633] +* Add privileges required for CDR misconfiguration features to work on AWS SecurityHub integration {es-pull}112574[#112574] + +Codec:: +* Remove zstd feature flag for index codec best compression {es-pull}112665[#112665] +* [8.x] Remove zstd feature flag for index codec best compression {es-pull}112857[#112857] + +Data streams:: +* Add 'verbose' flag retrieving `maximum_timestamp` for get data stream API {es-pull}112303[#112303] +* Display effective retention in the relevant data stream APIs {es-pull}112019[#112019] +* Expose global retention settings via data stream lifecycle API {es-pull}112210[#112210] +* Ignore warning on yaml test put template {es-pull}116201[#116201] (issue: {es-issue}116158[#116158]) +* Make ecs@mappings work with OTel attributes {es-pull}111600[#111600] + +Distributed:: +* Add link to Max Shards Per Node exception message {es-pull}110993[#110993] + +ES|QL:: +* Add EXP ES|QL function {es-pull}110879[#110879] +* Delay construction of warnings {es-pull}114368[#114368] +* Add `CircuitBreaker` to TDigest, Step 3: Connect with ESQL CB {es-pull}113387[#113387] +* Add `CircuitBreaker` to TDigest, Step 4: Take into account shallow classes size {es-pull}113613[#113613] (issue: {es-issue}113916[#113916]) +* Collect and display execution metadata for ES|QL cross cluster searches {es-pull}112595[#112595] (issue: {es-issue}112402[#112402]) +* Add support for multivalue fields in Arrow output {es-pull}114774[#114774] +* BUCKET: allow numerical spans as whole numbers {es-pull}111874[#111874] (issues: {es-issue}104646[#104646], {es-issue}109340[#109340], {es-issue}105375[#105375]) +* Have BUCKET generate friendlier intervals {es-pull}111879[#111879] (issue: {es-issue}110916[#110916]) +* Profile more timing information {es-pull}111855[#111855] +* Push down filters even in case of renames in Evals {es-pull}114411[#114411] +* Speed up CASE for some parameters {es-pull}112295[#112295] +* Speed up grouping by bytes {es-pull}114021[#114021] +* Use less memory in listener {es-pull}114358[#114358] +* Add support for cached strings in plan serialization {es-pull}112929[#112929] +* Add Telemetry API and track top functions {es-pull}111226[#111226] +* Enhance SORT push-down to Lucene to cover references to fields and ST_DISTANCE function {es-pull}112938[#112938] (issue: {es-issue}109973[#109973]) +* Siem ea 9521 improve test {es-pull}111552[#111552] +* Support multi-valued fields in compute engine for ST_DISTANCE {es-pull}114836[#114836] (issue: {es-issue}112910[#112910]) +* Add `SPACE` function {es-pull}112350[#112350] +* Add finish() elapsed time to aggregation profiling times {es-pull}113172[#113172] (issue: {es-issue}112950[#112950]) +* Make query wrapped by `SingleValueQuery` cacheable {es-pull}110116[#110116] +* Add hypot function {es-pull}114382[#114382] +* Cast mixed numeric types to a common numeric type for Coalesce and In at Analyzer {es-pull}111917[#111917] (issue: {es-issue}111486[#111486]) +* Combine Disjunctive CIDRMatch {es-pull}111501[#111501] (issue: {es-issue}105143[#105143]) +* Create `Range` in `PushFiltersToSource` for qualified pushable filters on the same field {es-pull}111437[#111437] +* Name parameter with leading underscore {es-pull}111950[#111950] (issue: {es-issue}111821[#111821]) +* Named parameter for field names and field name patterns {es-pull}112905[#112905] +* Validate index name in parser {es-pull}112081[#112081] +* Add reverse function {es-pull}113297[#113297] +* Explicit cast a string literal to `date_period` and `time_duration` in arithmetic operations {es-pull}109193[#109193] + +Experiences:: +* Integrate IBM watsonx to Inference API for text embeddings {es-pull}111770[#111770] + +Geo:: +* Add support for spatial relationships in point field mapper {es-pull}112126[#112126] +* Small performance improvement in h3 library {es-pull}113385[#113385] +* Support docvalues only query in shape field {es-pull}112199[#112199] + +Health:: +* (API) Cluster Health report `unassigned_primary_shards` {es-pull}112024[#112024] +* Do not treat replica as unassigned if primary recently created and unassigned time is below a threshold {es-pull}112066[#112066] + +ILM+SLM:: +* ILM: Add `total_shards_per_node` setting to searchable snapshot {es-pull}112972[#112972] (issue: {es-issue}112261[#112261]) +* PUT slm policy should only increase version if actually changed {es-pull}111079[#111079] +* Preserve Step Info Across ILM Auto Retries {es-pull}113187[#113187] +* Register SLM run before snapshotting to save stats {es-pull}110216[#110216] +* SLM interval schedule followup - add back `getFieldName` style getters {es-pull}112123[#112123] + +Infra/Core:: +* Add nanos support to `ZonedDateTime` serialization {es-pull}111689[#111689] (issue: {es-issue}68292[#68292]) +* Extend logging for dropped warning headers {es-pull}111624[#111624] (issue: {es-issue}90527[#90527]) +* Give the kibana system user permission to read security entities {es-pull}114363[#114363] + +Infra/Metrics:: +* Add `TaskManager` to `pluginServices` {es-pull}112687[#112687] + +Infra/REST API:: +* Optimize the loop processing of URL decoding {es-pull}110237[#110237] (issue: {es-issue}110235[#110235]) + +Infra/Scripting:: +* Expose `HexFormat` in Painless {es-pull}112412[#112412] + +Infra/Settings:: +* Improve exception message for bad environment variable placeholders in settings {es-pull}114552[#114552] (issue: {es-issue}110858[#110858]) +* Reprocess operator file settings when settings service starts, due to node restart or master node change {es-pull}114295[#114295] + +Ingest Node:: +* Add `size_in_bytes` to enrich cache stats {es-pull}110578[#110578] +* Add support for templates when validating mappings in the simulate ingest API {es-pull}111161[#111161] +* Adding `index_template_substitutions` to the simulate ingest API {es-pull}114128[#114128] +* Adding component template substitutions to the simulate ingest API {es-pull}113276[#113276] +* Adding mapping validation to the simulate ingest API {es-pull}110606[#110606] +* Adds example plugin for custom ingest processor {es-pull}112282[#112282] (issue: {es-issue}111539[#111539]) +* Fix unnecessary mustache template evaluation {es-pull}110986[#110986] (issue: {es-issue}110191[#110191]) +* Listing all available databases in the _ingest/geoip/database API {es-pull}113498[#113498] +* Make enrich cache based on memory usage {es-pull}111412[#111412] (issue: {es-issue}106081[#106081]) +* Tag redacted document in ingest metadata {es-pull}113552[#113552] +* Verify Maxmind database types in the geoip processor {es-pull}114527[#114527] + +Logs:: +* Add validation for synthetic source mode in logs mode indices {es-pull}110677[#110677] +* Store original source for keywords using a normalizer {es-pull}112151[#112151] + +Machine Learning:: +* Add Completion Inference API for Alibaba Cloud AI Search Model {es-pull}112512[#112512] +* Add Streaming Inference spec {es-pull}113812[#113812] +* Add chunking settings configuration to `CohereService,` `AmazonBedrockService,` and `AzureOpenAiService` {es-pull}113897[#113897] +* Add chunking settings configuration to `ElasticsearchService/ELSER` {es-pull}114429[#114429] +* Add custom rule parameters to force time shift {es-pull}110974[#110974] +* Adding chunking settings to `GoogleVertexAiService,` `AzureAiStudioService,` and `AlibabaCloudSearchService` {es-pull}113981[#113981] +* Adding chunking settings to `MistralService,` `GoogleAiStudioService,` and `HuggingFaceService` {es-pull}113623[#113623] +* Adds a new Inference API for streaming responses back to the user. {es-pull}113158[#113158] +* Allow users to force a detector to shift time series state by a specific amount {ml-pull}2695[#2695] +* Create `StreamingHttpResultPublisher` {es-pull}112026[#112026] +* Create an ml node inference endpoint referencing an existing model {es-pull}114750[#114750] +* Default inference endpoint for ELSER {es-pull}113873[#113873] +* Default inference endpoint for the multilingual-e5-small model {es-pull}114683[#114683] +* Dynamically get of num allocations {es-pull}114636[#114636] +* Enable OpenAI Streaming {es-pull}113911[#113911] +* Filter empty task settings objects from the API response {es-pull}114389[#114389] +* Migrate Inference to `ChunkedToXContent` {es-pull}111655[#111655] +* Register Task while Streaming {es-pull}112369[#112369] +* Server-Sent Events for Inference response {es-pull}112565[#112565] +* Stream Anthropic Completion {es-pull}114321[#114321] +* Stream Azure Completion {es-pull}114464[#114464] +* Stream Bedrock Completion {es-pull}114732[#114732] +* Stream Cohere Completion {es-pull}114080[#114080] +* Stream Google Completion {es-pull}114596[#114596] +* Stream OpenAI Completion {es-pull}112677[#112677] +* Support sparse embedding models in the elasticsearch inference service {es-pull}112270[#112270] +* Switch default chunking strategy to sentence {es-pull}114453[#114453] +* Update the Pytorch library to version 2.3.1 {ml-pull}2688[#2688] +* Upgrade to AWS SDK v2 {es-pull}114309[#114309] (issue: {es-issue}110590[#110590]) +* Use the same chunking configurations for models in the Elasticsearch service {es-pull}111336[#111336] +* Validate streaming HTTP Response {es-pull}112481[#112481] +* Wait for allocation on scale up {es-pull}114719[#114719] +* [Inference API] Add Alibaba Cloud AI Search Model support to Inference API {es-pull}111181[#111181] +* [Inference API] Add Docs for AlibabaCloud AI Search Support for the Inference API {es-pull}111181[#111181] +* [Inference API] Introduce Update API to change some aspects of existing inference endpoints {es-pull}114457[#114457] +* [Inference API] Prevent inference endpoints from being deleted if they are referenced by semantic text {es-pull}110399[#110399] +* [Inference API] alibabacloud ai search service support chunk infer to support semantic_text field {es-pull}110399[#110399] + +Mapping:: +* Add Field caps support for Semantic Text {es-pull}111809[#111809] +* Add Lucene segment-level fields stats {es-pull}111123[#111123] +* Add Search Inference ID To Semantic Text Mapping {es-pull}113051[#113051] +* Add object param for keeping synthetic source {es-pull}113690[#113690] +* Add support for multi-value dimensions {es-pull}112645[#112645] (issue: {es-issue}110387[#110387]) +* Allow dimension fields to have multiple values in standard and logsdb index mode {es-pull}112345[#112345] (issues: {es-issue}112232[#112232], {es-issue}112239[#112239]) +* Allow fields with dots in sparse vector field mapper {es-pull}111981[#111981] (issue: {es-issue}109118[#109118]) +* Allow querying `index_mode` {es-pull}110676[#110676] +* Configure keeping source in `FieldMapper` {es-pull}112706[#112706] +* Control storing array source with index setting {es-pull}112397[#112397] +* Introduce mode `subobjects=auto` for objects {es-pull}110524[#110524] +* Update `semantic_text` field to support indexing numeric and boolean data types {es-pull}111284[#111284] +* Use fallback synthetic source for `copy_to` and doc_values: false cases {es-pull}112294[#112294] (issues: {es-issue}110753[#110753], {es-issue}110038[#110038], {es-issue}109546[#109546]) + +Network:: +* Add links to network disconnect troubleshooting {es-pull}112330[#112330] + +Ranking:: +* Add timeout and cancellation check to rescore phase {es-pull}115048[#115048] + +Relevance:: +* Add a query rules tester API call {es-pull}114168[#114168] + +Search:: +* Add more `dense_vector` details for cluster stats field stats {es-pull}113607[#113607] +* Add range and regexp Intervals {es-pull}111465[#111465] +* Adding support for `allow_partial_search_results` in PIT {es-pull}111516[#111516] +* Allow incubating Panama Vector in simdvec, and add vectorized `ipByteBin` {es-pull}112933[#112933] +* Avoid using concurrent collector manager in `LuceneChangesSnapshot` {es-pull}113816[#113816] +* Bool query early termination should also consider `must_not` clauses {es-pull}115031[#115031] +* Deduplicate Kuromoji User Dictionary {es-pull}112768[#112768] +* Multi term intervals: increase max_expansions {es-pull}112826[#112826] (issue: {es-issue}110491[#110491]) +* Search coordinator uses `event.ingested` in cluster state to do rewrites {es-pull}111523[#111523] +* Update cluster stats for retrievers {es-pull}114109[#114109] + +Security:: +* (logger) change from error to warn for short circuiting user {es-pull}112895[#112895] +* Add asset criticality indices for `kibana_system_user` {es-pull}113588[#113588] +* Add tier preference to security index settings allowlist {es-pull}111818[#111818] +* [Service Account] Add `AutoOps` account {es-pull}111316[#111316] + +Snapshot/Restore:: +* Add `max_multipart_parts` setting to S3 repository {es-pull}113989[#113989] +* Add support for Azure Managed Identity {es-pull}111344[#111344] +* Add telemetry for repository usage {es-pull}112133[#112133] +* Add workaround for missing shard gen blob {es-pull}112337[#112337] +* Clean up dangling S3 multipart uploads {es-pull}111955[#111955] (issues: {es-issue}101169[#101169], {es-issue}44971[#44971]) +* Execute shard snapshot tasks in shard-id order {es-pull}111576[#111576] (issue: {es-issue}108739[#108739]) +* Include account name in Azure settings exceptions {es-pull}111274[#111274] +* Introduce repository integrity verification API {es-pull}112348[#112348] (issue: {es-issue}52622[#52622]) + +Stats:: +* Track search and fetch failure stats {es-pull}113988[#113988] + +TSDB:: +* Add support for boolean dimensions {es-pull}111457[#111457] (issue: {es-issue}111338[#111338]) +* Stop iterating over all fields to extract @timestamp value {es-pull}110603[#110603] (issue: {es-issue}92297[#92297]) +* Support booleans in routing path {es-pull}111445[#111445] + +Vector Search:: +* Dense vector field types updatable for int4 {es-pull}110928[#110928] +* Use native scalar scorer for int8_flat index {es-pull}111071[#111071] + +[[feature-8.16.0]] +[float] +=== New features + +Data streams:: +* Introduce global retention in data stream lifecycle. {es-pull}111972[#111972] +* X-pack/plugin/otel: introduce x-pack-otel plugin {es-pull}111091[#111091] + +ES|QL:: +* Add match function {es-pull}113374[#113374] +* Add `MV_PSERIES_WEIGHTED_SUM` for score calculations used by security solution {es-pull}109017[#109017] +* Add async ID and `is_running` headers to ESQL async query {es-pull}111840[#111840] +* Add boolean support to Max and Min aggs {es-pull}110527[#110527] +* Add boolean support to TOP aggregation {es-pull}110718[#110718] +* Added `mv_percentile` function {es-pull}111749[#111749] (issue: {es-issue}111591[#111591]) +* Introduce per agg filter {es-pull}113735[#113735] +* Strings support for MAX and MIN aggregations {es-pull}111544[#111544] +* Support IP fields in MAX and MIN aggregations {es-pull}110921[#110921] +* TOP aggregation IP support {es-pull}111105[#111105] +* TOP support for strings {es-pull}113183[#113183] (issue: {es-issue}109849[#109849]) +* `mv_median_absolute_deviation` function {es-pull}112055[#112055] (issue: {es-issue}111590[#111590]) +* Add MATCH operator {es-pull}110971[#110971] + +ILM+SLM:: +* SLM Interval based scheduling {es-pull}110847[#110847] + +Inference:: +* EIS integration {es-pull}111154[#111154] + +Ingest Node:: +* Add a `terminate` ingest processor {es-pull}114157[#114157] (issue: {es-issue}110218[#110218]) + +Machine Learning:: +* Inference autoscaling {es-pull}109667[#109667] +* Telemetry for inference adaptive allocations {es-pull}110630[#110630] + +Relevance:: +* [Query rules] Add `exclude` query rule type {es-pull}111420[#111420] + +Search:: +* Async search: Add ID and "is running" http headers {es-pull}112431[#112431] (issue: {es-issue}109576[#109576]) +* Cross-cluster search telemetry {es-pull}113825[#113825] + +Vector Search:: +* Adding new bbq index types behind a feature flag {es-pull}114439[#114439] + +[[upgrade-8.16.0]] +[float] +=== Upgrades + +Infra/Core:: +* Upgrade xcontent to Jackson 2.17.0 {es-pull}111948[#111948] +* Upgrade xcontent to Jackson 2.17.2 {es-pull}112320[#112320] + +Infra/Metrics:: +* Update APM Java Agent to support JDK 23 {es-pull}115194[#115194] (issues: {es-issue}115101[#115101], {es-issue}115100[#115100]) + +Search:: +* Upgrade to Lucene 9.12 {es-pull}113333[#113333] + +Snapshot/Restore:: +* Upgrade Azure SDK {es-pull}111225[#111225] +* Upgrade `repository-azure` dependencies {es-pull}112277[#112277] + diff --git a/docs/reference/release-notes/8.16.1.asciidoc b/docs/reference/release-notes/8.16.1.asciidoc new file mode 100644 index 0000000000000..4454b206c5d2f --- /dev/null +++ b/docs/reference/release-notes/8.16.1.asciidoc @@ -0,0 +1,50 @@ +[[release-notes-8.16.1]] +== {es} version 8.16.1 + +Also see <>. + +[[bug-8.16.1]] +[float] +=== Bug fixes + +Data streams:: +* Add missing header in `put_data_lifecycle` rest-api-spec {es-pull}116292[#116292] + +Mapping:: +* Ignore conflicting fields during dynamic mapping update {es-pull}114227[#114227] (issue: {es-issue}114228[#114228]) + +Ranking:: +* Propagating nested `inner_hits` to the parent compound retriever {es-pull}116408[#116408] (issue: {es-issue}116397[#116397]) + +Relevance:: +* Fix handling of bulk requests with semantic text fields and delete ops {es-pull}116942[#116942] + +Search:: +* Fix handling of time exceeded exception in fetch phase {es-pull}116676[#116676] +* Semantic text simple partial update {es-pull}116478[#116478] +* Validate missing shards after the coordinator rewrite {es-pull}116382[#116382] + +Vector Search:: +* Update Semantic Query To Handle Zero Size Responses {es-pull}116277[#116277] (issue: {es-issue}116083[#116083]) + +[[enhancement-8.16.1]] +[float] +=== Enhancements + +Data streams:: +* Apm-data: disable date_detection for all apm data streams {es-pull}116995[#116995] + +Ingest Node:: +* Add postal_code support to the City and Enterprise databases {es-pull}114193[#114193] +* Add support for registered country fields for maxmind geoip databases {es-pull}114521[#114521] +* Support IPinfo database configurations {es-pull}114548[#114548] +* Support more maxmind fields in the geoip processor {es-pull}114268[#114268] + +Relevance:: +* Add tracking for query rule types {es-pull}116357[#116357] + +Snapshot/Restore:: +* Improve message about insecure S3 settings {es-pull}116915[#116915] +* Split searchable snapshot into multiple repo operations {es-pull}116918[#116918] + + diff --git a/docs/reference/release-notes/8.17.0.asciidoc b/docs/reference/release-notes/8.17.0.asciidoc new file mode 100644 index 0000000000000..2c888d78d4c6b --- /dev/null +++ b/docs/reference/release-notes/8.17.0.asciidoc @@ -0,0 +1,209 @@ +[[release-notes-8.17.0]] +== {es} version 8.17.0 + +[IMPORTANT] +==== +The 8.17.0 release contains a fix for a potential security vulnerability. See our https://discuss.elastic.co/c/announcements/security-announcements/31[security advisory] for more details. +==== + +Also see <>. + +[[license-8.17.0]] +[float] +=== License changes + +[float] +==== Change to synthetic `_source` licensing + +Starting with this release, the <> feature is available exclusively with the Enterprise subscription. Synthetic `_source` is used in logs data streams (`logsdb` index mode), time series data streams (TSDS, using `time_series` index mode), application performance monitoring (APM), and Universal Profiling. + +If you are using these capabilities and are not on an Enterprise license, the change will result in increased storage requirements for new data, as the synthetic `_source` setting will be ignored. Existing indices that used synthetic `_source` will remain seamlessly accessible. + +Refer to the subscription page for https://www.elastic.co/subscriptions/cloud[Elastic Cloud] and {subscriptions}[Elastic Stack/self-managed] for the breakdown of available features and their associated subscription tiers. For further details and subscription options, contact your Elastic sales representative or https://www.elastic.co/contact[contact us]. + +[[bug-8.17.0]] +[float] +=== Bug fixes + +Analysis:: +* Adjust analyze limit exception to be a `bad_request` {es-pull}116325[#116325] + +CCS:: +* Fix long metric deserialize & add - auto-resize needs to be set manually {es-pull}117105[#117105] (issue: {es-issue}116914[#116914]) + +CRUD:: +* Standardize error code when bulk body is invalid {es-pull}114869[#114869] + +Data streams:: +* Acquire stats searcher for data stream stats {es-pull}117953[#117953] + +EQL:: +* Don't use a `BytesStreamOutput` to copy keys in `BytesRefBlockHash` {es-pull}114819[#114819] (issue: {es-issue}114599[#114599]) + +ES|QL:: +* Added stricter range type checks and runtime warnings for ENRICH {es-pull}115091[#115091] (issues: {es-issue}107357[#107357], {es-issue}116799[#116799]) +* Don't return TEXT type for functions that take TEXT {es-pull}114334[#114334] (issues: {es-issue}111537[#111537], {es-issue}114333[#114333]) +* ESQL: Fix sorts containing `_source` {es-pull}116980[#116980] (issue: {es-issue}116659[#116659]) +* ES|QL: Fix stats by constant expression {es-pull}114899[#114899] +* Fix BWC for ES|QL cluster request {es-pull}117865[#117865] +* Fix CCS exchange when multi cluster aliases point to same cluster {es-pull}117297[#117297] +* Fix COUNT filter pushdown {es-pull}117503[#117503] (issue: {es-issue}115522[#115522]) +* Fix NPE in `EnrichLookupService` on mixed clusters with <8.14 versions {es-pull}116583[#116583] (issues: {es-issue}116529[#116529], {es-issue}116544[#116544]) +* Fix stats by constant expresson with alias {es-pull}117551[#117551] +* Fix validation of SORT by aggregate functions {es-pull}117316[#117316] +* Fixing remote ENRICH by pushing the Enrich inside `FragmentExec` {es-pull}114665[#114665] (issue: {es-issue}105095[#105095]) +* Ignore cancellation exceptions {es-pull}117657[#117657] +* Limit size of `Literal#toString` {es-pull}117842[#117842] +* Opt into extra data stream resolution {es-pull}118378[#118378] +* Use `SearchStats` instead of field.isAggregatable in data node planning {es-pull}115744[#115744] (issue: {es-issue}115737[#115737]) +* [ESQL] Fix Binary Comparisons on Date Nanos {es-pull}116346[#116346] +* [ES|QL] To_DatePeriod and To_TimeDuration return better error messages on `union_type` fields {es-pull}114934[#114934] + +Infra/CLI:: +* Fix NPE on plugin sync {es-pull}115640[#115640] (issue: {es-issue}114818[#114818]) + +Ingest Node:: +* Fix enrich cache size setting name {es-pull}117575[#117575] +* Fix log message format bugs {es-pull}118354[#118354] +* Fix reconstituting version string from components {es-pull}117213[#117213] (issue: {es-issue}116950[#116950]) +* Reducing error-level stack trace logging for normal events in `GeoIpDownloader` {es-pull}114924[#114924] + +License:: +* Distinguish `LicensedFeature` by family field {es-pull}116809[#116809] + +Logs:: +* Prohibit changes to index mode, source, and sort settings during resize {es-pull}115812[#115812] + +Machine Learning:: +* Fix deberta tokenizer bug caused by bug in normalizer {es-pull}117189[#117189] +* Fix for Deberta tokenizer when input sequence exceeds 512 tokens {es-pull}117595[#117595] +* Hides `hugging_face_elser` service from the `GET _inference/_services API` {es-pull}116664[#116664] (issue: {es-issue}116644[#116644]) +* Mitigate IOSession timeouts {es-pull}115414[#115414] (issues: {es-issue}114385[#114385], {es-issue}114327[#114327], {es-issue}114105[#114105], {es-issue}114232[#114232]) +* Propagate scoring function through random sampler {es-pull}116957[#116957] (issue: {es-issue}110134[#110134]) +* Wait for the worker service to shutdown before closing task processor {es-pull}117920[#117920] (issue: {es-issue}117563[#117563]) + +Mapping:: +* Address mapping and compute engine runtime field issues {es-pull}117792[#117792] (issue: {es-issue}117644[#117644]) +* Always Emit Inference ID in Semantic Text Mapping {es-pull}117294[#117294] +* Fix concurrency issue with `ReinitializingSourceProvider` {es-pull}118370[#118370] (issue: {es-issue}118238[#118238]) +* Fix false positive date detection with trailing dot {es-pull}116953[#116953] (issue: {es-issue}116946[#116946]) +* Parse the contents of dynamic objects for [subobjects:false] {es-pull}117762[#117762] (issue: {es-issue}117544[#117544]) + +Network:: +* Use underlying `ByteBuf` `refCount` for `ReleasableBytesReference` {es-pull}116211[#116211] + +Ranking:: +* Fix for propagating filters from compound to inner retrievers {es-pull}117914[#117914] + +Search:: +* Add missing `async_search` query parameters to rest-api-spec {es-pull}117312[#117312] +* Don't skip shards in coord rewrite if timestamp is an alias {es-pull}117271[#117271] +* Fields caps does not honour ignore_unavailable {es-pull}116021[#116021] (issue: {es-issue}107767[#107767]) +* _validate does not honour ignore_unavailable {es-pull}116656[#116656] (issue: {es-issue}116594[#116594]) + +Vector Search:: +* Correct bit * byte and bit * float script comparisons {es-pull}117404[#117404] + +Watcher:: +* Watch Next Run Interval Resets On Shard Move or Node Restart {es-pull}115102[#115102] (issue: {es-issue}111433[#111433]) + +[[deprecation-8.17.0]] +[float] +=== Deprecations + +Infra/REST API:: +* Add a basic deprecation warning that the JSON format for non-detailed error responses is changing in v9 {es-pull}114739[#114739] (issue: {es-issue}89387[#89387]) + +Mapping:: +* Deprecate `_source.mode` in mappings {es-pull}116689[#116689] + +[[enhancement-8.17.0]] +[float] +=== Enhancements + +Authorization:: +* Add a `monitor_stats` privilege and allow that privilege for remote cluster privileges {es-pull}114964[#114964] + +Data streams:: +* Adding a deprecation info API warning for data streams with old indices {es-pull}116447[#116447] + +ES|QL:: +* Add ES|QL `bit_length` function {es-pull}115792[#115792] +* ESQL: Honor skip_unavailable setting for nonmatching indices errors at planning time {es-pull}116348[#116348] (issue: {es-issue}114531[#114531]) +* ESQL: Remove parent from `FieldAttribute` {es-pull}112881[#112881] +* ESQL: extract common filter from aggs {es-pull}115678[#115678] +* ESQL: optimise aggregations filtered by false/null into evals {es-pull}115858[#115858] +* ES|QL CCS uses `skip_unavailable` setting for handling disconnected remote clusters {es-pull}115266[#115266] (issue: {es-issue}114531[#114531]) +* ES|QL: add metrics for functions {es-pull}114620[#114620] +* Esql Enable Date Nanos (tech preview) {es-pull}117080[#117080] +* [ES|QL] Implicit casting string literal to intervals {es-pull}115814[#115814] (issue: {es-issue}115352[#115352]) + +Indices APIs:: +* Ensure class resource stream is closed in `ResourceUtils` {es-pull}116437[#116437] + +Inference:: +* Add version prefix to Inference Service API path {es-pull}117366[#117366] +* Update sparse text embeddings API route for Inference Service {es-pull}118368[#118368] + +Infra/Core:: +* Support for unsigned 64 bit numbers in Cpu stats {es-pull}114681[#114681] (issue: {es-issue}112274[#112274]) + +Ingest Node:: +* Adding support for additional mapping to simulate ingest API {es-pull}114742[#114742] +* Adding support for simulate ingest mapping adddition for indices with mappings that do not come from templates {es-pull}115359[#115359] + +Logs:: +* Add logsdb telemetry {es-pull}115994[#115994] +* Add num docs and size to logsdb telemetry {es-pull}116128[#116128] +* Feature: re-structure document ID generation favoring _id inverted index compression {es-pull}104683[#104683] + +Machine Learning:: +* Add special case for elastic reranker in inference API {es-pull}116962[#116962] +* Adding inference endpoint validation for `AzureAiStudioService` {es-pull}113713[#113713] +* Adds support for `input_type` field to Vertex inference service {es-pull}116431[#116431] +* Enable built-in Inference Endpoints and default for Semantic Text {es-pull}116931[#116931] +* Increase default `queue_capacity` to 10_000 and decrease max `queue_capacity` to 100_000 {es-pull}115041[#115041] +* [Inference API] Add API to get configuration of inference services {es-pull}114862[#114862] +* [Inference API] Improve chunked results error message {es-pull}115807[#115807] + +Recovery:: +* Attempt to clean up index before remote transfer {es-pull}115142[#115142] (issue: {es-issue}104473[#104473]) + +Relevance:: +* Add query rules retriever {es-pull}114855[#114855] + +Search:: +* Add Search Phase APM metrics {es-pull}113194[#113194] +* Add `docvalue_fields` Support for `dense_vector` Fields {es-pull}114484[#114484] (issue: {es-issue}108470[#108470]) +* Add initial support for `semantic_text` field type {es-pull}113920[#113920] +* Adds access to flags no_sub_matches and no_overlapping_matches to hyphenation-decompounder-tokenfilter {es-pull}115459[#115459] (issue: {es-issue}97849[#97849]) +* Better sizing `BytesRef` for Strings in Queries {es-pull}115655[#115655] +* Enable `_tier` based coordinator rewrites for all indices (not just mounted indices) {es-pull}115797[#115797] + +Vector Search:: +* Add support for bitwise inner-product in painless {es-pull}116082[#116082] +* Improve halfbyte transposition performance, marginally improving bbq performance {es-pull}117350[#117350] + +[[feature-8.17.0]] +[float] +=== New features + +Data streams:: +* Add default ILM policies and switch to ILM for apm-data plugin {es-pull}115687[#115687] + +ES|QL:: +* Add support for `BYTE_LENGTH` scalar function {es-pull}116591[#116591] +* Esql/lookup join grammar {es-pull}116515[#116515] +* Remove snapshot build restriction for match and qstr functions {es-pull}114482[#114482] + +Search:: +* ESQL - Add match operator (:) {es-pull}116819[#116819] + +[[upgrade-8.17.0]] +[float] +=== Upgrades + +Security:: +* Upgrade Bouncy Castle FIPS dependencies {es-pull}112989[#112989] + + diff --git a/docs/reference/release-notes/8.18.0.asciidoc b/docs/reference/release-notes/8.18.0.asciidoc new file mode 100644 index 0000000000000..332edfbc23eb7 --- /dev/null +++ b/docs/reference/release-notes/8.18.0.asciidoc @@ -0,0 +1,8 @@ +[[release-notes-8.18.0]] +== {es} version 8.18.0 + +coming[8.18.0] + +Also see <>. + + diff --git a/docs/reference/release-notes/highlights.asciidoc b/docs/reference/release-notes/highlights.asciidoc index 007dd740f34cf..c82fd66795a81 100644 --- a/docs/reference/release-notes/highlights.asciidoc +++ b/docs/reference/release-notes/highlights.asciidoc @@ -11,7 +11,9 @@ For detailed information about this release, see the <> and // Add previous release to the list Other versions: -{ref-bare}/8.15/release-highlights.html[8.15] +{ref-bare}/8.17/release-highlights.html[8.17] +| {ref-bare}/8.16/release-highlights.html[8.16] +| {ref-bare}/8.15/release-highlights.html[8.15] | {ref-bare}/8.14/release-highlights.html[8.14] | {ref-bare}/8.13/release-highlights.html[8.13] | {ref-bare}/8.12/release-highlights.html[8.12] @@ -30,195 +32,13 @@ Other versions: endif::[] +// The notable-highlights tag marks entries that +// should be featured in the Stack Installation and Upgrade Guide: // tag::notable-highlights[] - -[discrete] -[[stored_fields_are_compressed_with_zstandard_instead_of_lz4_deflate]] -=== Stored fields are now compressed with ZStandard instead of LZ4/DEFLATE -Stored fields are now compressed by splitting documents into blocks, which -are then compressed independently with ZStandard. `index.codec: default` -(default) uses blocks of at most 14kB or 128 documents compressed with level -0, while `index.codec: best_compression` uses blocks of at most 240kB or -2048 documents compressed at level 3. On most datasets that we tested -against, this yielded storage improvements in the order of 10%, slightly -faster indexing and similar retrieval latencies. - -{es-pull}103374[#103374] - -[discrete] -[[stricter_failure_handling_in_multi_repo_get_snapshots_request_handling]] -=== Stricter failure handling in multi-repo get-snapshots request handling -If a multi-repo get-snapshots request encounters a failure in one of the -targeted repositories then earlier versions of Elasticsearch would proceed -as if the faulty repository did not exist, except for a per-repository -failure report in a separate section of the response body. This makes it -impossible to paginate the results properly in the presence of failures. In -versions 8.15.0 and later this API's failure handling behaviour has been -made stricter, reporting an overall failure if any targeted repository's -contents cannot be listed. - -{es-pull}107191[#107191] - -[discrete] -[[add_new_int4_quantization_to_dense_vector]] -=== Add new int4 quantization to dense_vector -New int4 (half-byte) scalar quantization support via two knew index types: `int4_hnsw` and `int4_flat`. -This gives an 8x reduction from `float32` with some accuracy loss. In addition to less memory required, this -improves query and merge speed significantly when compared to raw vectors. - -{es-pull}109317[#109317] - -[discrete] -[[esql_inlinestats]] -=== ESQL: INLINESTATS -This adds the `INLINESTATS` command to ESQL which performs a STATS and -then enriches the results into the output stream. So, this query: - -[source,esql] ----- -FROM test -| INLINESTATS m=MAX(a * b) BY b -| WHERE m == a * b -| SORT a DESC, b DESC -| LIMIT 3 ----- - -Produces output like: - -| a | b | m | -| --- | --- | ----- | -| 99 | 999 | 98901 | -| 99 | 998 | 98802 | -| 99 | 997 | 98703 | - -{es-pull}109583[#109583] - -[discrete] -[[mark_query_rules_as_ga]] -=== Mark Query Rules as GA -This PR marks query rules as Generally Available. All APIs are no longer -in tech preview. - -{es-pull}110004[#110004] - -[discrete] -[[adds_new_bit_element_type_for_dense_vectors]] -=== Adds new `bit` `element_type` for `dense_vectors` -This adds `bit` vector support by adding `element_type: bit` for -vectors. This new element type works for indexed and non-indexed -vectors. Additionally, it works with `hnsw` and `flat` index types. No -quantization based codec works with this element type, this is -consistent with `byte` vectors. - -`bit` vectors accept up to `32768` dimensions in size and expect vectors -that are being indexed to be encoded either as a hexidecimal string or a -`byte[]` array where each element of the `byte` array represents `8` -bits of the vector. - -`bit` vectors support script usage and regular query usage. When -indexed, all comparisons done are `xor` and `popcount` summations (aka, -hamming distance), and the scores are transformed and normalized given -the vector dimensions. - -For scripts, `l1norm` is the same as `hamming` distance and `l2norm` is -`sqrt(l1norm)`. `dotProduct` and `cosineSimilarity` are not supported. - -Note, the dimensions expected by this element_type are always to be -divisible by `8`, and the `byte[]` vectors provided for index must be -have size `dim/8` size, where each byte element represents `8` bits of -the vectors. - -{es-pull}110059[#110059] - -[discrete] -[[redact_processor_generally_available]] -=== The Redact processor is Generally Available -The Redact processor uses the Grok rules engine to obscure text in the input document matching the given Grok patterns. The Redact processor was initially released as Technical Preview in `8.7.0`, and is now released as Generally Available. - -{es-pull}110395[#110395] - -[discrete] -[[always_allow_rebalancing_by_default]] -=== Always allow rebalancing by default -In earlier versions of {es} the `cluster.routing.allocation.allow_rebalance` setting defaults to -`indices_all_active` which blocks all rebalancing moves while the cluster is in `yellow` or `red` health. This was -appropriate for the legacy allocator which might do too many rebalancing moves otherwise. Today's allocator has -better support for rebalancing a cluster that is not in `green` health, and expects to be able to rebalance some -shards away from over-full nodes to avoid allocating shards to undesirable locations in the first place. From -version 8.16 `allow_rebalance` setting defaults to `always` unless the legacy allocator is explicitly enabled. - -{es-pull}111015[#111015] - +// [discrete] +// === Heading +// +// Description. // end::notable-highlights[] -[discrete] -[[new_custom_parser_for_iso_8601_datetimes]] -=== New custom parser for ISO-8601 datetimes -This introduces a new custom parser for ISO-8601 datetimes, for the `iso8601`, `strict_date_optional_time`, and -`strict_date_optional_time_nanos` built-in date formats. This provides a performance improvement over the -default Java date-time parsing. Whilst it maintains much of the same behaviour, -the new parser does not accept nonsensical date-time strings that have multiple fractional seconds fields -or multiple timezone specifiers. If the new parser fails to parse a string, it will then use the previous parser -to parse it. If a large proportion of the input data consists of these invalid strings, this may cause -a small performance degradation. If you wish to force the use of the old parsers regardless, -set the JVM property `es.datetime.java_time_parsers=true` on all ES nodes. - -{es-pull}106486[#106486] - -[discrete] -[[new_custom_parser_for_more_iso_8601_date_formats]] -=== New custom parser for more ISO-8601 date formats -Following on from #106486, this extends the custom ISO-8601 datetime parser to cover the `strict_year`, -`strict_year_month`, `strict_date_time`, `strict_date_time_no_millis`, `strict_date_hour_minute_second`, -`strict_date_hour_minute_second_millis`, and `strict_date_hour_minute_second_fraction` date formats. -As before, the parser will use the existing java.time parser if there are parsing issues, and the -`es.datetime.java_time_parsers=true` JVM property will force the use of the old parsers regardless. - -{es-pull}108606[#108606] - -[discrete] -[[preview_support_for_connection_type_domain_isp_databases_in_geoip_processor]] -=== Preview: Support for the 'Connection Type, 'Domain', and 'ISP' databases in the geoip processor -As a Technical Preview, the {ref}/geoip-processor.html[`geoip`] processor can now use the commercial -https://dev.maxmind.com/geoip/docs/databases/connection-type[GeoIP2 'Connection Type'], -https://dev.maxmind.com/geoip/docs/databases/domain[GeoIP2 'Domain'], -and -https://dev.maxmind.com/geoip/docs/databases/isp[GeoIP2 'ISP'] -databases from MaxMind. - -{es-pull}108683[#108683] - -[discrete] -[[update_elasticsearch_to_lucene_9_11]] -=== Update Elasticsearch to Lucene 9.11 -Elasticsearch is now updated using the latest Lucene version 9.11. -Here are the full release notes: -But, here are some particular highlights: -- Usage of MADVISE for better memory management: https://github.com/apache/lucene/pull/13196 -- Use RWLock to access LRUQueryCache to reduce contention: https://github.com/apache/lucene/pull/13306 -- Speedup multi-segment HNSW graph search for nested kNN queries: https://github.com/apache/lucene/pull/13121 -- Add a MemorySegment Vector scorer - for scoring without copying on-heap vectors: https://github.com/apache/lucene/pull/13339 - -{es-pull}109219[#109219] - -[discrete] -[[synthetic_source_improvements]] -=== Synthetic `_source` improvements -There are multiple improvements to synthetic `_source` functionality: - -* Synthetic `_source` is now supported for all field types including `nested` and `object`. `object` fields are supported with `enabled` set to `false`. - -* Synthetic `_source` can be enabled together with `ignore_malformed` and `ignore_above` parameters for all field types that support them. - -{es-pull}109501[#109501] - -[discrete] -[[index_sorting_on_indexes_with_nested_fields]] -=== Index sorting on indexes with nested fields -Index sorting is now supported for indexes with mappings containing nested objects. -The index sort spec (as specified by `index.sort.field`) can't contain any nested -fields, still. - -{es-pull}110251[#110251] - diff --git a/docs/reference/repositories-metering-api/apis/clear-repositories-metering-archive.asciidoc b/docs/reference/repositories-metering-api/apis/clear-repositories-metering-archive.asciidoc index addcd65f0e84a..bd05d76705eb9 100644 --- a/docs/reference/repositories-metering-api/apis/clear-repositories-metering-archive.asciidoc +++ b/docs/reference/repositories-metering-api/apis/clear-repositories-metering-archive.asciidoc @@ -5,6 +5,12 @@ Clear repositories metering archive ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/operation/operation-nodes-clear-repositories-metering-archive[Clear the archived repositories metering API]. +-- + Removes the archived repositories metering information present in the cluster. [[clear-repositories-metering-archive-api-request]] diff --git a/docs/reference/repositories-metering-api/apis/get-repositories-metering.asciidoc b/docs/reference/repositories-metering-api/apis/get-repositories-metering.asciidoc index 314f85a7dba52..6b6d98a69ded2 100644 --- a/docs/reference/repositories-metering-api/apis/get-repositories-metering.asciidoc +++ b/docs/reference/repositories-metering-api/apis/get-repositories-metering.asciidoc @@ -5,6 +5,12 @@ Get repositories metering information ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/operation/operation-nodes-get-repositories-metering-info[Get cluster repositories metering API]. +-- + Returns cluster repositories metering information. [[get-repositories-metering-api-request]] diff --git a/docs/reference/repositories-metering-api/repositories-metering-apis.asciidoc b/docs/reference/repositories-metering-api/repositories-metering-apis.asciidoc index b838e0fb213f4..ca9bffa32a917 100644 --- a/docs/reference/repositories-metering-api/repositories-metering-apis.asciidoc +++ b/docs/reference/repositories-metering-api/repositories-metering-apis.asciidoc @@ -4,6 +4,12 @@ experimental[] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-cluster[Cluster APIs]. +-- + You can use the following APIs to retrieve repositories metering information. This is an API used by Elastic's commercial offerings. diff --git a/docs/reference/reranking/index.asciidoc b/docs/reference/reranking/index.asciidoc new file mode 100644 index 0000000000000..3171be7c872d4 --- /dev/null +++ b/docs/reference/reranking/index.asciidoc @@ -0,0 +1,70 @@ +[[re-ranking-overview]] += Re-ranking + +Many search systems are built on multi-stage retrieval pipelines. + +Earlier stages use cheap, fast algorithms to find a broad set of possible matches. + +Later stages use more powerful models, often machine learning-based, to reorder the documents. +This step is called re-ranking. +Because the resource-intensive model is only applied to the smaller set of pre-filtered results, this approach returns more relevant results while still optimizing for search performance and computational costs. + +{es} supports various ranking and re-ranking techniques to optimize search relevance and performance. + +[float] +[[re-ranking-two-stage-pipeline]] +== Two-stage retrieval pipelines + + +[float] +[[re-ranking-first-stage-pipeline]] +=== Initial retrieval + +[float] +[[re-ranking-ranking-overview-bm25]] +==== Full-text search: BM25 scoring + +{es} ranks documents based on term frequency and inverse document frequency, adjusted for document length. +https://en.wikipedia.org/wiki/Okapi_BM25[BM25] is the default statistical scoring algorithm in {es}. + +[float] +[[re-ranking-ranking-overview-vector]] +==== Vector search: similarity scoring + +Vector search involves transforming data into dense or sparse vector embeddings to capture semantic meanings, and computing similarity scores for query vectors. +Store vectors using `semantic_text` fields for automatic inference and vectorization or `dense_vector` and `sparse_vector` fields when you need more control over the underlying embedding model. +Query vector fields with `semantic`, `knn` or `sparse_vector` queries to compute similarity scores. +Refer to <> for more information. + +[float] +[[re-ranking-ranking-overview-hybrid]] +==== Hybrid techniques + +Hybrid search techniques combine results from full-text and vector search pipelines. +{es} enables combining lexical matching (BM25) and vector search scores using the <> algorithm. + +[float] +[[re-ranking-overview-second-stage]] +=== Re-ranking + +When using the following advanced re-ranking pipelines, first-stage retrieval mechanisms effectively generate a set of candidates. +These candidates are funneled into the re-ranker to perform more computationally expensive re-ranking tasks. + +[float] +[[re-ranking-overview-semantic]] +==== Semantic re-ranking + +<> uses machine learning models to reorder search results based on their semantic similarity to a query. +Models can be hosted directly in your {es} cluster, or you can use <> to call models provided by third-party services. +Semantic re-ranking enables out-of-the-box semantic search capabilities on existing full-text search indices. + +[float] +[[re-ranking-overview-ltr]] +==== Learning to Rank (LTR) + +<> is for advanced users. +Learning To Rank involves training a machine learning model to build a ranking function for your search experience that updates over time. +LTR is best suited for when you have ample training data and need highly customized relevance tuning. + +include::semantic-reranking.asciidoc[] +include::learning-to-rank.asciidoc[] diff --git a/docs/reference/search/search-your-data/learning-to-rank-model-training.asciidoc b/docs/reference/reranking/learning-to-rank-model-training.asciidoc similarity index 84% rename from docs/reference/search/search-your-data/learning-to-rank-model-training.asciidoc rename to docs/reference/reranking/learning-to-rank-model-training.asciidoc index 0f4640ebdf347..8e0b3f9ae94ce 100644 --- a/docs/reference/search/search-your-data/learning-to-rank-model-training.asciidoc +++ b/docs/reference/reranking/learning-to-rank-model-training.asciidoc @@ -38,11 +38,21 @@ Feature extractors are defined using templated queries. https://eland.readthedoc from eland.ml.ltr import QueryFeatureExtractor feature_extractors=[ - # We want to use the score of the match query for the title field as a feature: + # We want to use the BM25 score of the match query for the title field as a feature: QueryFeatureExtractor( feature_name="title_bm25", query={"match": {"title": "{{query}}"}} ), + # We want to use the the number of matched terms in the title field as a feature: + QueryFeatureExtractor( + feature_name="title_matched_term_count", + query={ + "script_score": { + "query": {"match": {"title": "{{query}}"}}, + "script": {"source": "return _termStats.matchedTermsCount();"}, + } + }, + ), # We can use a script_score query to get the value # of the field rating directly as a feature: QueryFeatureExtractor( @@ -54,19 +64,13 @@ feature_extractors=[ } }, ), - # We can execute a script on the value of the query - # and use the return value as a feature: - QueryFeatureExtractor( - feature_name="query_length", + # We extract the number of terms in the query as feature. + QueryFeatureExtractor( + feature_name="query_term_count", query={ "script_score": { - "query": {"match_all": {}}, - "script": { - "source": "return params['query'].splitOnToken(' ').length;", - "params": { - "query": "{{query}}", - } - }, + "query": {"match": {"title": "{{query}}"}}, + "script": {"source": "return _termStats.uniqueTermsCount();"}, } }, ), @@ -74,6 +78,15 @@ feature_extractors=[ ---- // NOTCONSOLE +[NOTE] +.Tern statistics as features +=================================================== + +It is very common for an LTR model to leverage raw term statistics as features. +To extract this information, you can use the {ref}/modules-scripting-fields.html#scripting-term-statistics[term statistics feature] provided as part of the <> query. + +=================================================== + Once the feature extractors have been defined, they are wrapped in an `eland.ml.ltr.LTRModelConfig` object for use in later training steps: [source,python] diff --git a/docs/reference/search/search-your-data/learning-to-rank-search-usage.asciidoc b/docs/reference/reranking/learning-to-rank-search-usage.asciidoc similarity index 91% rename from docs/reference/search/search-your-data/learning-to-rank-search-usage.asciidoc rename to docs/reference/reranking/learning-to-rank-search-usage.asciidoc index f14219e24bc11..afb623dc2b1c9 100644 --- a/docs/reference/search/search-your-data/learning-to-rank-search-usage.asciidoc +++ b/docs/reference/reranking/learning-to-rank-search-usage.asciidoc @@ -61,10 +61,3 @@ When exposing pagination to users, `window_size` should remain constant as each ====== Negative scores Depending on how your model is trained, it’s possible that the model will return negative scores for documents. While negative scores are not allowed from first-stage retrieval and ranking, it is possible to use them in the LTR rescorer. - -[discrete] -[[learning-to-rank-rescorer-limitations-term-statistics]] -====== Term statistics as features - -We do not currently support term statistics as features, however future releases will introduce this capability. - diff --git a/docs/reference/search/search-your-data/learning-to-rank.asciidoc b/docs/reference/reranking/learning-to-rank.asciidoc similarity index 99% rename from docs/reference/search/search-your-data/learning-to-rank.asciidoc rename to docs/reference/reranking/learning-to-rank.asciidoc index ebd6d67fe42da..f49f7074d2dbc 100644 --- a/docs/reference/search/search-your-data/learning-to-rank.asciidoc +++ b/docs/reference/reranking/learning-to-rank.asciidoc @@ -131,4 +131,4 @@ In the next pages of this guide you will learn to: * <> include::learning-to-rank-model-training.asciidoc[] -include::learning-to-rank-search-usage.asciidoc[] +include::learning-to-rank-search-usage.asciidoc[] \ No newline at end of file diff --git a/docs/reference/reranking/semantic-reranking.asciidoc b/docs/reference/reranking/semantic-reranking.asciidoc new file mode 100644 index 0000000000000..e1e2abd224a8e --- /dev/null +++ b/docs/reference/reranking/semantic-reranking.asciidoc @@ -0,0 +1,140 @@ +[[semantic-reranking]] +== Semantic re-ranking + +preview::[] + +[TIP] +==== +This overview focuses more on the high-level concepts and use cases for semantic re-ranking. For full implementation details on how to set up and use semantic re-ranking in {es}, see the <> in the Search API docs. +==== + +Re-rankers improve the relevance of results from earlier-stage retrieval mechanisms. +_Semantic_ re-rankers use machine learning models to reorder search results based on their semantic similarity to a query. + +Semantic re-ranking requires relatively large and complex machine learning models and operates in real-time in response to queries. +This technique makes sense on a small _top-k_ result set, as one the of the final steps in a pipeline. +This is a powerful technique for improving search relevance that works equally well with keyword, semantic, or hybrid retrieval algorithms. + +The next sections provide more details on the benefits, use cases, and model types used for semantic re-ranking. +The final sections include a practical, high-level overview of how to implement <> and links to the full reference documentation. + +[discrete] +[[semantic-reranking-use-cases]] +=== Use cases + +Semantic re-ranking enables a variety of use cases: + +* *Lexical (BM25) retrieval results re-ranking* +** Out-of-the-box semantic search by adding a simple API call to any lexical/BM25 retrieval pipeline. +** Adds semantic search capabilities on top of existing indices without reindexing, perfect for quick improvements. +** Ideal for environments with complex existing indices. + +* *Semantic retrieval results re-ranking* +** Improves results from semantic retrievers using ELSER sparse vector embeddings or dense vector embeddings by using more powerful models. +** Adds a refinement layer on top of hybrid retrieval with <>. + +* *General applications* +** Supports automatic and transparent chunking, eliminating the need for pre-chunking at index time. +** Provides explicit control over document relevance in retrieval-augmented generation (RAG) uses cases or other scenarios involving language model (LLM) inputs. + +Now that we've outlined the value of semantic re-ranking, we'll explore the specific models that power this process and how they differ. + +[discrete] +[[semantic-reranking-models]] +=== Cross-encoder and bi-encoder models + +At a high level, two model types are used for semantic re-ranking: cross-encoders and bi-encoders. + +NOTE: In this version, {es} *only supports cross-encoders* for semantic re-ranking. + +* A *cross-encoder model* can be thought of as a more powerful, all-in-one solution, because it generates query-aware document representations. +It takes the query and document texts as a single, concatenated input. +* A *bi-encoder model* takes as input either document or query text. +Documents and query embeddings are computed separately, so they aren't aware of each other. +** To compute a ranking score, an external operation is required. This typically involves computing dot-product or cosine similarity between the query and document embeddings. + +In brief, cross-encoders provide high accuracy but are more resource-intensive. +Bi-encoders are faster and more cost-effective but less precise. + +In future versions, {es} will also support bi-encoders. +If you're interested in a more detailed analysis of the practical differences between cross-encoders and bi-encoders, untoggle the next section. + +.Comparisons between cross-encoder and bi-encoder +[%collapsible] +============== +The following is a non-exhaustive list of considerations when choosing between cross-encoders and bi-encoders for semantic re-ranking: + +* Because a cross-encoder model simultaneously processes both query and document texts, it can better infer their relevance, making it more effective as a reranker than a bi-encoder. +* Cross-encoder models are generally larger and more computationally intensive, resulting in higher latencies and increased computational costs. +* There are significantly fewer open-source cross-encoders, while bi-encoders offer a wide variety of sizes, languages, and other trade-offs. +* The effectiveness of cross-encoders can also improve the relevance of semantic retrievers. +For example, their ability to take word order into account can improve on dense or sparse embedding retrieval. +* When trained in tandem with specific retrievers (like lexical/BM25), cross-encoders can “correct” typical errors made by those retrievers. +* Cross-encoders output scores that are consistent across queries. +This enables you to maintain high relevance in result sets, by setting a minimum score threshold for all queries. +For example, this is important when using results in a RAG workflow or if you're otherwise feeding results to LLMs. +Note that similarity scores from bi-encoders/embedding similarities are _query-dependent_, meaning you cannot set universal cut-offs. +* Bi-encoders rerank using embeddings. You can improve your re-ranking latency by creating embeddings at ingest-time. These embeddings can be stored for re-ranking without being indexed for retrieval, reducing your memory footprint. +============== + +[discrete] +[[semantic-reranking-in-es]] +=== Semantic re-ranking in {es} + +In {es}, semantic re-rankers are implemented using the {es} <> and a <>. + +To use semantic re-ranking in {es}, you need to: + +. *Select and configure a re-ranking model*. +You have the following options: +.. Use the <> cross-encoder model via the inference API's {es} service. +.. Use the <> to create a `rerank` endpoint. +.. Use the <> to create a `rerank` endpoint. +.. Upload a model to {es} from Hugging Face with {eland-docs}/machine-learning.html#ml-nlp-pytorch[Eland]. You'll need to use the `text_similarity` NLP task type when loading the model using Eland. Then set up an <> with the `rerank` endpoint type. ++ +Refer to {ml-docs}/ml-nlp-model-ref.html#ml-nlp-model-ref-text-similarity[the Elastic NLP model reference] for a list of third party text similarity models supported by {es} for semantic re-ranking. + +. *Create a `rerank` endpoint using the <>*. +The Inference API creates an inference endpoint and configures your chosen machine learning model to perform the re-ranking task. +. *Define a `text_similarity_reranker` retriever in your search request*. +The retriever syntax makes it simple to configure both the retrieval and re-ranking of search results in a single API call. + +.*Example search request* with semantic reranker +[%collapsible] +============== +The following example shows a search request that uses a semantic reranker to reorder the top-k documents based on their semantic similarity to the query. +[source,console] +---- +POST _search +{ + "retriever": { + "text_similarity_reranker": { + "retriever": { + "standard": { + "query": { + "match": { + "text": "How often does the moon hide the sun?" + } + } + } + }, + "field": "text", + "inference_id": "elastic-rerank", + "inference_text": "How often does the moon hide the sun?", + "rank_window_size": 100, + "min_score": 0.5 + } + } +} +---- +// TEST[skip:TBD] +============== + +[discrete] +[[semantic-reranking-learn-more]] +=== Learn more + +* Read the <> for syntax and implementation details +* Learn more about the <> abstraction +* Learn more about the Elastic <> +* Check out our https://github.com/elastic/elasticsearch-labs/blob/main/notebooks/integrations/cohere/cohere-elasticsearch.ipynb[Python notebook] for using Cohere with {es} \ No newline at end of file diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc index fabd495cdc525..993bb8cb894f9 100644 --- a/docs/reference/rest-api/common-parms.asciidoc +++ b/docs/reference/rest-api/common-parms.asciidoc @@ -1298,10 +1298,11 @@ tag::wait_for_active_shards[] `wait_for_active_shards`:: + -- -(Optional, string) The number of shard copies that must be active before -proceeding with the operation. Set to `all` or any positive integer up -to the total number of shards in the index (`number_of_replicas+1`). -Default: 1, the primary shard. +(Optional, string) The number of copies of each shard that must be active +before proceeding with the operation. Set to `all` or any non-negative integer +up to the total number of copies of each shard in the index +(`number_of_replicas+1`). Defaults to `1`, meaning to wait just for each +primary shard to be active. See <>. -- diff --git a/docs/reference/rest-api/index.asciidoc b/docs/reference/rest-api/index.asciidoc index 50c9f96ad81b0..9d9047c93cc97 100644 --- a/docs/reference/rest-api/index.asciidoc +++ b/docs/reference/rest-api/index.asciidoc @@ -6,9 +6,9 @@ {es} exposes REST APIs that are used by the UI components and can be called directly to configure and access {es} features. -[NOTE] -We are working on including more {es} APIs in this section. Some content might -not be included yet. +.New API reference +[sidebar] +For the most up-to-date API details, refer to {api-es}[{es} APIs]. * <> * <> diff --git a/docs/reference/rest-api/info.asciidoc b/docs/reference/rest-api/info.asciidoc index 28b6df215a18d..b9547163b07ba 100644 --- a/docs/reference/rest-api/info.asciidoc +++ b/docs/reference/rest-api/info.asciidoc @@ -2,6 +2,12 @@ [[info-api]] == Info API +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-info[Info APIs]. +-- + Provides general information about the installed {xpack} features. [discrete] @@ -172,6 +178,10 @@ Example response: "universal_profiling": { "available": true, "enabled": true + }, + "logsdb": { + "available": true, + "enabled": false } }, "tagline" : "You know, for X" diff --git a/docs/reference/rest-api/logstash/delete-pipeline.asciidoc b/docs/reference/rest-api/logstash/delete-pipeline.asciidoc index ff7494d34e615..2e56b0289a394 100644 --- a/docs/reference/rest-api/logstash/delete-pipeline.asciidoc +++ b/docs/reference/rest-api/logstash/delete-pipeline.asciidoc @@ -5,6 +5,12 @@ Delete {ls} pipeline ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-logstash[{ls} APIs]. +-- + This API deletes a pipeline used for {logstash-ref}/logstash-centralized-pipeline-management.html[{ls} Central Management]. diff --git a/docs/reference/rest-api/logstash/get-pipeline.asciidoc b/docs/reference/rest-api/logstash/get-pipeline.asciidoc index 8409a5128d525..8e440f218aa09 100644 --- a/docs/reference/rest-api/logstash/get-pipeline.asciidoc +++ b/docs/reference/rest-api/logstash/get-pipeline.asciidoc @@ -5,6 +5,12 @@ Get {ls} pipeline ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-logstash[{ls} APIs]. +-- + This API retrieves pipelines used for {logstash-ref}/logstash-centralized-pipeline-management.html[{ls} Central Management]. diff --git a/docs/reference/rest-api/logstash/index.asciidoc b/docs/reference/rest-api/logstash/index.asciidoc index a52be19a75df4..7f28020a8b183 100644 --- a/docs/reference/rest-api/logstash/index.asciidoc +++ b/docs/reference/rest-api/logstash/index.asciidoc @@ -2,6 +2,12 @@ [[logstash-apis]] == {ls} APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-logstash[{ls} APIs]. +-- + The following APIs are used to manage pipelines used by {logstash-ref}/logstash-centralized-pipeline-management.html[{ls} Central Management]: diff --git a/docs/reference/rest-api/logstash/put-pipeline.asciidoc b/docs/reference/rest-api/logstash/put-pipeline.asciidoc index a0d2f02f2bf6f..26af8d0f124cb 100644 --- a/docs/reference/rest-api/logstash/put-pipeline.asciidoc +++ b/docs/reference/rest-api/logstash/put-pipeline.asciidoc @@ -5,6 +5,12 @@ Create or update {ls} pipeline ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-logstash[{ls} APIs]. +-- + This API creates or updates a {ls} pipeline used for {logstash-ref}/logstash-centralized-pipeline-management.html[{ls} Central Management]. diff --git a/docs/reference/rest-api/root.asciidoc b/docs/reference/rest-api/root.asciidoc index 8821981c2afe3..aaf40b31db21c 100644 --- a/docs/reference/rest-api/root.asciidoc +++ b/docs/reference/rest-api/root.asciidoc @@ -4,6 +4,12 @@ Root API ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-info[Info APIs]. +-- + The Elasticsearch API's base url returns its basic build, version, and cluster information. diff --git a/docs/reference/rest-api/security.asciidoc b/docs/reference/rest-api/security.asciidoc index 82cf38e52bd80..57726b074ac3f 100644 --- a/docs/reference/rest-api/security.asciidoc +++ b/docs/reference/rest-api/security.asciidoc @@ -1,6 +1,13 @@ [role="xpack"] [[security-api]] == Security APIs + +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + To use the security APIs, you must set `xpack.security.enabled` to `true` in the `elasticsearch.yml` file. diff --git a/docs/reference/rest-api/security/activate-user-profile.asciidoc b/docs/reference/rest-api/security/activate-user-profile.asciidoc index f6ce32e1bb19e..0db41937f2ff2 100644 --- a/docs/reference/rest-api/security/activate-user-profile.asciidoc +++ b/docs/reference/rest-api/security/activate-user-profile.asciidoc @@ -5,6 +5,12 @@ Activate user profile ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + NOTE: The user profile feature is designed only for use by {kib} and Elastic’s {observability}, {ents}, and {elastic-sec} solutions. Individual users and external applications should not call this API directly. Elastic reserves diff --git a/docs/reference/rest-api/security/authenticate.asciidoc b/docs/reference/rest-api/security/authenticate.asciidoc index a02deb444628d..78d9cc2bcaa9f 100644 --- a/docs/reference/rest-api/security/authenticate.asciidoc +++ b/docs/reference/rest-api/security/authenticate.asciidoc @@ -5,6 +5,12 @@ Authenticate ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Enables you to submit a request with a basic auth header to authenticate a user and retrieve information about the authenticated user. diff --git a/docs/reference/rest-api/security/bulk-create-roles.asciidoc b/docs/reference/rest-api/security/bulk-create-roles.asciidoc index e4b6ef7f765c2..030e5e42bf29a 100644 --- a/docs/reference/rest-api/security/bulk-create-roles.asciidoc +++ b/docs/reference/rest-api/security/bulk-create-roles.asciidoc @@ -1,11 +1,16 @@ [role="xpack"] [[security-api-bulk-put-role]] === Bulk create or update roles API -preview::[] ++++ Bulk create or update roles API ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Bulk adds and updates roles in the native realm. [[security-api-bulk-put-role-request]] @@ -75,7 +80,7 @@ that begin with `_` are reserved for system usage. For more information, see <>. -`remote_indices`:: beta:[] (list) A list of remote indices permissions entries. +`remote_indices`:: (list) A list of remote indices permissions entries. + -- NOTE: Remote indices are effective for <>. @@ -94,6 +99,20 @@ have on the specified indices. read access to. A document within the specified indices must match this query in order for it to be accessible by the owners of the role. +`remote_cluster`:: (list) A list of remote cluster permissions entries. ++ +-- +NOTE: Remote cluster permissions are effective for <>. +They have no effect for remote clusters configured with the <>. +-- +`clusters` (required)::: (list) A list of cluster aliases to which the permissions +in this entry apply. +`privileges`(required)::: (list) The cluster level privileges that the owners of the role +have in the specified clusters. Note - only a subset of the cluster privileges are supported for remote clusters. +The <> can be used to determine +which privileges are allowed per version. + + For more information, see <>. ==== @@ -315,7 +334,7 @@ The result would then have the `errors` field set to `true` and hold the error f "details": { "my_admin_role": { <4> "type": "action_request_validation_exception", - "reason": "Validation Failed: 1: unknown cluster privilege [bad_cluster_privilege]. a privilege must be either one of the predefined cluster privilege names [manage_own_api_key,manage_data_stream_global_retention,monitor_data_stream_global_retention,none,cancel_task,cross_cluster_replication,cross_cluster_search,delegate_pki,grant_api_key,manage_autoscaling,manage_index_templates,manage_logstash_pipelines,manage_oidc,manage_saml,manage_search_application,manage_search_query_rules,manage_search_synonyms,manage_service_account,manage_token,manage_user_profile,monitor_connector,monitor_enrich,monitor_inference,monitor_ml,monitor_rollup,monitor_snapshot,monitor_text_structure,monitor_watcher,post_behavioral_analytics_event,read_ccr,read_connector_secrets,read_fleet_secrets,read_ilm,read_pipeline,read_security,read_slm,transport_client,write_connector_secrets,write_fleet_secrets,create_snapshot,manage_behavioral_analytics,manage_ccr,manage_connector,manage_enrich,manage_ilm,manage_inference,manage_ml,manage_rollup,manage_slm,manage_watcher,monitor_data_frame_transforms,monitor_transform,manage_api_key,manage_ingest_pipelines,manage_pipeline,manage_data_frame_transforms,manage_transform,manage_security,monitor,manage,all] or a pattern over one of the available cluster actions;" + "reason": "Validation Failed: 1: unknown cluster privilege [bad_cluster_privilege]. a privilege must be either one of the predefined cluster privilege names [manage_own_api_key,manage_data_stream_global_retention,monitor_data_stream_global_retention,none,cancel_task,cross_cluster_replication,cross_cluster_search,delegate_pki,grant_api_key,manage_autoscaling,manage_index_templates,manage_logstash_pipelines,manage_oidc,manage_saml,manage_search_application,manage_search_query_rules,manage_search_synonyms,manage_service_account,manage_token,manage_user_profile,monitor_connector,monitor_enrich,monitor_inference,monitor_ml,monitor_rollup,monitor_snapshot,monitor_stats,monitor_text_structure,monitor_watcher,post_behavioral_analytics_event,read_ccr,read_connector_secrets,read_fleet_secrets,read_ilm,read_pipeline,read_security,read_slm,transport_client,write_connector_secrets,write_fleet_secrets,create_snapshot,manage_behavioral_analytics,manage_ccr,manage_connector,manage_enrich,manage_ilm,manage_inference,manage_ml,manage_rollup,manage_slm,manage_watcher,monitor_data_frame_transforms,monitor_transform,manage_api_key,manage_ingest_pipelines,manage_pipeline,manage_data_frame_transforms,manage_transform,manage_security,monitor,manage,all] or a pattern over one of the available cluster actions;" } } } diff --git a/docs/reference/rest-api/security/bulk-delete-roles.asciidoc b/docs/reference/rest-api/security/bulk-delete-roles.asciidoc index a782b5e37fcb9..899591b3276db 100644 --- a/docs/reference/rest-api/security/bulk-delete-roles.asciidoc +++ b/docs/reference/rest-api/security/bulk-delete-roles.asciidoc @@ -1,11 +1,16 @@ [role="xpack"] [[security-api-bulk-delete-role]] === Bulk delete roles API -preview::[] ++++ Bulk delete roles API ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Bulk deletes roles in the native realm. [[security-api-bulk-delete-role-request]] diff --git a/docs/reference/rest-api/security/bulk-update-api-keys.asciidoc b/docs/reference/rest-api/security/bulk-update-api-keys.asciidoc index faf87c67d1ccc..8206ac5d9d4f3 100644 --- a/docs/reference/rest-api/security/bulk-update-api-keys.asciidoc +++ b/docs/reference/rest-api/security/bulk-update-api-keys.asciidoc @@ -6,6 +6,12 @@ Bulk update API keys ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + [[security-api-bulk-update-api-keys-request]] ==== {api-request-title} diff --git a/docs/reference/rest-api/security/change-password.asciidoc b/docs/reference/rest-api/security/change-password.asciidoc index c035661cdd707..f2dc45cad20b0 100644 --- a/docs/reference/rest-api/security/change-password.asciidoc +++ b/docs/reference/rest-api/security/change-password.asciidoc @@ -5,6 +5,12 @@ Change passwords ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Changes the passwords of users in the native realm and built-in users. [[security-api-change-password-request]] diff --git a/docs/reference/rest-api/security/clear-api-key-cache.asciidoc b/docs/reference/rest-api/security/clear-api-key-cache.asciidoc index 7828026b604f0..61f05a85dfaff 100644 --- a/docs/reference/rest-api/security/clear-api-key-cache.asciidoc +++ b/docs/reference/rest-api/security/clear-api-key-cache.asciidoc @@ -5,6 +5,12 @@ Clear API key cache ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Evicts a subset of all entries from the API key cache. The cache is also automatically cleared on state changes of the security index. diff --git a/docs/reference/rest-api/security/clear-cache.asciidoc b/docs/reference/rest-api/security/clear-cache.asciidoc index 3e4e5432768bd..270856ba28e6b 100644 --- a/docs/reference/rest-api/security/clear-cache.asciidoc +++ b/docs/reference/rest-api/security/clear-cache.asciidoc @@ -5,6 +5,12 @@ Clear cache ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Evicts users from the user cache. You can completely clear the cache or evict specific users. diff --git a/docs/reference/rest-api/security/clear-privileges-cache.asciidoc b/docs/reference/rest-api/security/clear-privileges-cache.asciidoc index 69a5743419d2c..cf615010779b7 100644 --- a/docs/reference/rest-api/security/clear-privileges-cache.asciidoc +++ b/docs/reference/rest-api/security/clear-privileges-cache.asciidoc @@ -5,6 +5,12 @@ Clear privileges cache ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Evicts privileges from the native application privilege cache. The cache is also automatically cleared for applications that have their privileges updated. diff --git a/docs/reference/rest-api/security/clear-roles-cache.asciidoc b/docs/reference/rest-api/security/clear-roles-cache.asciidoc index 63c54b51dcf2f..edcb2c2723512 100644 --- a/docs/reference/rest-api/security/clear-roles-cache.asciidoc +++ b/docs/reference/rest-api/security/clear-roles-cache.asciidoc @@ -5,6 +5,12 @@ Clear roles cache ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Evicts roles from the native role cache. [[security-api-clear-role-cache-request]] diff --git a/docs/reference/rest-api/security/clear-service-token-caches.asciidoc b/docs/reference/rest-api/security/clear-service-token-caches.asciidoc index ff4587549534d..26158220418e0 100644 --- a/docs/reference/rest-api/security/clear-service-token-caches.asciidoc +++ b/docs/reference/rest-api/security/clear-service-token-caches.asciidoc @@ -6,6 +6,12 @@ Clear service account token caches ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Evicts a subset of all entries from the <> token caches. diff --git a/docs/reference/rest-api/security/create-api-keys.asciidoc b/docs/reference/rest-api/security/create-api-keys.asciidoc index f740e9413e3e9..20f1c2cb155ce 100644 --- a/docs/reference/rest-api/security/create-api-keys.asciidoc +++ b/docs/reference/rest-api/security/create-api-keys.asciidoc @@ -5,6 +5,12 @@ Create API keys ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Creates an API key for access without requiring basic authentication. [[security-api-create-api-key-request]] diff --git a/docs/reference/rest-api/security/create-cross-cluster-api-key.asciidoc b/docs/reference/rest-api/security/create-cross-cluster-api-key.asciidoc index d7d55bafc6d18..63b2f37063f9f 100644 --- a/docs/reference/rest-api/security/create-cross-cluster-api-key.asciidoc +++ b/docs/reference/rest-api/security/create-cross-cluster-api-key.asciidoc @@ -1,11 +1,16 @@ [role="xpack"] [[security-api-create-cross-cluster-api-key]] === Create Cross-Cluster API key API - ++++ Create Cross-Cluster API key ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Creates an API key of the `cross_cluster` type for the <> access. A `cross_cluster` API key cannot be used to authenticate through the REST interface. On the contrary, a <> is meant to be used through the REST interface diff --git a/docs/reference/rest-api/security/create-role-mappings.asciidoc b/docs/reference/rest-api/security/create-role-mappings.asciidoc index e78d06a5676e4..71c931260865f 100644 --- a/docs/reference/rest-api/security/create-role-mappings.asciidoc +++ b/docs/reference/rest-api/security/create-role-mappings.asciidoc @@ -5,6 +5,12 @@ Create or update role mappings ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Creates and updates role mappings. [[security-api-put-role-mapping-request]] diff --git a/docs/reference/rest-api/security/create-roles.asciidoc b/docs/reference/rest-api/security/create-roles.asciidoc index 75f1d7c799187..0b0cd828140a1 100644 --- a/docs/reference/rest-api/security/create-roles.asciidoc +++ b/docs/reference/rest-api/security/create-roles.asciidoc @@ -5,6 +5,12 @@ Create or update roles ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Adds and updates roles in the native realm. [[security-api-put-role-request]] @@ -96,6 +102,19 @@ have on the specified indices. read access to. A document within the specified indices must match this query in order for it to be accessible by the owners of the role. +`remote_cluster`:: (list) A list of remote cluster permissions entries. ++ +-- +NOTE: Remote cluster permissions are effective for <>. +They have no effect for remote clusters configured with the <>. +-- +`clusters` (required)::: (list) A list of cluster aliases to which the permissions +in this entry apply. +`privileges`(required)::: (list) The cluster level privileges that the owners of the role +have in the specified clusters. Note - only a subset of the cluster privileges are supported for remote clusters. +The <> can be used to determine +which privileges are allowed per version. + For more information, see <>. [[security-api-put-role-example]] @@ -165,21 +184,29 @@ POST /_security/role/cli_or_drivers_minimal -------------------------------------------------- // end::sql-queries-permission[] -The following example configures a role with remote indices privileges on a remote cluster: +The following example configures a role with remote indices and remote cluster privileges for a remote cluster: [source,console] -------------------------------------------------- -POST /_security/role/role_with_remote_indices +POST /_security/role/only_remote_access_role { "remote_indices": [ { - "clusters": [ "my_remote" ], <1> + "clusters": ["my_remote"], <1> "names": ["logs*"], <2> "privileges": ["read", "read_cross_cluster", "view_index_metadata"] <3> } + ], + "remote_cluster": [ + { + "clusters": ["my_remote"], <1> + "privileges": ["monitor_stats"] <4> + } ] } -------------------------------------------------- -<1> The remote indices privileges apply to remote cluster with the alias `my_remote`. -<2> Privileges are granted for indices matching pattern `logs*` on the remote cluster ( `my_remote`). +<1> The remote indices and remote cluster privileges apply to remote cluster with the alias `my_remote`. +<2> Privileges are granted for indices matching pattern `logs*` on the remote cluster (`my_remote`). <3> The actual <> granted for `logs*` on `my_remote`. +<4> The actual <> granted for `my_remote`. +Note - only a subset of the cluster privileges are supported for remote clusters. diff --git a/docs/reference/rest-api/security/create-service-token.asciidoc b/docs/reference/rest-api/security/create-service-token.asciidoc index 9a3c2b678c92e..30195d89ff47d 100644 --- a/docs/reference/rest-api/security/create-service-token.asciidoc +++ b/docs/reference/rest-api/security/create-service-token.asciidoc @@ -5,6 +5,12 @@ Create service account tokens ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Creates a <> token for access without requiring basic authentication. diff --git a/docs/reference/rest-api/security/create-users.asciidoc b/docs/reference/rest-api/security/create-users.asciidoc index 428df1102329c..6d8e1cdaa9ba0 100644 --- a/docs/reference/rest-api/security/create-users.asciidoc +++ b/docs/reference/rest-api/security/create-users.asciidoc @@ -5,6 +5,12 @@ Create or update users ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Adds and updates users in the native realm. These users are commonly referred to as _native users_. diff --git a/docs/reference/rest-api/security/delegate-pki-authentication.asciidoc b/docs/reference/rest-api/security/delegate-pki-authentication.asciidoc index 860708011cd25..00a80b3fd27bc 100644 --- a/docs/reference/rest-api/security/delegate-pki-authentication.asciidoc +++ b/docs/reference/rest-api/security/delegate-pki-authentication.asciidoc @@ -5,6 +5,12 @@ Delegate PKI authentication ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Implements the exchange of an _X509Certificate_ chain into an {es} access token. diff --git a/docs/reference/rest-api/security/delete-app-privileges.asciidoc b/docs/reference/rest-api/security/delete-app-privileges.asciidoc index 39ac1706c6dc2..dad57bcac409c 100644 --- a/docs/reference/rest-api/security/delete-app-privileges.asciidoc +++ b/docs/reference/rest-api/security/delete-app-privileges.asciidoc @@ -5,6 +5,12 @@ Delete application privileges ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Removes <>. [[security-api-delete-privilege-request]] diff --git a/docs/reference/rest-api/security/delete-role-mappings.asciidoc b/docs/reference/rest-api/security/delete-role-mappings.asciidoc index c5dd1aa9c909f..4ec7e3817b031 100644 --- a/docs/reference/rest-api/security/delete-role-mappings.asciidoc +++ b/docs/reference/rest-api/security/delete-role-mappings.asciidoc @@ -5,6 +5,12 @@ Delete role mappings ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Removes role mappings. [[security-api-delete-role-mapping-request]] diff --git a/docs/reference/rest-api/security/delete-roles.asciidoc b/docs/reference/rest-api/security/delete-roles.asciidoc index 427e7c6b1860d..d30a4c5251809 100644 --- a/docs/reference/rest-api/security/delete-roles.asciidoc +++ b/docs/reference/rest-api/security/delete-roles.asciidoc @@ -5,6 +5,12 @@ Delete roles ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Removes roles in the native realm. [[security-api-delete-role-request]] diff --git a/docs/reference/rest-api/security/delete-service-token.asciidoc b/docs/reference/rest-api/security/delete-service-token.asciidoc index b704fb9121263..f7c488e9e7139 100644 --- a/docs/reference/rest-api/security/delete-service-token.asciidoc +++ b/docs/reference/rest-api/security/delete-service-token.asciidoc @@ -5,6 +5,12 @@ Delete service account token ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Deletes <> tokens for a `service` in a specified `namespace`. diff --git a/docs/reference/rest-api/security/delete-users.asciidoc b/docs/reference/rest-api/security/delete-users.asciidoc index b08f99e809b44..ff781a7d9ef0b 100644 --- a/docs/reference/rest-api/security/delete-users.asciidoc +++ b/docs/reference/rest-api/security/delete-users.asciidoc @@ -5,6 +5,12 @@ Delete users ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Deletes users from the native realm. [[security-api-delete-user-request]] diff --git a/docs/reference/rest-api/security/disable-user-profile.asciidoc b/docs/reference/rest-api/security/disable-user-profile.asciidoc index 35658f071679b..f665b8955e0d0 100644 --- a/docs/reference/rest-api/security/disable-user-profile.asciidoc +++ b/docs/reference/rest-api/security/disable-user-profile.asciidoc @@ -5,6 +5,12 @@ Disable user profile ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + NOTE: The user profile feature is designed only for use by {kib} and Elastic’s {observability}, {ents}, and {elastic-sec} solutions. Individual users and external applications should not call this API directly. Elastic reserves diff --git a/docs/reference/rest-api/security/disable-users.asciidoc b/docs/reference/rest-api/security/disable-users.asciidoc index 9859085cb1824..3f8bc74d7e106 100644 --- a/docs/reference/rest-api/security/disable-users.asciidoc +++ b/docs/reference/rest-api/security/disable-users.asciidoc @@ -5,6 +5,12 @@ Disable users ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Disables users in the native realm. diff --git a/docs/reference/rest-api/security/enable-user-profile.asciidoc b/docs/reference/rest-api/security/enable-user-profile.asciidoc index e27673b07f598..9f74d90f88b99 100644 --- a/docs/reference/rest-api/security/enable-user-profile.asciidoc +++ b/docs/reference/rest-api/security/enable-user-profile.asciidoc @@ -5,6 +5,12 @@ Enable user profile ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + NOTE: The user profile feature is designed only for use by {kib} and Elastic’s {observability}, {ents}, and {elastic-sec} solutions. Individual users and external applications should not call this API directly. Elastic reserves diff --git a/docs/reference/rest-api/security/enable-users.asciidoc b/docs/reference/rest-api/security/enable-users.asciidoc index 04193e6c27944..db78a5c222116 100644 --- a/docs/reference/rest-api/security/enable-users.asciidoc +++ b/docs/reference/rest-api/security/enable-users.asciidoc @@ -5,6 +5,12 @@ Enable users ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Enables users in the native realm. diff --git a/docs/reference/rest-api/security/enroll-kibana.asciidoc b/docs/reference/rest-api/security/enroll-kibana.asciidoc index 55de31b5407d1..78bc0569fa193 100644 --- a/docs/reference/rest-api/security/enroll-kibana.asciidoc +++ b/docs/reference/rest-api/security/enroll-kibana.asciidoc @@ -4,6 +4,12 @@ Enroll {kib} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Enables a {kib} instance to configure itself for communication with a secured {es} cluster. NOTE: This API is currently intended for internal use only by {kib}. diff --git a/docs/reference/rest-api/security/enroll-node.asciidoc b/docs/reference/rest-api/security/enroll-node.asciidoc index 81cae73fb22d6..d5c2ce2a51746 100644 --- a/docs/reference/rest-api/security/enroll-node.asciidoc +++ b/docs/reference/rest-api/security/enroll-node.asciidoc @@ -4,6 +4,12 @@ Enroll node ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Allows a new node to join an existing cluster with security features enabled. [[security-api-node-enrollment-api-request]] diff --git a/docs/reference/rest-api/security/get-api-keys.asciidoc b/docs/reference/rest-api/security/get-api-keys.asciidoc index bf49297539895..6a133f136d4b3 100644 --- a/docs/reference/rest-api/security/get-api-keys.asciidoc +++ b/docs/reference/rest-api/security/get-api-keys.asciidoc @@ -5,6 +5,12 @@ Get API key information ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Retrieves information for one or more API keys. [[security-api-get-api-key-request]] diff --git a/docs/reference/rest-api/security/get-app-privileges.asciidoc b/docs/reference/rest-api/security/get-app-privileges.asciidoc index f0f3f1b69071c..c8bb709f96d5a 100644 --- a/docs/reference/rest-api/security/get-app-privileges.asciidoc +++ b/docs/reference/rest-api/security/get-app-privileges.asciidoc @@ -5,6 +5,12 @@ Get application privileges ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Retrieves <>. [[security-api-get-privileges-request]] diff --git a/docs/reference/rest-api/security/get-builtin-privileges.asciidoc b/docs/reference/rest-api/security/get-builtin-privileges.asciidoc index 8435f5539ab9d..08a03a5b1e830 100644 --- a/docs/reference/rest-api/security/get-builtin-privileges.asciidoc +++ b/docs/reference/rest-api/security/get-builtin-privileges.asciidoc @@ -6,6 +6,12 @@ Get builtin privileges ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Retrieves the list of <> and <> that are available in this version of {es}. @@ -111,6 +117,7 @@ A successful call returns an object with "cluster", "index", and "remote_cluster "monitor_ml", "monitor_rollup", "monitor_snapshot", + "monitor_stats", "monitor_text_structure", "monitor_transform", "monitor_watcher", @@ -152,7 +159,8 @@ A successful call returns an object with "cluster", "index", and "remote_cluster "write" ], "remote_cluster" : [ - "monitor_enrich" + "monitor_enrich", + "monitor_stats" ] } -------------------------------------------------- diff --git a/docs/reference/rest-api/security/get-role-mappings.asciidoc b/docs/reference/rest-api/security/get-role-mappings.asciidoc index 8272ec4d015a8..49063e775982b 100644 --- a/docs/reference/rest-api/security/get-role-mappings.asciidoc +++ b/docs/reference/rest-api/security/get-role-mappings.asciidoc @@ -5,6 +5,12 @@ Get role mappings ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Retrieves role mappings. [[security-api-get-role-mapping-request]] diff --git a/docs/reference/rest-api/security/get-roles.asciidoc b/docs/reference/rest-api/security/get-roles.asciidoc index 3cc2f95c6ea7e..03f083e8202f0 100644 --- a/docs/reference/rest-api/security/get-roles.asciidoc +++ b/docs/reference/rest-api/security/get-roles.asciidoc @@ -5,6 +5,12 @@ Get roles ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Retrieves roles in the native realm. [[security-api-get-role-request]] diff --git a/docs/reference/rest-api/security/get-service-accounts.asciidoc b/docs/reference/rest-api/security/get-service-accounts.asciidoc index 526c6e65ccf33..63853a92f74ab 100644 --- a/docs/reference/rest-api/security/get-service-accounts.asciidoc +++ b/docs/reference/rest-api/security/get-service-accounts.asciidoc @@ -6,6 +6,12 @@ Get service accounts ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Retrieves information about <>. NOTE: Currently, only the `elastic/fleet-server` service account is available. diff --git a/docs/reference/rest-api/security/get-service-credentials.asciidoc b/docs/reference/rest-api/security/get-service-credentials.asciidoc index 3da6c3d860558..7a24aef059ae9 100644 --- a/docs/reference/rest-api/security/get-service-credentials.asciidoc +++ b/docs/reference/rest-api/security/get-service-credentials.asciidoc @@ -1,11 +1,16 @@ [role="xpack"] [[security-api-get-service-credentials]] === Get service account credentials API - ++++ Get service account credentials ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Retrieves all service credentials for a <>. [[security-api-get-service-credentials-request]] diff --git a/docs/reference/rest-api/security/get-settings.asciidoc b/docs/reference/rest-api/security/get-settings.asciidoc index 46e4e0cf529bb..c99b9bcedba21 100644 --- a/docs/reference/rest-api/security/get-settings.asciidoc +++ b/docs/reference/rest-api/security/get-settings.asciidoc @@ -5,6 +5,12 @@ Get Security settings ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Retrieves settings for the security internal indices. [[security-api-get-settings-prereqs]] diff --git a/docs/reference/rest-api/security/get-tokens.asciidoc b/docs/reference/rest-api/security/get-tokens.asciidoc index 9f5261a477bcb..eefc86528ef43 100644 --- a/docs/reference/rest-api/security/get-tokens.asciidoc +++ b/docs/reference/rest-api/security/get-tokens.asciidoc @@ -5,6 +5,12 @@ Get token ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Creates a bearer token for access without requiring basic authentication. [[security-api-get-token-request]] diff --git a/docs/reference/rest-api/security/get-user-privileges.asciidoc b/docs/reference/rest-api/security/get-user-privileges.asciidoc index 8115cd365c5a3..4e0d68b4b679c 100644 --- a/docs/reference/rest-api/security/get-user-privileges.asciidoc +++ b/docs/reference/rest-api/security/get-user-privileges.asciidoc @@ -5,6 +5,12 @@ Get user privileges ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Retrieves the <> for the logged in user. diff --git a/docs/reference/rest-api/security/get-user-profile.asciidoc b/docs/reference/rest-api/security/get-user-profile.asciidoc index 9f0ba64d136ac..60732cf0ab395 100644 --- a/docs/reference/rest-api/security/get-user-profile.asciidoc +++ b/docs/reference/rest-api/security/get-user-profile.asciidoc @@ -5,6 +5,12 @@ Get user profiles ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + NOTE: The user profile feature is designed only for use by {kib} and Elastic’s {observability}, {ents}, and {elastic-sec} solutions. Individual users and external applications should not call this API directly. Elastic reserves diff --git a/docs/reference/rest-api/security/get-users.asciidoc b/docs/reference/rest-api/security/get-users.asciidoc index 59a390f6f2538..8770c90cdec0a 100644 --- a/docs/reference/rest-api/security/get-users.asciidoc +++ b/docs/reference/rest-api/security/get-users.asciidoc @@ -5,8 +5,13 @@ Get users ++++ -Retrieves information about users in the native realm and built-in users. +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- +Retrieves information about users in the native realm and built-in users. [[security-api-get-user-request]] ==== {api-request-title} diff --git a/docs/reference/rest-api/security/grant-api-keys.asciidoc b/docs/reference/rest-api/security/grant-api-keys.asciidoc index 10c109b00bbf9..4ab599c18dd61 100644 --- a/docs/reference/rest-api/security/grant-api-keys.asciidoc +++ b/docs/reference/rest-api/security/grant-api-keys.asciidoc @@ -5,6 +5,12 @@ Grant API keys ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Creates an API key on behalf of another user. [[security-api-grant-api-key-request]] diff --git a/docs/reference/rest-api/security/has-privileges-user-profile.asciidoc b/docs/reference/rest-api/security/has-privileges-user-profile.asciidoc index afadf394aa43c..3fc825b60bce8 100644 --- a/docs/reference/rest-api/security/has-privileges-user-profile.asciidoc +++ b/docs/reference/rest-api/security/has-privileges-user-profile.asciidoc @@ -5,6 +5,12 @@ Has privileges user profile ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + NOTE: The user profile feature is designed only for use by {kib} and Elastic’s {observability}, {ents}, and {elastic-sec} solutions. Individual users and external applications should not call this API directly. Elastic reserves diff --git a/docs/reference/rest-api/security/has-privileges.asciidoc b/docs/reference/rest-api/security/has-privileges.asciidoc index 229ffb4997273..1e08b41d92fe0 100644 --- a/docs/reference/rest-api/security/has-privileges.asciidoc +++ b/docs/reference/rest-api/security/has-privileges.asciidoc @@ -6,6 +6,12 @@ ++++ [[security-api-has-privilege]] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Determines whether the logged in user has a specified list of privileges. [[security-api-has-privileges-request]] diff --git a/docs/reference/rest-api/security/invalidate-api-keys.asciidoc b/docs/reference/rest-api/security/invalidate-api-keys.asciidoc index 57a36a97634ac..27c5a8c831808 100644 --- a/docs/reference/rest-api/security/invalidate-api-keys.asciidoc +++ b/docs/reference/rest-api/security/invalidate-api-keys.asciidoc @@ -5,6 +5,12 @@ Invalidate API key ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Invalidates one or more API keys. [[security-api-invalidate-api-key-request]] diff --git a/docs/reference/rest-api/security/invalidate-tokens.asciidoc b/docs/reference/rest-api/security/invalidate-tokens.asciidoc index 58f20fdcdc425..9a6fe6185b477 100644 --- a/docs/reference/rest-api/security/invalidate-tokens.asciidoc +++ b/docs/reference/rest-api/security/invalidate-tokens.asciidoc @@ -5,6 +5,12 @@ Invalidate token ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Invalidates one or more access tokens or refresh tokens. [[security-api-invalidate-token-request]] diff --git a/docs/reference/rest-api/security/oidc-authenticate-api.asciidoc b/docs/reference/rest-api/security/oidc-authenticate-api.asciidoc index 282a054717e6d..7bf8a1aad3ee1 100644 --- a/docs/reference/rest-api/security/oidc-authenticate-api.asciidoc +++ b/docs/reference/rest-api/security/oidc-authenticate-api.asciidoc @@ -5,6 +5,12 @@ OpenID Connect authenticate ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Submits the response to an oAuth 2.0 authentication request for consumption from {es}. Upon successful validation, {es} will respond with an {es} internal Access Token and Refresh Token that can be subsequently used for authentication. diff --git a/docs/reference/rest-api/security/oidc-logout-api.asciidoc b/docs/reference/rest-api/security/oidc-logout-api.asciidoc index a181f4c836fbd..d8bd60b3cd85b 100644 --- a/docs/reference/rest-api/security/oidc-logout-api.asciidoc +++ b/docs/reference/rest-api/security/oidc-logout-api.asciidoc @@ -5,6 +5,12 @@ OpenID Connect logout ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Submits a request to invalidate a refresh token and an access token that was generated as a response to a call to `/_security/oidc/authenticate`. diff --git a/docs/reference/rest-api/security/oidc-prepare-authentication-api.asciidoc b/docs/reference/rest-api/security/oidc-prepare-authentication-api.asciidoc index 4452020b4547c..227f154934306 100644 --- a/docs/reference/rest-api/security/oidc-prepare-authentication-api.asciidoc +++ b/docs/reference/rest-api/security/oidc-prepare-authentication-api.asciidoc @@ -5,6 +5,12 @@ OpenID Connect prepare authentication ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Creates an oAuth 2.0 authentication request as a URL string based on the configuration of the respective OpenID Connect authentication realm in {es}. diff --git a/docs/reference/rest-api/security/put-app-privileges.asciidoc b/docs/reference/rest-api/security/put-app-privileges.asciidoc index 28be4c08c4e71..cf903d99a724d 100644 --- a/docs/reference/rest-api/security/put-app-privileges.asciidoc +++ b/docs/reference/rest-api/security/put-app-privileges.asciidoc @@ -5,6 +5,12 @@ Create or update application privileges ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Adds or updates <>. [[security-api-put-privileges-request]] diff --git a/docs/reference/rest-api/security/query-api-key.asciidoc b/docs/reference/rest-api/security/query-api-key.asciidoc index 513cb99a55a4c..1dec37c166f89 100644 --- a/docs/reference/rest-api/security/query-api-key.asciidoc +++ b/docs/reference/rest-api/security/query-api-key.asciidoc @@ -1,11 +1,16 @@ [role="xpack"] [[security-api-query-api-key]] === Query API key information API - ++++ Query API key information ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + //// [source,console] ---- diff --git a/docs/reference/rest-api/security/query-role.asciidoc b/docs/reference/rest-api/security/query-role.asciidoc index 937bd263140fc..acdfbb45b84f6 100644 --- a/docs/reference/rest-api/security/query-role.asciidoc +++ b/docs/reference/rest-api/security/query-role.asciidoc @@ -6,6 +6,12 @@ Query Role ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Retrieves roles with <> in a <> fashion. [[security-api-query-role-request]] diff --git a/docs/reference/rest-api/security/saml-authenticate-api.asciidoc b/docs/reference/rest-api/security/saml-authenticate-api.asciidoc index aa556a42d699c..4c156df6e1bb3 100644 --- a/docs/reference/rest-api/security/saml-authenticate-api.asciidoc +++ b/docs/reference/rest-api/security/saml-authenticate-api.asciidoc @@ -5,6 +5,12 @@ SAML authenticate ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Submits a SAML `Response` message to {es} for consumption. NOTE: This API is intended for use by custom web applications other than {kib}. diff --git a/docs/reference/rest-api/security/saml-complete-logout-api.asciidoc b/docs/reference/rest-api/security/saml-complete-logout-api.asciidoc index 1fb4ab1581abc..d4847fb481cdb 100644 --- a/docs/reference/rest-api/security/saml-complete-logout-api.asciidoc +++ b/docs/reference/rest-api/security/saml-complete-logout-api.asciidoc @@ -5,6 +5,12 @@ SAML complete logout ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Verifies the logout response sent from the SAML IdP. NOTE: This API is intended for use by custom web applications other than {kib}. diff --git a/docs/reference/rest-api/security/saml-invalidate-api.asciidoc b/docs/reference/rest-api/security/saml-invalidate-api.asciidoc index 21c10341c6fee..fb233f3903d1c 100644 --- a/docs/reference/rest-api/security/saml-invalidate-api.asciidoc +++ b/docs/reference/rest-api/security/saml-invalidate-api.asciidoc @@ -5,6 +5,12 @@ SAML invalidate ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Submits a SAML LogoutRequest message to {es} for consumption. NOTE: This API is intended for use by custom web applications other than {kib}. diff --git a/docs/reference/rest-api/security/saml-logout-api.asciidoc b/docs/reference/rest-api/security/saml-logout-api.asciidoc index 71729365865dc..560a713b5bf15 100644 --- a/docs/reference/rest-api/security/saml-logout-api.asciidoc +++ b/docs/reference/rest-api/security/saml-logout-api.asciidoc @@ -5,6 +5,12 @@ SAML logout ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Submits a request to invalidate an access token and refresh token. NOTE: This API is intended for use by custom web applications other than {kib}. diff --git a/docs/reference/rest-api/security/saml-prepare-authentication-api.asciidoc b/docs/reference/rest-api/security/saml-prepare-authentication-api.asciidoc index b62d3d2ac9f75..60b3451cc531d 100644 --- a/docs/reference/rest-api/security/saml-prepare-authentication-api.asciidoc +++ b/docs/reference/rest-api/security/saml-prepare-authentication-api.asciidoc @@ -5,6 +5,12 @@ SAML prepare authentication ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Creates a SAML authentication request (``) as a URL string, based on the configuration of the respective SAML realm in {es}. NOTE: This API is intended for use by custom web applications other than {kib}. diff --git a/docs/reference/rest-api/security/saml-sp-metadata.asciidoc b/docs/reference/rest-api/security/saml-sp-metadata.asciidoc index deecbf5f0e664..0f66b7a9f76f8 100644 --- a/docs/reference/rest-api/security/saml-sp-metadata.asciidoc +++ b/docs/reference/rest-api/security/saml-sp-metadata.asciidoc @@ -5,6 +5,12 @@ SAML service provider metadata ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Generate SAML metadata for a SAML 2.0 Service Provider. [[security-api-saml-sp-metadata-request]] diff --git a/docs/reference/rest-api/security/ssl.asciidoc b/docs/reference/rest-api/security/ssl.asciidoc index 3b8ba0eab6888..78b9aee301096 100644 --- a/docs/reference/rest-api/security/ssl.asciidoc +++ b/docs/reference/rest-api/security/ssl.asciidoc @@ -5,6 +5,12 @@ SSL certificate ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + The `certificates` API enables you to retrieve information about the X.509 certificates that are used to encrypt communications in your {es} cluster. diff --git a/docs/reference/rest-api/security/suggest-user-profile.asciidoc b/docs/reference/rest-api/security/suggest-user-profile.asciidoc index ad01987a1e704..4ea04864e4a6e 100644 --- a/docs/reference/rest-api/security/suggest-user-profile.asciidoc +++ b/docs/reference/rest-api/security/suggest-user-profile.asciidoc @@ -5,6 +5,12 @@ Suggest user profile ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + NOTE: The user profile feature is designed only for use by {kib} and Elastic’s {observability}, {ents}, and {elastic-sec} solutions. Individual users and external applications should not call this API directly. Elastic reserves diff --git a/docs/reference/rest-api/security/update-api-key.asciidoc b/docs/reference/rest-api/security/update-api-key.asciidoc index f297e3922a657..12dd883b5955d 100644 --- a/docs/reference/rest-api/security/update-api-key.asciidoc +++ b/docs/reference/rest-api/security/update-api-key.asciidoc @@ -6,6 +6,12 @@ Update API key ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + [[security-api-update-api-key-request]] ==== {api-request-title} diff --git a/docs/reference/rest-api/security/update-cross-cluster-api-key.asciidoc b/docs/reference/rest-api/security/update-cross-cluster-api-key.asciidoc index b90cb6368eefb..5c5f55b73597a 100644 --- a/docs/reference/rest-api/security/update-cross-cluster-api-key.asciidoc +++ b/docs/reference/rest-api/security/update-cross-cluster-api-key.asciidoc @@ -6,6 +6,12 @@ Update Cross-Cluster API key ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Update an existing cross-cluster API Key that is used for <> access. diff --git a/docs/reference/rest-api/security/update-settings.asciidoc b/docs/reference/rest-api/security/update-settings.asciidoc index b227bb70b31d7..3ec2ef98153c4 100644 --- a/docs/reference/rest-api/security/update-settings.asciidoc +++ b/docs/reference/rest-api/security/update-settings.asciidoc @@ -5,6 +5,12 @@ Update Security settings ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + Updates the settings of the security internal indices. diff --git a/docs/reference/rest-api/security/update-user-profile-data.asciidoc b/docs/reference/rest-api/security/update-user-profile-data.asciidoc index 01fa5e11d10e8..c461c169a517a 100644 --- a/docs/reference/rest-api/security/update-user-profile-data.asciidoc +++ b/docs/reference/rest-api/security/update-user-profile-data.asciidoc @@ -5,6 +5,12 @@ Update user profile data ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-security[Security APIs]. +-- + NOTE: The user profile feature is designed only for use by {kib} and Elastic’s {observability}, {ents}, and {elastic-sec} solutions. Individual users and external applications should not call this API directly. Elastic reserves diff --git a/docs/reference/rest-api/usage.asciidoc b/docs/reference/rest-api/usage.asciidoc index a54dbe21b46c6..cb9d80cc3a97b 100644 --- a/docs/reference/rest-api/usage.asciidoc +++ b/docs/reference/rest-api/usage.asciidoc @@ -2,6 +2,12 @@ [[usage-api]] == Usage API +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-xpack[Usage APIs]. +-- + Provides usage information about the installed {xpack} features. [discrete] @@ -38,9 +44,10 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] ------------------------------------------------------------ GET /_xpack/usage ------------------------------------------------------------ -// TEST[s/usage/usage?filter_path=-watcher.execution.actions.index*\,-watcher.execution.actions.logging*,-watcher.execution.actions.email*/] +// TEST[s/usage/usage?filter_path=-watcher.execution.actions.index*\,-watcher.execution.actions.logging*,-watcher.execution.actions.email*,-esql.functions*/] // This response filter removes watcher logging results if they are included // to avoid errors in the CI builds. +// Same for ES|QL functions, that is a long list and quickly evolving. [source,console-result] ------------------------------------------------------------ @@ -195,12 +202,20 @@ GET /_xpack/usage } } }, - "node_count" : 1 + "node_count" : 1, + "memory": { + anomaly_detectors_memory_bytes: 0, + data_frame_analytics_memory_bytes: 0, + pytorch_inference_memory_bytes: 0, + total_used_memory_bytes: 0 + } }, "inference": { "available" : true, "enabled" : true, - "models" : [] + "models" : [ + ... + ] }, "logstash" : { "available" : true, @@ -228,8 +243,7 @@ GET /_xpack/usage "keep" : 0, "enrich" : 0, "from" : 0, - "row" : 0, - "meta" : 0 + "row" : 0 }, "queries" : { "rest" : { @@ -502,6 +516,15 @@ GET /_xpack/usage "universal_profiling" : { "available" : true, "enabled" : true + }, + "logsdb": { + "available": true, + "enabled": false, + "indices_count": 0, + "indices_with_synthetic_source": 0, + "num_docs": 0, + "size_in_bytes": 0, + "has_custom_cutoff_date": false } } ------------------------------------------------------------ @@ -513,6 +536,7 @@ GET /_xpack/usage // TESTRESPONSE[s/"policy_stats" : \[[^\]]*\]/"policy_stats" : $body.$_path/] // TESTRESPONSE[s/"slm" : \{[^\}]*\},/"slm" : $body.$_path,/] // TESTRESPONSE[s/"health_api" : \{[^\}]*\}\s*\}/"health_api" : $body.$_path/] +// TESTRESPONSE[s/"models" : \[[^\]]*\]/"models" : $body.$_path/] // TESTRESPONSE[s/"data_streams" : \{[^\}]*\},/"data_streams" : $body.$_path,/] // TESTRESPONSE[s/ : true/ : $body.$_path/] // TESTRESPONSE[s/ : false/ : $body.$_path/] @@ -529,4 +553,5 @@ GET /_xpack/usage // 5. All of the numbers and strings on the right hand side of *every* field in // the response are ignored. So we're really only asserting things about the // the shape of this response, not the values in it. -// 6. Ignore the contents of data streams until the failure store is tech preview. +// 6. Ignore the contents of the `inference.models` array because the models might not yet have been initialized +// 7. Ignore the contents of data streams until the failure store is tech preview. diff --git a/docs/reference/rest-api/watcher.asciidoc b/docs/reference/rest-api/watcher.asciidoc index 4c4ce1ab7ee00..227eda6605465 100644 --- a/docs/reference/rest-api/watcher.asciidoc +++ b/docs/reference/rest-api/watcher.asciidoc @@ -2,6 +2,12 @@ [[watcher-api]] == Watcher APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs]. +-- + * <> * <> * <> diff --git a/docs/reference/rest-api/watcher/ack-watch.asciidoc b/docs/reference/rest-api/watcher/ack-watch.asciidoc index 3c45b068a34c6..b3ea9b5d6b41f 100644 --- a/docs/reference/rest-api/watcher/ack-watch.asciidoc +++ b/docs/reference/rest-api/watcher/ack-watch.asciidoc @@ -5,6 +5,12 @@ Ack watch ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs]. +-- + <> enables you to manually throttle execution of the watch's actions. diff --git a/docs/reference/rest-api/watcher/activate-watch.asciidoc b/docs/reference/rest-api/watcher/activate-watch.asciidoc index d8af79854c83e..c37d85cc50299 100644 --- a/docs/reference/rest-api/watcher/activate-watch.asciidoc +++ b/docs/reference/rest-api/watcher/activate-watch.asciidoc @@ -5,6 +5,12 @@ Activate watch ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs]. +-- + A watch can be either <>. This API enables you to activate a currently inactive watch. diff --git a/docs/reference/rest-api/watcher/deactivate-watch.asciidoc b/docs/reference/rest-api/watcher/deactivate-watch.asciidoc index ba4170174343a..058ada195f97e 100644 --- a/docs/reference/rest-api/watcher/deactivate-watch.asciidoc +++ b/docs/reference/rest-api/watcher/deactivate-watch.asciidoc @@ -5,6 +5,12 @@ Deactivate watch ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs]. +-- + A watch can be either <>. This API enables you to deactivate a currently active watch. diff --git a/docs/reference/rest-api/watcher/delete-watch.asciidoc b/docs/reference/rest-api/watcher/delete-watch.asciidoc index 3ffcb43ed65e2..536ec293ab8fd 100644 --- a/docs/reference/rest-api/watcher/delete-watch.asciidoc +++ b/docs/reference/rest-api/watcher/delete-watch.asciidoc @@ -5,6 +5,12 @@ Delete watch ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs]. +-- + Removes a watch from {watcher}. [[watcher-api-delete-watch-request]] diff --git a/docs/reference/rest-api/watcher/execute-watch.asciidoc b/docs/reference/rest-api/watcher/execute-watch.asciidoc index 7acecf1709034..eab15a152b154 100644 --- a/docs/reference/rest-api/watcher/execute-watch.asciidoc +++ b/docs/reference/rest-api/watcher/execute-watch.asciidoc @@ -5,6 +5,12 @@ Execute watch ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs]. +-- + Forces the execution of a stored watch. [[watcher-api-execute-watch-request]] diff --git a/docs/reference/rest-api/watcher/get-settings.asciidoc b/docs/reference/rest-api/watcher/get-settings.asciidoc index c5773e6ee32b0..80d86cc455daa 100644 --- a/docs/reference/rest-api/watcher/get-settings.asciidoc +++ b/docs/reference/rest-api/watcher/get-settings.asciidoc @@ -5,6 +5,12 @@ Get Watcher settings ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs]. +-- + This API allows a user to retrieve the user-configurable settings for the Watcher internal index (`.watches`). Only a subset of the index settings—those that are user-configurable—will be shown. This includes: - `index.auto_expand_replicas` diff --git a/docs/reference/rest-api/watcher/get-watch.asciidoc b/docs/reference/rest-api/watcher/get-watch.asciidoc index e80bfed88b6e5..1b5dbe2d0e47e 100644 --- a/docs/reference/rest-api/watcher/get-watch.asciidoc +++ b/docs/reference/rest-api/watcher/get-watch.asciidoc @@ -5,6 +5,12 @@ Get watch ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs]. +-- + Retrieves a watch by its ID. [[watcher-api-get-watch-request]] diff --git a/docs/reference/rest-api/watcher/put-watch.asciidoc b/docs/reference/rest-api/watcher/put-watch.asciidoc index deab44f106fb6..134e8149fde79 100644 --- a/docs/reference/rest-api/watcher/put-watch.asciidoc +++ b/docs/reference/rest-api/watcher/put-watch.asciidoc @@ -5,6 +5,12 @@ Create or update watch ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs]. +-- + Either registers a new watch in {watcher} or updates an existing one. [[watcher-api-put-watch-request]] diff --git a/docs/reference/rest-api/watcher/query-watches.asciidoc b/docs/reference/rest-api/watcher/query-watches.asciidoc index 7a006243ed7f4..b96261d4a47ea 100644 --- a/docs/reference/rest-api/watcher/query-watches.asciidoc +++ b/docs/reference/rest-api/watcher/query-watches.asciidoc @@ -5,6 +5,12 @@ Query watches ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs]. +-- + Retrieves all registered watches. [[watcher-api-query-watches-request]] diff --git a/docs/reference/rest-api/watcher/start.asciidoc b/docs/reference/rest-api/watcher/start.asciidoc index b153410ed2901..15606ba0a8571 100644 --- a/docs/reference/rest-api/watcher/start.asciidoc +++ b/docs/reference/rest-api/watcher/start.asciidoc @@ -5,6 +5,12 @@ Start watch service ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs]. +-- + Starts the {watcher} service if it is not already running. [[watcher-api-start-request]] diff --git a/docs/reference/rest-api/watcher/stats.asciidoc b/docs/reference/rest-api/watcher/stats.asciidoc index 2dbca69a67616..e0bdc1b5a8892 100644 --- a/docs/reference/rest-api/watcher/stats.asciidoc +++ b/docs/reference/rest-api/watcher/stats.asciidoc @@ -6,6 +6,12 @@ Get {watcher} stats ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs]. +-- + Retrieves the current {watcher} metrics. [[watcher-api-stats-request]] diff --git a/docs/reference/rest-api/watcher/stop.asciidoc b/docs/reference/rest-api/watcher/stop.asciidoc index 50acd6e9eb2d1..272899a2cfa3f 100644 --- a/docs/reference/rest-api/watcher/stop.asciidoc +++ b/docs/reference/rest-api/watcher/stop.asciidoc @@ -5,6 +5,12 @@ Stop watch service ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs]. +-- + Stops the {watcher} service if it is running. [[watcher-api-stop-request]] diff --git a/docs/reference/rest-api/watcher/update-settings.asciidoc b/docs/reference/rest-api/watcher/update-settings.asciidoc index 8602c6776997d..9ad38064e34ab 100644 --- a/docs/reference/rest-api/watcher/update-settings.asciidoc +++ b/docs/reference/rest-api/watcher/update-settings.asciidoc @@ -5,6 +5,12 @@ Update Watcher settings ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-watcher[{watcher} APIs]. +-- + This API allows a user to modify the settings for the Watcher internal index (`.watches`). Only a subset of settings are allowed to by modified. This includes: - `index.auto_expand_replicas` diff --git a/docs/reference/rollup/apis/delete-job.asciidoc b/docs/reference/rollup/apis/delete-job.asciidoc index c563e705039e2..03f5349e15d4f 100644 --- a/docs/reference/rollup/apis/delete-job.asciidoc +++ b/docs/reference/rollup/apis/delete-job.asciidoc @@ -8,6 +8,12 @@ deprecated::[8.11.0,"Rollups will be removed in a future version. Use <> instead."] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-rollup[Rollup APIs]. +-- + Deletes an existing {rollup-job}. [[rollup-delete-job-request]] diff --git a/docs/reference/rollup/apis/get-job.asciidoc b/docs/reference/rollup/apis/get-job.asciidoc index fcafbbe95159b..9fff4d665f5fd 100644 --- a/docs/reference/rollup/apis/get-job.asciidoc +++ b/docs/reference/rollup/apis/get-job.asciidoc @@ -7,6 +7,12 @@ deprecated::[8.11.0,"Rollups will be removed in a future version. Use <> instead."] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-rollup[Rollup APIs]. +-- + Retrieves the configuration, stats, and status of {rollup-jobs}. [[rollup-get-job-request]] diff --git a/docs/reference/rollup/apis/put-job.asciidoc b/docs/reference/rollup/apis/put-job.asciidoc index c52e7a042e0ca..a60f20a3de5bf 100644 --- a/docs/reference/rollup/apis/put-job.asciidoc +++ b/docs/reference/rollup/apis/put-job.asciidoc @@ -12,6 +12,12 @@ WARNING: From 8.15.0 invoking this API in a cluster with no rollup usage will fa deprecation and planned removal. A cluster either needs to contain a rollup job or a rollup index in order for this API to be allowed to execute. +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-rollup[Rollup APIs]. +-- + Creates a {rollup-job}. [[rollup-put-job-api-request]] diff --git a/docs/reference/rollup/apis/rollup-caps.asciidoc b/docs/reference/rollup/apis/rollup-caps.asciidoc index 95f652f6d4415..be1c3ed171a23 100644 --- a/docs/reference/rollup/apis/rollup-caps.asciidoc +++ b/docs/reference/rollup/apis/rollup-caps.asciidoc @@ -7,6 +7,12 @@ deprecated::[8.11.0,"Rollups will be removed in a future version. Use <> instead."] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-rollup[Rollup APIs]. +-- + Returns the capabilities of any {rollup-jobs} that have been configured for a specific index or index pattern. diff --git a/docs/reference/rollup/apis/rollup-index-caps.asciidoc b/docs/reference/rollup/apis/rollup-index-caps.asciidoc index c5b729f2e52e6..830cc332e8f40 100644 --- a/docs/reference/rollup/apis/rollup-index-caps.asciidoc +++ b/docs/reference/rollup/apis/rollup-index-caps.asciidoc @@ -7,6 +7,12 @@ deprecated::[8.11.0,"Rollups will be removed in a future version. Use <> instead."] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-rollup[Rollup APIs]. +-- + Returns the rollup capabilities of all jobs inside of a rollup index (e.g. the index where rollup data is stored). diff --git a/docs/reference/rollup/apis/rollup-search.asciidoc b/docs/reference/rollup/apis/rollup-search.asciidoc index 491dcc6c38ae2..088a74973806b 100644 --- a/docs/reference/rollup/apis/rollup-search.asciidoc +++ b/docs/reference/rollup/apis/rollup-search.asciidoc @@ -7,6 +7,12 @@ deprecated::[8.11.0,"Rollups will be removed in a future version. Use <> instead."] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-rollup[Rollup APIs]. +-- + Enables searching rolled-up data using the standard Query DSL. [[rollup-search-request]] diff --git a/docs/reference/rollup/apis/start-job.asciidoc b/docs/reference/rollup/apis/start-job.asciidoc index c102c26ea5d8e..dbeed8b09d1c8 100644 --- a/docs/reference/rollup/apis/start-job.asciidoc +++ b/docs/reference/rollup/apis/start-job.asciidoc @@ -8,6 +8,12 @@ deprecated::[8.11.0,"Rollups will be removed in a future version. Use <> instead."] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-rollup[Rollup APIs]. +-- + Starts an existing, stopped {rollup-job}. [[rollup-start-job-request]] diff --git a/docs/reference/rollup/apis/stop-job.asciidoc b/docs/reference/rollup/apis/stop-job.asciidoc index 61e561b4ceac9..8c0fd6ab2f3af 100644 --- a/docs/reference/rollup/apis/stop-job.asciidoc +++ b/docs/reference/rollup/apis/stop-job.asciidoc @@ -8,6 +8,12 @@ deprecated::[8.11.0,"Rollups will be removed in a future version. Use <> instead."] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-rollup[Rollup APIs]. +-- + Stops an existing, started {rollup-job}. [[rollup-stop-job-request]] diff --git a/docs/reference/rollup/rollup-apis.asciidoc b/docs/reference/rollup/rollup-apis.asciidoc index 44833a0846c2f..71922f0932a0e 100644 --- a/docs/reference/rollup/rollup-apis.asciidoc +++ b/docs/reference/rollup/rollup-apis.asciidoc @@ -4,6 +4,12 @@ deprecated::[8.11.0,"Rollups will be removed in a future version. Please <> to <> instead."] +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-rollup[Rollup APIs]. +-- + [discrete] [[rollup-jobs-endpoint]] === Jobs diff --git a/docs/reference/run-elasticsearch-locally.asciidoc b/docs/reference/run-elasticsearch-locally.asciidoc new file mode 100644 index 0000000000000..7efe9f243b215 --- /dev/null +++ b/docs/reference/run-elasticsearch-locally.asciidoc @@ -0,0 +1,72 @@ +//// +IMPORTANT: This content is replicated in the Elasticsearch repo root readme. Ensure both files are in sync. + +https://github.com/elastic/start-local is the source of truth. +//// + +[[run-elasticsearch-locally]] +== Run {es} locally +++++ +Run {es} locally +++++ + +[WARNING] +==== +*DO NOT USE THESE INSTRUCTIONS FOR PRODUCTION DEPLOYMENTS* + +The instructions on this page are for *local development only*. Do not use this configuration for production deployments, because it is not secure. +Refer to <> for a list of production deployment options. +==== + +Quickly set up {es} and {kib} in Docker for local development or testing, using the https://github.com/elastic/start-local?tab=readme-ov-file#-try-elasticsearch-and-kibana-locally[`start-local` script]. + +This setup comes with a one-month trial of the Elastic *Platinum* license. +After the trial period, the license reverts to *Free and open - Basic*. +Refer to https://www.elastic.co/subscriptions[Elastic subscriptions] for more information. + +[discrete] +[[local-dev-prerequisites]] +=== Prerequisites + +- If you don't have Docker installed, https://www.docker.com/products/docker-desktop[download and install Docker Desktop] for your operating system. +- If you're using Microsoft Windows, then install https://learn.microsoft.com/en-us/windows/wsl/install[Windows Subsystem for Linux (WSL)]. + +[discrete] +[[local-dev-quick-start]] +=== Run `start-local` + +To set up {es} and {kib} locally, run the `start-local` script: + +[source,sh] +---- +curl -fsSL https://elastic.co/start-local | sh +---- +// NOTCONSOLE +// REVIEWED[OCT.28.2024] + +This script creates an `elastic-start-local` folder containing configuration files and starts both {es} and {kib} using Docker. + +After running the script, you can access Elastic services at the following endpoints: + +* *{es}*: http://localhost:9200 +* *{kib}*: http://localhost:5601 + +The script generates a random password for the `elastic` user, and an API key, stored in the `.env` file. + +[CAUTION] +==== +This setup is for local testing only. HTTPS is disabled, and Basic authentication is used for {es}. For security, {es} and {kib} are accessible only through `localhost`. +==== + +[discrete] +[[local-dev-additional-info]] +=== Learn more + +For more detailed information about the `start-local` setup, refer to the https://github.com/elastic/start-local[README on GitHub]. +Learn about customizing the setup, logging, and more. + +[discrete] +[[local-dev-next-steps]] +=== Next steps + +Use our <> to learn the basics of {es}. \ No newline at end of file diff --git a/docs/reference/scripting/apis/create-stored-script-api.asciidoc b/docs/reference/scripting/apis/create-stored-script-api.asciidoc index dab1314e65dc4..5636e212180b3 100644 --- a/docs/reference/scripting/apis/create-stored-script-api.asciidoc +++ b/docs/reference/scripting/apis/create-stored-script-api.asciidoc @@ -4,6 +4,12 @@ Create or update stored script ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-script[Script APIs]. +-- + Creates or updates a <> or <>. diff --git a/docs/reference/scripting/apis/delete-stored-script-api.asciidoc b/docs/reference/scripting/apis/delete-stored-script-api.asciidoc index e233922c9a7d5..c6e570f2013e4 100644 --- a/docs/reference/scripting/apis/delete-stored-script-api.asciidoc +++ b/docs/reference/scripting/apis/delete-stored-script-api.asciidoc @@ -4,6 +4,12 @@ Delete stored script ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-script[Script APIs]. +-- + Deletes a <> or <>. diff --git a/docs/reference/scripting/apis/get-script-contexts-api.asciidoc b/docs/reference/scripting/apis/get-script-contexts-api.asciidoc index ca24c97e494ee..0ef6eccf947ad 100644 --- a/docs/reference/scripting/apis/get-script-contexts-api.asciidoc +++ b/docs/reference/scripting/apis/get-script-contexts-api.asciidoc @@ -4,6 +4,12 @@ Get script contexts ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-script[Script APIs]. +-- + Retrieves a list of supported script contexts and their methods. [source,console] diff --git a/docs/reference/scripting/apis/get-script-languages-api.asciidoc b/docs/reference/scripting/apis/get-script-languages-api.asciidoc index dd5935bc4dcd8..a35c979b2fccb 100644 --- a/docs/reference/scripting/apis/get-script-languages-api.asciidoc +++ b/docs/reference/scripting/apis/get-script-languages-api.asciidoc @@ -4,6 +4,12 @@ Get script languages ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-script[Script APIs]. +-- + Retrieves a list of supported <> and their contexts. diff --git a/docs/reference/scripting/apis/get-stored-script-api.asciidoc b/docs/reference/scripting/apis/get-stored-script-api.asciidoc index fffeb24e0331f..d2e5a7beedad1 100644 --- a/docs/reference/scripting/apis/get-stored-script-api.asciidoc +++ b/docs/reference/scripting/apis/get-stored-script-api.asciidoc @@ -4,6 +4,12 @@ Get stored script ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-script[Script APIs]. +-- + Retrieves a <> or <>. diff --git a/docs/reference/scripting/apis/script-apis.asciidoc b/docs/reference/scripting/apis/script-apis.asciidoc index e344cb00ee6fe..f5499ade8458a 100644 --- a/docs/reference/scripting/apis/script-apis.asciidoc +++ b/docs/reference/scripting/apis/script-apis.asciidoc @@ -1,6 +1,12 @@ [[script-apis]] == Script APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-script[Script APIs]. +-- + Use the following APIs to manage, store, and test your <>. diff --git a/docs/reference/scripting/fields.asciidoc b/docs/reference/scripting/fields.asciidoc index c2a40d4519f9f..8a9bb3c712789 100644 --- a/docs/reference/scripting/fields.asciidoc +++ b/docs/reference/scripting/fields.asciidoc @@ -80,6 +80,79 @@ GET my-index-000001/_search } ------------------------------------- +[discrete] +[[scripting-term-statistics]] +=== Accessing term statistics of a document within a script + +Scripts used in a <> query have access to the `_termStats` variable which provides statistical information about the terms in the child query. + +In the following example, `_termStats` is used within a <> query to retrieve the average term frequency for the terms `quick`, `brown`, and `fox` in the `text` field: + +[source,console] +------------------------------------- +PUT my-index-000001/_doc/1?refresh +{ + "text": "quick brown fox" +} + +PUT my-index-000001/_doc/2?refresh +{ + "text": "quick fox" +} + +GET my-index-000001/_search +{ + "query": { + "script_score": { + "query": { <1> + "match": { + "text": "quick brown fox" + } + }, + "script": { + "source": "_termStats.termFreq().getAverage()" <2> + } + } + } +} +------------------------------------- + +<1> Child query used to infer the field and the terms considered in term statistics. + +<2> The script calculates the average document frequency for the terms in the query using `_termStats`. + +`_termStats` provides access to the following functions for working with term statistics: + +- `uniqueTermsCount`: Returns the total number of unique terms in the query. This value is the same across all documents. +- `matchedTermsCount`: Returns the count of query terms that matched within the current document. +- `docFreq`: Provides document frequency statistics for the terms in the query, indicating how many documents contain each term. This value is consistent across all documents. +- `totalTermFreq`: Provides the total frequency of terms across all documents, representing how often each term appears in the entire corpus. This value is consistent across all documents. +- `termFreq`: Returns the frequency of query terms within the current document, showing how often each term appears in that document. + +[NOTE] +.Functions returning aggregated statistics +=================================================== + +The `docFreq`, `termFreq` and `totalTermFreq` functions return objects that represent statistics across all terms of the child query. + +Statistics provides support for the following methods: + +`getAverage()`: Returns the average value of the metric. +`getMin()`: Returns the minimum value of the metric. +`getMax()`: Returns the maximum value of the metric. +`getSum()`: Returns the sum of the metric values. +`getCount()`: Returns the count of terms included in the metric calculation. + +=================================================== + + +[NOTE] +.Painless language required +=================================================== + +The `_termStats` variable is only available when using the <> scripting language. + +=================================================== [discrete] [[modules-scripting-doc-vals]] diff --git a/docs/reference/search-application/apis/delete-search-application.asciidoc b/docs/reference/search-application/apis/delete-search-application.asciidoc index 4043942b09503..52a32247a79e1 100644 --- a/docs/reference/search-application/apis/delete-search-application.asciidoc +++ b/docs/reference/search-application/apis/delete-search-application.asciidoc @@ -1,13 +1,18 @@ [role="xpack"] [[delete-search-application]] === Delete Search Application - -beta::[] - ++++ Delete Search Application ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-search_application[Search application APIs]. +-- + +beta::[] + Removes a Search Application and its associated alias. Indices attached to the Search Application are not removed. diff --git a/docs/reference/search-application/apis/get-search-application.asciidoc b/docs/reference/search-application/apis/get-search-application.asciidoc index f0c107011eb40..adeb84bdbe0ae 100644 --- a/docs/reference/search-application/apis/get-search-application.asciidoc +++ b/docs/reference/search-application/apis/get-search-application.asciidoc @@ -1,13 +1,18 @@ [role="xpack"] [[get-search-application]] === Get Search Application - -beta::[] - ++++ Get Search Application ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-search_application[Search application APIs]. +-- + +beta::[] + Retrieves information about a search application. [[get-search-application-request]] diff --git a/docs/reference/search-application/apis/index.asciidoc b/docs/reference/search-application/apis/index.asciidoc index 1df38f6a841cc..a01b93a1ee4ac 100644 --- a/docs/reference/search-application/apis/index.asciidoc +++ b/docs/reference/search-application/apis/index.asciidoc @@ -9,6 +9,12 @@ beta::[] --- +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-search_application[Search application APIs]. +-- + Use Search Application APIs to manage tasks and resources related to Search Applications. * <> diff --git a/docs/reference/search-application/apis/list-search-applications.asciidoc b/docs/reference/search-application/apis/list-search-applications.asciidoc index 3cc077bf682d6..33bd8ddee009e 100644 --- a/docs/reference/search-application/apis/list-search-applications.asciidoc +++ b/docs/reference/search-application/apis/list-search-applications.asciidoc @@ -1,13 +1,18 @@ [role="xpack"] [[list-search-applications]] === List Search Applications - -beta::[] - ++++ List Search Applications ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-search_application[Search application APIs]. +-- + +beta::[] + Returns information about Search Applications. [[list-search-applications-request]] diff --git a/docs/reference/search-application/apis/put-search-application.asciidoc b/docs/reference/search-application/apis/put-search-application.asciidoc index eb559acc8cdc7..3dffb86782227 100644 --- a/docs/reference/search-application/apis/put-search-application.asciidoc +++ b/docs/reference/search-application/apis/put-search-application.asciidoc @@ -1,13 +1,18 @@ [role="xpack"] [[put-search-application]] === Put Search Application - -beta::[] - ++++ Put Search Application ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-search_application[Search application APIs]. +-- + +beta::[] + Creates or updates a Search Application. [[put-search-application-request]] diff --git a/docs/reference/search-application/apis/search-application-render-query.asciidoc b/docs/reference/search-application/apis/search-application-render-query.asciidoc index 687176b4fb070..42586c7186bc8 100644 --- a/docs/reference/search-application/apis/search-application-render-query.asciidoc +++ b/docs/reference/search-application/apis/search-application-render-query.asciidoc @@ -1,13 +1,18 @@ [role="xpack"] [[search-application-render-query]] === Render Search Application Query - -preview::[] - ++++ Render Search Application Query ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-search_application[Search application APIs]. +-- + +preview::[] + Given specified query parameters, generates an {es} query using the search template associated with the search application or a default template if none is specified. Unspecified template parameters will be assigned their default values (if applicable). diff --git a/docs/reference/search-application/apis/search-application-search.asciidoc b/docs/reference/search-application/apis/search-application-search.asciidoc index 2d13ed5f11037..1aab4ddf84763 100644 --- a/docs/reference/search-application/apis/search-application-search.asciidoc +++ b/docs/reference/search-application/apis/search-application-search.asciidoc @@ -1,13 +1,18 @@ [role="xpack"] [[search-application-search]] === Search Application Search - -beta::[] - ++++ Search Application Search ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-search_application[Search application APIs]. +-- + +beta::[] + Given specified query parameters, generates and executes an {es} query using the search template associated with the search application or a default template if none is specified. Unspecified template parameters will be assigned their default values (if applicable). diff --git a/docs/reference/search.asciidoc b/docs/reference/search.asciidoc index b39afff876eed..268fa4d04603d 100644 --- a/docs/reference/search.asciidoc +++ b/docs/reference/search.asciidoc @@ -1,6 +1,12 @@ [[search]] == Search APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs]. +-- + Search APIs are used to search and aggregate data stored in {es} indices and data streams. For an overview and related tutorials, see <>. diff --git a/docs/reference/search/async-search.asciidoc b/docs/reference/search/async-search.asciidoc index 786cfaee8024c..9a9e9ca45e817 100644 --- a/docs/reference/search/async-search.asciidoc +++ b/docs/reference/search/async-search.asciidoc @@ -2,6 +2,12 @@ [[async-search]] === Async search +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs]. +-- + The async search API let you asynchronously execute a search request, monitor its progress, and retrieve partial results as they become available. diff --git a/docs/reference/search/clear-scroll-api.asciidoc b/docs/reference/search/clear-scroll-api.asciidoc index a005babfd1bef..a6a2cd4b3cab5 100644 --- a/docs/reference/search/clear-scroll-api.asciidoc +++ b/docs/reference/search/clear-scroll-api.asciidoc @@ -4,6 +4,12 @@ Clear scroll ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs]. +-- + Clears the search context and results for a <>. diff --git a/docs/reference/search/count.asciidoc b/docs/reference/search/count.asciidoc index 399545adf8d1d..e0e86dd131291 100644 --- a/docs/reference/search/count.asciidoc +++ b/docs/reference/search/count.asciidoc @@ -4,6 +4,12 @@ Count ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs]. +-- + Gets the number of matches for a search query. [source,console] diff --git a/docs/reference/search/explain.asciidoc b/docs/reference/search/explain.asciidoc index 77e2d5bd63efd..01c7bb4611a12 100644 --- a/docs/reference/search/explain.asciidoc +++ b/docs/reference/search/explain.asciidoc @@ -4,6 +4,12 @@ Explain ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs]. +-- + Returns information about why a specific document matches (or doesn't match) a query. diff --git a/docs/reference/search/field-caps.asciidoc b/docs/reference/search/field-caps.asciidoc index 2ff2b8d18604e..f4d9146b8ea9d 100644 --- a/docs/reference/search/field-caps.asciidoc +++ b/docs/reference/search/field-caps.asciidoc @@ -4,6 +4,11 @@ Field capabilities ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs]. +-- Allows you to retrieve the capabilities of fields among multiple indices. For data streams, the API returns field capabilities among the stream's backing diff --git a/docs/reference/search/multi-search-template-api.asciidoc b/docs/reference/search/multi-search-template-api.asciidoc index b1c9518b1f2bc..010320c6b05ed 100644 --- a/docs/reference/search/multi-search-template-api.asciidoc +++ b/docs/reference/search/multi-search-template-api.asciidoc @@ -4,6 +4,12 @@ Multi search template ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs]. +-- + Runs multiple <> with a single request. diff --git a/docs/reference/search/multi-search.asciidoc b/docs/reference/search/multi-search.asciidoc index 9cafa756f035e..ea2dd59779332 100644 --- a/docs/reference/search/multi-search.asciidoc +++ b/docs/reference/search/multi-search.asciidoc @@ -4,6 +4,12 @@ Multi search ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs]. +-- + Executes several searches with a single API request. [source,console] diff --git a/docs/reference/search/point-in-time-api.asciidoc b/docs/reference/search/point-in-time-api.asciidoc index 9cd91626c7600..bc7cbd01ebb36 100644 --- a/docs/reference/search/point-in-time-api.asciidoc +++ b/docs/reference/search/point-in-time-api.asciidoc @@ -4,6 +4,12 @@ Point in time ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs]. +-- + A search request by default executes against the most recent visible data of the target indices, which is called point in time. Elasticsearch pit (point in time) is a lightweight view into the state of the data as it existed when initiated. diff --git a/docs/reference/search/profile.asciidoc b/docs/reference/search/profile.asciidoc index 3fed14231808c..f69092847c232 100644 --- a/docs/reference/search/profile.asciidoc +++ b/docs/reference/search/profile.asciidoc @@ -24,6 +24,11 @@ The output from the Profile API is *very* verbose, especially for complicated requests executed across many shards. Pretty-printing the response is recommended to help understand the output. +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs]. +-- [[search-profile-api-example]] ==== {api-examples-title} diff --git a/docs/reference/search/rank-eval.asciidoc b/docs/reference/search/rank-eval.asciidoc index 05862ebbbcca5..4a03371c4da3d 100644 --- a/docs/reference/search/rank-eval.asciidoc +++ b/docs/reference/search/rank-eval.asciidoc @@ -4,6 +4,12 @@ Ranking evaluation ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs]. +-- + Allows you to evaluate the quality of ranked search results over a set of typical search queries. diff --git a/docs/reference/search/render-search-template-api.asciidoc b/docs/reference/search/render-search-template-api.asciidoc index 0c782f26068e6..42e82b6e352be 100644 --- a/docs/reference/search/render-search-template-api.asciidoc +++ b/docs/reference/search/render-search-template-api.asciidoc @@ -4,6 +4,12 @@ Render search template ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs]. +-- + Renders a <> as a <>. diff --git a/docs/reference/search/retriever.asciidoc b/docs/reference/search/retriever.asciidoc index 58cc8ce9ef459..74e899df8d8ab 100644 --- a/docs/reference/search/retriever.asciidoc +++ b/docs/reference/search/retriever.asciidoc @@ -1,22 +1,25 @@ [[retriever]] === Retriever -preview::["This functionality is in technical preview and may be changed or removed in a future release. The syntax will likely change before GA. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] - -A retriever is a specification to describe top documents returned from a -search. A retriever replaces other elements of the <> +A retriever is a specification to describe top documents returned from a search. +A retriever replaces other elements of the <> that also return top documents such as <> and -<>. A retriever may have child retrievers where a -retriever with two or more children is considered a compound retriever. This -allows for complex behavior to be depicted in a tree-like structure, called -the retriever tree, to better clarify the order of operations that occur -during a search. +<>. +A retriever may have child retrievers where a retriever with two or more children is considered a compound retriever. +This allows for complex behavior to be depicted in a tree-like structure, called the retriever tree, which clarifies the order of operations that occur during a search. [TIP] ==== Refer to <> for a high level overview of the retrievers abstraction. +Refer to <> for additional examples. ==== +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs]. +-- + The following retrievers are available: `standard`:: @@ -25,12 +28,18 @@ A <> that replaces the functionality of a traditi `knn`:: A <> that replaces the functionality of a <>. +`rescorer`:: +A <> that replaces the functionality of the <>. + `rrf`:: A <> that produces top documents from <>. `text_similarity_reranker`:: A <> that enhances search results by re-ranking documents based on semantic similarity to a specified inference text, using a machine learning model. +`rule`:: +A <> that applies contextual <> to pin or exclude documents for specific queries. + [[standard-retriever]] ==== Standard Retriever @@ -46,8 +55,7 @@ Defines a query to retrieve a set of top documents. `filter`:: (Optional, <>) + -Applies a <> to this retriever -where all documents must match this query but do not contribute to the score. +Applies a <> to this retriever, where all documents must match this query but do not contribute to the score. `search_after`:: (Optional, <>) @@ -58,14 +66,13 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=terminate_after] `sort`:: + -(Optional, <>) -A sort object that that specifies the order of matching documents. +(Optional, <>) A sort object that specifies the order of matching documents. `min_score`:: (Optional, `float`) + -Minimum <> for matching documents. Documents with a -lower `_score` are not included in the top documents. +Minimum <> for matching documents. +Documents with a lower `_score` are not included in the top documents. `collapse`:: (Optional, <>) @@ -74,14 +81,76 @@ Collapses the top documents by a specified key into a single top document per ke ===== Restrictions -When a retriever tree contains a compound retriever (a retriever with two or more child -retrievers) *only* the query element is allowed. +When a retriever tree contains a compound retriever (a retriever with two or more child retrievers) the <> parameter is not supported. [discrete] [[standard-retriever-example]] ==== Example -[source,js] +//// +[source,console] +---- +PUT /restaurants +{ + "mappings": { + "properties": { + "region": { "type": "keyword" }, + "year": { "type": "keyword" }, + "vector": { + "type": "dense_vector", + "dims": 3 + } + } + } +} + +POST /restaurants/_bulk?refresh +{"index":{}} +{"region": "Austria", "year": "2019", "vector": [10, 22, 77]} +{"index":{}} +{"region": "France", "year": "2019", "vector": [10, 22, 78]} +{"index":{}} +{"region": "Austria", "year": "2020", "vector": [10, 22, 79]} +{"index":{}} +{"region": "France", "year": "2020", "vector": [10, 22, 80]} + +PUT /movies + +PUT _query_rules/my-ruleset +{ + "rules": [ + { + "rule_id": "my-rule1", + "type": "pinned", + "criteria": [ + { + "type": "exact", + "metadata": "query_string", + "values": [ "pugs" ] + } + ], + "actions": { + "ids": [ + "id1" + ] + } + } + ] +} + +---- +// TESTSETUP + +[source,console] +-------------------------------------------------- +DELETE /restaurants + +DELETE /movies +-------------------------------------------------- +// TEARDOWN +//// + +[source,console] ---- GET /restaurants/_search { @@ -109,12 +178,13 @@ GET /restaurants/_search } } ---- -// NOTCONSOLE + <1> Opens the `retriever` object. -<2> The `standard` retriever is used for definining traditional {es} queries. +<2> The `standard` retriever is used for defining traditional {es} queries. <3> The entry point for defining the search query. <4> The `bool` object allows for combining multiple query clauses logically. -<5> The `should` array indicates conditions under which a document will match. Documents matching these conditions will increase their relevancy score. +<5> The `should` array indicates conditions under which a document will match. +Documents matching these conditions will have increased relevancy scores. <6> The `match` object finds documents where the `region` field contains the word "Austria." <7> The `filter` array provides filtering conditions that must be met but do not contribute to the relevancy score. <8> The `term` object is used for exact matches, in this case, filtering documents by the `year` field. @@ -145,8 +215,8 @@ Defines a <> to build a query vector. `k`:: (Required, integer) + -Number of nearest neighbors to return as top hits. This value must be fewer than -or equal to `num_candidates`. +Number of nearest neighbors to return as top hits. +This value must be fewer than or equal to `num_candidates`. `num_candidates`:: (Required, integer) @@ -171,9 +241,9 @@ The parameters `query_vector` and `query_vector_builder` cannot be used together [[knn-retriever-example]] ==== Example -[source,js] +[source,console] ---- -GET my-embeddings/_search +GET /restaurants/_search { "retriever": { "knn": { <1> @@ -185,21 +255,19 @@ GET my-embeddings/_search } } ---- -// NOTCONSOLE - +// TEST[continued] <1> Configuration for k-nearest neighbor (knn) search, which is based on vector similarity. <2> Specifies the field name that contains the vectors. <3> The query vector against which document vectors are compared in the `knn` search. -<4> The number of nearest neighbors to return as top hits. This value must be fewer than or equal to `num_candidates`. +<4> The number of nearest neighbors to return as top hits. +This value must be fewer than or equal to `num_candidates`. <5> The size of the initial candidate set from which the final `k` nearest neighbors are selected. [[rrf-retriever]] ==== RRF Retriever -An <> retriever returns top documents based on the RRF formula, -equally weighting two or more child retrievers. -Reciprocal rank fusion (RRF) is a method for combining multiple result -sets with different relevance indicators into a single result set. +An <> retriever returns top documents based on the RRF formula, equally weighting two or more child retrievers. +Reciprocal rank fusion (RRF) is a method for combining multiple result sets with different relevance indicators into a single result set. ===== Parameters @@ -211,19 +279,13 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-rank-window-size] include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-filter] -===== Restrictions - -An RRF retriever is a compound retriever. Child retrievers may not use -elements that are restricted by having a compound retriever as part of -the retriever tree. - [discrete] [[rrf-retriever-example-hybrid]] ==== Example: Hybrid search A simple hybrid search example (lexical search + dense vector search) combining a `standard` retriever with a `knn` retriever using RRF: -[source,js] +[source,console] ---- GET /restaurants/_search { @@ -234,7 +296,7 @@ GET /restaurants/_search "standard": { <3> "query": { "multi_match": { - "query": "San Francisco", + "query": "Austria", "fields": [ "city", "region" @@ -258,7 +320,7 @@ GET /restaurants/_search } } ---- -// NOTCONSOLE +// TEST[continued] <1> Defines a retriever tree with an RRF retriever. <2> The sub-retriever array. <3> The first sub-retriever is a `standard` retriever. @@ -272,7 +334,7 @@ GET /restaurants/_search A more complex hybrid search example (lexical search + ELSER sparse vector search + dense vector search) using RRF: -[source,js] +[source,console] ---- GET movies/_search { @@ -316,7 +378,123 @@ GET movies/_search } } ---- -// NOTCONSOLE +// TEST[skip:uses ELSER] + +[[rescorer-retriever]] +==== Rescorer Retriever + +The `rescorer` retriever re-scores only the results produced by its child retriever. +For the `standard` and `knn` retrievers, the `window_size` parameter specifies the number of documents examined per shard. + +For compound retrievers like `rrf`, the `window_size` parameter defines the total number of documents examined globally. + +When using the `rescorer`, an error is returned if the following conditions are not met: + +* The minimum configured rescore's `window_size` is: +** Greater than or equal to the `size` of the parent retriever for nested `rescorer` setups. +** Greater than or equal to the `size` of the search request when used as the primary retriever in the tree. + +* And the maximum rescore's `window_size` is: +** Smaller than or equal to the `size` or `rank_window_size` of the child retriever. + +[discrete] +[[rescorer-retriever-parameters]] +===== Parameters + +`rescore`:: +(Required. <>) ++ +Defines the <> applied sequentially to the top documents returned by the child retriever. + +`retriever`:: +(Required. <>) ++ +Specifies the child retriever responsible for generating the initial set of top documents to be re-ranked. + +`filter`:: +(Optional. <>) ++ +Applies a <> to the retriever, ensuring that all documents match the filter criteria without affecting their scores. + +[discrete] +[[rescorer-retriever-example]] +==== Example + +The `rescorer` retriever can be placed at any level within the retriever tree. +The following example demonstrates a `rescorer` applied to the results produced by an `rrf` retriever: + +[source,console] +---- +GET movies/_search +{ + "size": 10, <1> + "retriever": { + "rescorer": { <2> + "rescore": { + "query": { <3> + "window_size": 50, <4> + "rescore_query": { + "script_score": { + "script": { + "source": "cosineSimilarity(params.queryVector, 'product-vector_final_stage') + 1.0", + "params": { + "queryVector": [-0.5, 90.0, -10, 14.8, -156.0] + } + } + } + } + } + }, + "retriever": { <5> + "rrf": { + "rank_window_size": 100, <6> + "retrievers": [ + { + "standard": { + "query": { + "sparse_vector": { + "field": "plot_embedding", + "inference_id": "my-elser-model", + "query": "films that explore psychological depths" + } + } + } + }, + { + "standard": { + "query": { + "multi_match": { + "query": "crime", + "fields": [ + "plot", + "title" + ] + } + } + } + }, + { + "knn": { + "field": "vector", + "query_vector": [10, 22, 77], + "k": 10, + "num_candidates": 10 + } + } + ] + } + } + } + } +} +---- +// TEST[skip:uses ELSER] +<1> Specifies the number of top documents to return in the final response. +<2> A `rescorer` retriever applied as the final step. +<3> The definition of the `query` rescorer. +<4> Defines the number of documents to rescore from the child retriever. +<5> Specifies the child retriever definition. +<6> Defines the number of documents returned by the `rrf` retriever, which limits the available documents to [[text-similarity-reranker-retriever]] ==== Text Similarity Re-ranker Retriever @@ -325,23 +503,26 @@ The `text_similarity_reranker` retriever uses an NLP model to improve search res [TIP] ==== -Refer to <> for a high level overview of semantic reranking. +Refer to <> for a high level overview of semantic re-ranking. ==== ===== Prerequisites -To use `text_similarity_reranker` you must first set up a `rerank` task using the <>. -The `rerank` task should be set up with a machine learning model that can compute text similarity. Refer to {ml-docs}/ml-nlp-model-ref.html#ml-nlp-model-ref-text-similarity[the Elastic NLP model reference] for a list of third-party text similarity models supported by {es}. +To use `text_similarity_reranker` you must first set up an inference endpoint for the `rerank` task using the <>. +The endpoint should be set up with a machine learning model that can compute text similarity. +Refer to {ml-docs}/ml-nlp-model-ref.html#ml-nlp-model-ref-text-similarity[the Elastic NLP model reference] for a list of third-party text similarity models supported by {es}. -Currently you can: +You have the following options: -* Integrate directly with the <> using the `rerank` task type -* Integrate directly with the <> using the `rerank` task type +* Use the the built-in <> cross-encoder model via the inference API's {es} service. +* Use the <> with the `rerank` task type. +* Use the <> with the `rerank` task type. * Upload a model to {es} with {eland-docs}/machine-learning.html#ml-nlp-pytorch[Eland] using the `text_similarity` NLP task type. -** Then set up an <> with the `rerank` task type +** Then set up an <> with the `rerank` task type. ** Refer to the <> on this page for a step-by-step guide. ===== Parameters + `retriever`:: (Required, <>) + @@ -350,7 +531,8 @@ The child retriever that generates the initial set of top documents to be re-ran `field`:: (Required, `string`) + -The document field to be used for text similarity comparisons. This field should contain the text that will be evaluated against the `inferenceText`. +The document field to be used for text similarity comparisons. +This field should contain the text that will be evaluated against the `inferenceText`. `inference_id`:: (Required, `string`) @@ -365,32 +547,88 @@ The text snippet used as the basis for similarity comparison. `rank_window_size`:: (Optional, `int`) + -The number of top documents to consider in the re-ranking process. Defaults to `10`. +The number of top documents to consider in the re-ranking process. +Defaults to `10`. `min_score`:: (Optional, `float`) + -Sets a minimum threshold score for including documents in the re-ranked results. Documents with similarity scores below this threshold will be excluded. Note that score calculations vary depending on the model used. +Sets a minimum threshold score for including documents in the re-ranked results. +Documents with similarity scores below this threshold will be excluded. +Note that score calculations vary depending on the model used. `filter`:: (Optional, <>) + Applies the specified <> to the child <>. -If the child retriever already specifies any filters, then this top-level filter is applied in conjuction -with the filter defined in the child retriever. +If the child retriever already specifies any filters, then this top-level filter is applied in conjuction with the filter defined in the child retriever. -===== Restrictions +[discrete] +[[text-similarity-reranker-retriever-example-elastic-rerank]] +==== Example: Elastic Rerank + +This examples demonstrates how to deploy the Elastic Rerank model and use it to re-rank search results using the `text_similarity_reranker` retriever. -A text similarity re-ranker retriever is a compound retriever. Child retrievers may not use elements that are restricted by having a compound retriever as part of the retriever tree. +Follow these steps: + +. Create an inference endpoint for the `rerank` task using the <>. ++ +[source,console] +---- +PUT _inference/rerank/my-elastic-rerank +{ + "service": "elasticsearch", + "service_settings": { + "model_id": ".rerank-v1", + "num_threads": 1, + "adaptive_allocations": { <1> + "enabled": true, + "min_number_of_allocations": 1, + "max_number_of_allocations": 10 + } + } +} +---- +// TEST[skip:uses ML] +<1> {ml-docs}/ml-nlp-auto-scale.html#nlp-model-adaptive-allocations[Adaptive allocations] will be enabled with the minimum of 1 and the maximum of 10 allocations. ++ +. Define a `text_similarity_rerank` retriever: ++ +[source,console] +---- +POST _search +{ + "retriever": { + "text_similarity_reranker": { + "retriever": { + "standard": { + "query": { + "match": { + "text": "How often does the moon hide the sun?" + } + } + } + }, + "field": "text", + "inference_id": "my-elastic-rerank", + "inference_text": "How often does the moon hide the sun?", + "rank_window_size": 100, + "min_score": 0.5 + } + } +} +---- +// TEST[skip:uses ML] [discrete] [[text-similarity-reranker-retriever-example-cohere]] ==== Example: Cohere Rerank -This example enables out-of-the-box semantic search by reranking top documents using the Cohere Rerank API. This approach eliminate the need to generate and store embeddings for all indexed documents. -This requires a <> using the `rerank` task type. +This example enables out-of-the-box semantic search by re-ranking top documents using the Cohere Rerank API. +This approach eliminates the need to generate and store embeddings for all indexed documents. +This requires a <> that is set up for the `rerank` task type. -[source,js] +[source,console] ---- GET /index/_search { @@ -414,11 +652,11 @@ GET /index/_search } } ---- -// NOTCONSOLE +// TEST[skip:uses ML] [discrete] [[text-similarity-reranker-retriever-example-eland]] -==== Example: Semantic reranking with a Hugging Face model +==== Example: Semantic re-ranking with a Hugging Face model The following example uses the `cross-encoder/ms-marco-MiniLM-L-6-v2` model from Hugging Face to rerank search results based on semantic similarity. The model must be uploaded to {es} using https://www.elastic.co/guide/en/elasticsearch/client/eland/current/machine-learning.html#ml-nlp-pytorch[Eland]. @@ -428,7 +666,7 @@ The model must be uploaded to {es} using https://www.elastic.co/guide/en/elastic Refer to {ml-docs}/ml-nlp-model-ref.html#ml-nlp-model-ref-text-similarity[the Elastic NLP model reference] for a list of third party text similarity models supported by {es}. ==== -Follow these steps to load the model and create a semantic reranker. +Follow these steps to load the model and create a semantic re-ranker. . Install Eland using `pip` + @@ -437,7 +675,9 @@ Follow these steps to load the model and create a semantic reranker. python -m pip install eland[pytorch] ---- + -. Upload the model to {es} using Eland. This example assumes you have an Elastic Cloud deployment and an API key. Refer to the https://www.elastic.co/guide/en/elasticsearch/client/eland/current/machine-learning.html#ml-nlp-pytorch-auth[Eland documentation] for more authentication options. +. Upload the model to {es} using Eland. +This example assumes you have an Elastic Cloud deployment and an API key. +Refer to the https://www.elastic.co/guide/en/elasticsearch/client/eland/current/machine-learning.html#ml-nlp-pytorch-auth[Eland documentation] for more authentication options. + [source,sh] ---- @@ -452,7 +692,7 @@ eland_import_hub_model \ + . Create an inference endpoint for the `rerank` task + -[source,js] +[source,console] ---- PUT _inference/rerank/my-msmarco-minilm-model { @@ -464,11 +704,11 @@ PUT _inference/rerank/my-msmarco-minilm-model } } ---- -// NOTCONSOLE +// TEST[skip:uses ML] + . Define a `text_similarity_rerank` retriever. + -[source,js] +[source,console] ---- POST movies/_search { @@ -490,35 +730,171 @@ POST movies/_search } } ---- -// NOTCONSOLE +// TEST[skip:uses ML] + This retriever uses a standard `match` query to search the `movie` index for films tagged with the genre "drama". It then re-ranks the results based on semantic similarity to the text in the `inference_text` parameter, using the model we uploaded to {es}. +[[rule-retriever]] +==== Query Rules Retriever + +The `rule` retriever enables fine-grained control over search results by applying contextual <> to pin or exclude documents for specific queries. +This retriever has similar functionality to the <>, but works out of the box with other retrievers. + +===== Prerequisites + +To use the `rule` retriever you must first create one or more query rulesets using the <>. + +[discrete] +[[rule-retriever-parameters]] +===== Parameters + +`retriever`:: +(Required, <>) ++ +The child retriever that returns the results to apply query rules on top of. +This can be a standalone retriever such as the <> or <> retriever, or it can be a compound retriever. + +`ruleset_ids`:: +(Required, `array`) ++ +An array of one or more unique <> IDs with query-based rules to match and apply as applicable. +Rulesets and their associated rules are evaluated in the order in which they are specified in the query and ruleset. +The maximum number of rulesets to specify is 10. + +`match_criteria`:: +(Required, `object`) ++ +Defines the match criteria to apply to rules in the given query ruleset(s). +Match criteria should match the keys defined in the `criteria.metadata` field of the rule. + +`rank_window_size`:: +(Optional, `int`) ++ +The number of top documents to return from the `rule` retriever. +Defaults to `10`. + +[discrete] +[[rule-retriever-example]] +==== Example: Rule retriever + +This example shows the rule retriever executed without any additional retrievers. +It runs the query defined by the `retriever` and applies the rules from `my-ruleset` on top of the returned results. + +[source,console] +---- +GET movies/_search +{ + "retriever": { + "rule": { + "match_criteria": { + "query_string": "harry potter" + }, + "ruleset_ids": [ + "my-ruleset" + ], + "retriever": { + "standard": { + "query": { + "query_string": { + "query": "harry potter" + } + } + } + } + } + } +} +---- + +[discrete] +[[rule-retriever-example-rrf]] +==== Example: Rule retriever combined with RRF + +This example shows how to combine the `rule` retriever with other rerank retrievers such as <> or <>. + +[WARNING] +==== +The `rule` retriever will apply rules to any documents returned from its defined `retriever` or any of its sub-retrievers. +This means that for the best results, the `rule` retriever should be the outermost defined retriever. +Nesting a `rule` retriever as a sub-retriever under a reranker such as `rrf` or `text_similarity_reranker` may not produce the expected results. +==== + +[source,console] +---- +GET movies/_search +{ + "retriever": { + "rule": { <1> + "match_criteria": { + "query_string": "harry potter" + }, + "ruleset_ids": [ + "my-ruleset" + ], + "retriever": { + "rrf": { <2> + "retrievers": [ + { + "standard": { + "query": { + "query_string": { + "query": "sorcerer's stone" + } + } + } + }, + { + "standard": { + "query": { + "query_string": { + "query": "chamber of secrets" + } + } + } + } + ] + } + } + } + } +} +---- + +<1> The `rule` retriever is the outermost retriever, applying rules to the search results that were previously reranked using the `rrf` retriever. +<2> The `rrf` retriever returns results from all of its sub-retrievers, and the output of the `rrf` retriever is used as input to the `rule` retriever. + +[discrete] +[[retriever-common-parameters]] +=== Common usage guidelines + +[discrete] +[[retriever-size-pagination]] ==== Using `from` and `size` with a retriever tree The <> and <> parameters are provided globally as part of the general -<>. They are applied to all retrievers in a -retriever tree unless a specific retriever overrides the `size` parameter -using a different parameter such as `rank_window_size`. Though, the final -search hits are always limited to `size`. +<>. +They are applied to all retrievers in a retriever tree, unless a specific retriever overrides the `size` parameter using a different parameter such as `rank_window_size`. +Though, the final search hits are always limited to `size`. +[discrete] +[[retriever-aggregations]] ==== Using aggregations with a retriever tree <> are globally specified as part of a search request. The query used for an aggregation is the combination of all leaf retrievers as `should` clauses in a <>. +[discrete] +[[retriever-restrictions]] ==== Restrictions on search parameters when specifying a retriever -When a retriever is specified as part of a search the following elements are not allowed -at the top-level and instead are only allowed as elements of specific retrievers: +When a retriever is specified as part of a search, the following elements are not allowed at the top-level: * <> * <> * <> * <> * <> -* <> -* <> +* <> use a <> instead diff --git a/docs/reference/search/rrf.asciidoc b/docs/reference/search/rrf.asciidoc index 2525dfff23b94..842bd7049e3bf 100644 --- a/docs/reference/search/rrf.asciidoc +++ b/docs/reference/search/rrf.asciidoc @@ -1,8 +1,6 @@ [[rrf]] === Reciprocal rank fusion -preview::["This functionality is in technical preview and may be changed or removed in a future release. The syntax will likely change before GA. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] - https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf[Reciprocal rank fusion (RRF)] is a method for combining multiple result sets with different relevance indicators into a single result set. RRF requires no tuning, and the different relevance indicators do not have to be related to each other to achieve high-quality results. @@ -29,6 +27,12 @@ return score [[rrf-api]] ==== Reciprocal rank fusion API +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs]. +-- + You can use RRF as part of a <> to combine and rank documents using separate sets of top documents (result sets) from a combination of <> using an <>. A minimum of *two* child retrievers is required for ranking. @@ -95,20 +99,22 @@ The `rrf` retriever supports: * <> * <> +* <> +* <> +* <> +* <> The `rrf` retriever does not currently support: * <> -* <> * <> * <> -* <> -* <> -* <> -* <> Using unsupported features as part of a search with an `rrf` retriever results in an exception. +IMPORTANT: It is best to avoid providing a <> as part of the request, as +RRF creates one internally that is shared by all sub-retrievers to ensure consistent results. + [[rrf-using-multiple-standard-retrievers]] ==== Reciprocal rank fusion using multiple standard retrievers @@ -300,13 +306,12 @@ We have both the ranker's `score` and the `_rank` option to show our top-ranked "value" : 5, "relation" : "eq" }, - "max_score" : null, + "max_score" : ..., "hits" : [ { "_index" : "example-index", "_id" : "3", "_score" : 0.8333334, - "_rank" : 1, "_source" : { "integer" : 1, "vector" : [ @@ -319,7 +324,6 @@ We have both the ranker's `score` and the `_rank` option to show our top-ranked "_index" : "example-index", "_id" : "2", "_score" : 0.5833334, - "_rank" : 2, "_source" : { "integer" : 2, "vector" : [ @@ -332,7 +336,6 @@ We have both the ranker's `score` and the `_rank` option to show our top-ranked "_index" : "example-index", "_id" : "4", "_score" : 0.5, - "_rank" : 3, "_source" : { "integer" : 2, "text" : "rrf rrf rrf rrf" @@ -499,7 +502,6 @@ Working with the example above, and by adding `explain=true` to the search reque "_index": "example-index", "_id": "3", "_score": 0.8333334, - "_rank": 1, "_explanation": { "value": 0.8333334, <1> @@ -608,7 +610,6 @@ The response would now include the named query in the explanation: "_index": "example-index", "_id": "3", "_score": 0.8333334, - "_rank": 1, "_explanation": { "value": 0.8333334, @@ -708,3 +709,99 @@ So for the same params as above, we would now have: * `from=0, size=2` would return [`1`, `5`] with ranks `[1, 2]` * `from=2, size=2` would return an empty result set as it would fall outside the available `rank_window_size` results. + +==== Aggregations in RRF + +The `rrf` retriever supports aggregations from all specified sub-retrievers. Important notes about aggregations: + +* They operate on the complete result set from all sub-retrievers +* They are not limited by the `rank_window_size` parameter +* They process the union of all matching documents + +For example, consider the following document set: +[source,js] +---- +{ + "_id": 1, "termA": "foo", + "_id": 2, "termA": "foo", "termB": "bar", + "_id": 3, "termA": "aardvark", "termB": "bar", + "_id": 4, "termA": "foo", "termB": "bar" +} +---- +// NOTCONSOLE + +Perform a term aggregation on the `termA` field using an `rrf` retriever: +[source,js] +---- +{ + "retriever": { + "rrf": { + "retrievers": [ + { + "standard": { + "query": { + "term": { + "termB": "bar" + } + } + } + }, + { + "standard": { + "query": { + "match_all": { } + } + } + } + ], + "rank_window_size": 1 + } + }, + "size": 1, + "aggs": { + "termA_agg": { + "terms": { + "field": "termA" + } + } + } +} +---- +// NOTCONSOLE + +The aggregation results will include *all* matching documents, regardless of `rank_window_size`. +[source, js] +---- +{ + "foo": 3, + "aardvark": 1 +} + +---- +// NOTCONSOLE + +==== Highlighting in RRF + +Using the `rrf` retriever, you can add <> to show relevant text snippets in your search results. Highlighted snippets are computed based +on the matching text queries defined on the sub-retrievers. + +IMPORTANT: Highlighting on vector fields, using either the `knn` retriever or a `knn` query, is not supported. + +A more specific example of highlighting in RRF can also be found in the <> page. + +==== Inner hits in RRF + +The `rrf` retriever supports <> functionality, allowing you to retrieve +related nested or parent/child documents alongside your main search results. Inner hits can be +specified as part of any nested sub-retriever and will be propagated to the top-level parent +retriever. Note that the inner hit computation will take place only at end of `rrf` retriever's +evaluation on the top matching documents, and not as part of the query execution of the nested +sub-retrievers. + +[IMPORTANT] +==== +When defining multiple `inner_hits` sections across sub-retrievers: + +* Each `inner_hits` section must have a unique name +* Names must be unique across all sub-retrievers in the search request +==== diff --git a/docs/reference/search/scroll-api.asciidoc b/docs/reference/search/scroll-api.asciidoc index e3b4123ddff68..0f89df877ba74 100644 --- a/docs/reference/search/scroll-api.asciidoc +++ b/docs/reference/search/scroll-api.asciidoc @@ -8,6 +8,12 @@ IMPORTANT: We no longer recommend using the scroll API for deep pagination. If you need to preserve the index state while paging through more than 10,000 hits, use the <> parameter with a point in time (PIT). +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs]. +-- + Retrieves the next batch of results for a <>. diff --git a/docs/reference/search/search-shards.asciidoc b/docs/reference/search/search-shards.asciidoc index 49045acf4c484..13f9ae8772858 100644 --- a/docs/reference/search/search-shards.asciidoc +++ b/docs/reference/search/search-shards.asciidoc @@ -4,6 +4,12 @@ Search shards ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs]. +-- + Returns the indices and shards that a search request would be executed against. [source,console] diff --git a/docs/reference/search/search-template-api.asciidoc b/docs/reference/search/search-template-api.asciidoc index c60b5281c05e5..2094ee8924014 100644 --- a/docs/reference/search/search-template-api.asciidoc +++ b/docs/reference/search/search-template-api.asciidoc @@ -4,6 +4,12 @@ Search template ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs]. +-- + Runs a search with a <>. //// diff --git a/docs/reference/search/search-vector-tile-api.asciidoc b/docs/reference/search/search-vector-tile-api.asciidoc index 2cdc29918a699..f63abda6fcb47 100644 --- a/docs/reference/search/search-vector-tile-api.asciidoc +++ b/docs/reference/search/search-vector-tile-api.asciidoc @@ -4,6 +4,11 @@ Vector tile search ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs]. +-- Searches a vector tile for geospatial values. Returns results as a binary https://docs.mapbox.com/vector-tiles/specification[Mapbox vector tile]. diff --git a/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc b/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc index 6b9b13b124e9f..a68f20fb1c656 100644 --- a/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc +++ b/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc @@ -1,24 +1,26 @@ |==== -| 19+^h| Remote cluster version +| 21+^h| Remote cluster version h| Local cluster version - | 6.8 | 7.1–7.16 | 7.17 | 8.0 | 8.1 | 8.2 | 8.3 | 8.4 | 8.5 | 8.6 | 8.7 | 8.8 | 8.9 | 8.10 | 8.11 | 8.12 | 8.13 | 8.14 | 8.15 -| 6.8 | {yes-icon} | {yes-icon} | {yes-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} -| 7.1–7.16 | {yes-icon} | {yes-icon} | {yes-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} -| 7.17 | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.0 | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.1 | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.2 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.3 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.4 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.5 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.6 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.7 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.8 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.9 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.10 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.11 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.12 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.13 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.14 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.15 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} + | 6.8 | 7.1–7.16 | 7.17 | 8.0 | 8.1 | 8.2 | 8.3 | 8.4 | 8.5 | 8.6 | 8.7 | 8.8 | 8.9 | 8.10 | 8.11 | 8.12 | 8.13 | 8.14 | 8.15 | 8.16 | 8.17 +| 6.8 | {yes-icon} | {yes-icon} | {yes-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} +| 7.1–7.16 | {yes-icon} | {yes-icon} | {yes-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} +| 7.17 | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.0 | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.1 | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.2 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.3 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.4 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.5 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.6 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.7 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.8 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.9 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.10 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.11 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.12 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.13 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.14 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.15 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.16 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.17 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} |==== diff --git a/docs/reference/search/search-your-data/full-text-search.asciidoc b/docs/reference/search/search-your-data/full-text-search.asciidoc new file mode 100644 index 0000000000000..8641d0e45748a --- /dev/null +++ b/docs/reference/search/search-your-data/full-text-search.asciidoc @@ -0,0 +1,82 @@ +[[full-text-search]] +== Full-text search + +.Hands-on introduction to full-text search +[TIP] +==== +Would you prefer to jump straight into a hands-on tutorial? +Refer to our quick start <>. +==== + +Full-text search, also known as lexical search, is a technique for fast, efficient searching through text fields in documents. +Documents and search queries are transformed to enable returning https://www.elastic.co/what-is/search-relevance[relevant] results instead of simply exact term matches. +Fields of type <> are analyzed and indexed for full-text search. + +Built on decades of information retrieval research, full-text search delivers reliable results that scale predictably as your data grows. Because it runs efficiently on CPUs, {es}'s full-text search requires minimal computational resources compared to GPU-intensive vector operations. + +You can combine full-text search with <> to build modern hybrid search applications. While vector search may require additional GPU resources, the full-text component remains cost-effective by leveraging existing CPU infrastructure. + +[discrete] +[[full-text-search-how-it-works]] +=== How full-text search works + +The following diagram illustrates the components of full-text search. + +image::images/search/full-text-search-overview.svg[Components of full-text search from analysis to relevance scoring, align=center, width=500] + +At a high level, full-text search involves the following: + +* <>: Analysis consists of a pipeline of sequential transformations. Text is transformed into a format optimized for searching using techniques such as stemming, lowercasing, and stop word elimination. {es} contains a number of built-in <> and tokenizers, including options to analyze specific language text. You can also create custom analyzers. ++ +[TIP] +==== +Refer to <> to learn how to test an analyzer and inspect the tokens and metadata it generates. +==== +* *Inverted index creation*: After analysis is complete, {es} builds an inverted index from the resulting tokens. +An inverted index is a data structure that maps each token to the documents that contain it. +It's made up of two key components: +** *Dictionary*: A sorted list of all unique terms in the collection of documents in your index. +** *Posting list*: For each term, a list of document IDs where the term appears, along with optional metadata like term frequency and position. +* *Relevance scoring*: Results are ranked by how relevant they are to the given query. The relevance score of each document is represented by a positive floating-point number called the `_score`. The higher the `_score`, the more relevant the document. ++ +The default <> {es} uses for calculating relevance scores is https://en.wikipedia.org/wiki/Okapi_BM25[Okapi BM25], a variation of the https://en.wikipedia.org/wiki/Tf–idf[TF-IDF algorithm]. BM25 calculates relevance scores based on term frequency, document frequency, and document length. +Refer to this https://www.elastic.co/blog/practical-bm25-part-2-the-bm25-algorithm-and-its-variables[technical blog post] for a deep dive into BM25. +* *Full-text search query*: Query text is analyzed <>, and the resulting tokens are used to search the inverted index. ++ +Query DSL supports a number of <>. ++ +As of 8.17, {esql} also supports <> functions. + +[discrete] +[[full-text-search-getting-started]] +=== Getting started + +For a hands-on introduction to full-text search, refer to the <>. + +[discrete] +[[full-text-search-learn-more]] +=== Learn more + +Here are some resources to help you learn more about full-text search with {es}. + +*Core concepts* + +Learn about the core components of full-text search: + +* <> +* <> +** <> +** <> + +*{es} query languages* + +Learn how to build full-text search queries using {es}'s query languages: + +* <> +* <> + +*Advanced topics* + +For a technical deep dive into {es}'s BM25 implementation read this blog post: https://www.elastic.co/blog/practical-bm25-part-2-the-bm25-algorithm-and-its-variables[The BM25 Algorithm and its Variables]. + +To learn how to optimize the relevance of your search results, refer to <>. \ No newline at end of file diff --git a/docs/reference/search/search-your-data/ingest-vectors.asciidoc b/docs/reference/search/search-your-data/ingest-vectors.asciidoc new file mode 100644 index 0000000000000..f288293d2b03a --- /dev/null +++ b/docs/reference/search/search-your-data/ingest-vectors.asciidoc @@ -0,0 +1,141 @@ +[[bring-your-own-vectors]] +=== Bring your own dense vector embeddings to {es} +++++ +Bring your own dense vectors +++++ + +This tutorial demonstrates how to index documents that already have dense vector embeddings into {es}. +You'll also learn the syntax for searching these documents using a `knn` query. + +You'll find links at the end of this tutorial for more information about deploying a text embedding model in {es}, so you can generate embeddings for queries on the fly. + +[TIP] +==== +This is an advanced use case. +Refer to <> for an overview of your options for semantic search with {es}. +==== + +[discrete] +[[bring-your-own-vectors-create-index]] +=== Step 1: Create an index with `dense_vector` mapping + +Each document in our simple dataset will have: + +* A review: stored in a `review_text` field +* An embedding of that review: stored in a `review_vector` field +** The `review_vector` field is defined as a <> data type. + +[TIP] +==== +The `dense_vector` type automatically uses `int8_hnsw` quantization by default to reduce the memory footprint required when searching float vectors. +Learn more about balancing performance and accuracy in <>. +==== + +[source,console] +---- +PUT /amazon-reviews +{ + "mappings": { + "properties": { + "review_vector": { + "type": "dense_vector", + "dims": 8, <1> + "index": true, <2> + "similarity": "cosine" <3> + }, + "review_text": { + "type": "text" + } + } + } +} +---- +// TEST SETUP +<1> The `dims` parameter must match the length of the embedding vector. Here we're using a simple 8-dimensional embedding for readability. If not specified, `dims` will be dynamically calculated based on the first indexed document. +<2> The `index` parameter is set to `true` to enable the use of the `knn` query. +<3> The `similarity` parameter defines the similarity function used to compare the query vector to the document vectors. `cosine` is the default similarity function for `dense_vector` fields in {es}. + +[discrete] +[[bring-your-own-vectors-index-documents]] +=== Step 2: Index documents with embeddings + +[discrete] +==== Index a single document + +First, index a single document to understand the document structure. + +[source,console] +---- +PUT /amazon-reviews/_doc/1 +{ + "review_text": "This product is lifechanging! I'm telling all my friends about it.", + "review_vector": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8] <1> +} +---- +// TEST +<1> The size of the `review_vector` array is 8, matching the `dims` count specified in the mapping. + +[discrete] +==== Bulk index multiple documents + +In a production scenario, you'll want to index many documents at once using the <>. + +Here's an example of indexing multiple documents in a single `_bulk` request. + +[source,console] +---- +POST /_bulk +{ "index": { "_index": "amazon-reviews", "_id": "2" } } +{ "review_text": "This product is amazing! I love it.", "review_vector": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8] } +{ "index": { "_index": "amazon-reviews", "_id": "3" } } +{ "review_text": "This product is terrible. I hate it.", "review_vector": [0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1] } +{ "index": { "_index": "amazon-reviews", "_id": "4" } } +{ "review_text": "This product is great. I can do anything with it.", "review_vector": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8] } +{ "index": { "_index": "amazon-reviews", "_id": "5" } } +{ "review_text": "This product has ruined my life and the lives of my family and friends.", "review_vector": [0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1] } +---- +// TEST[continued] + +[discrete] +[[bring-your-own-vectors-search-documents]] +=== Step 3: Search documents with embeddings + +Now you can query these document vectors using a <>. +`knn` is a type of vector search, which finds the `k` most similar documents to a query vector. +Here we're simply using a raw vector for the query text, for demonstration purposes. + +[source,console] +---- +POST /amazon-reviews/_search +{ + "retriever": { + "knn": { + "field": "review_vector", + "query_vector": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8], <1> + "k": 2, <2> + "num_candidates": 5 <3> + } + } +} +---- +// TEST[skip:flakeyknnerror] +<1> In this simple example, we're sending a raw vector as the query text. In a real-world scenario, you'll need to generate vectors for queries using an embedding model. +<2> The `k` parameter specifies the number of results to return. +<3> The `num_candidates` parameter is optional. It limits the number of candidates returned by the search node. This can improve performance and reduce costs. + +[discrete] +[[bring-your-own-vectors-learn-more]] +=== Learn more + +In this simple example, we're sending a raw vector for the query text. +In a real-world scenario you won't know the query text ahead of time. +You'll need to generate query vectors, on the fly, using the same embedding model that generated the document vectors. + +For this you'll need to deploy a text embedding model in {es} and use the <>. Alternatively, you can generate vectors client-side and send them directly with the search request. + +Learn how to <> for semantic search. + +[TIP] +==== +If you're just getting started with vector search in {es}, refer to <>. +==== diff --git a/docs/reference/search/search-your-data/knn-search.asciidoc b/docs/reference/search/search-your-data/knn-search.asciidoc index 70cf9eec121d7..6fb7f1747051f 100644 --- a/docs/reference/search/search-your-data/knn-search.asciidoc +++ b/docs/reference/search/search-your-data/knn-search.asciidoc @@ -1149,3 +1149,95 @@ POST product-index/_search ---- //TEST[continued] +[discrete] +[[dense-vector-knn-search-reranking]] +==== Oversampling and rescoring for quantized vectors + +All forms of quantization will result in some accuracy loss and as the quantization level increases the accuracy loss will also increase. +Generally, we have found that: +- `int8` requires minimal if any rescoring +- `int4` requires some rescoring for higher accuracy and larger recall scenarios. Generally, oversampling by 1.5x-2x recovers most of the accuracy loss. +- `bbq` requires rescoring except on exceptionally large indices or models specifically designed for quantization. We have found that between 3x-5x oversampling is generally sufficient. But for fewer dimensions or vectors that do not quantize well, higher oversampling may be required. + +There are two main ways to oversample and rescore. The first is to utilize the <> in the `_search` request. + +Here is an example using the top level `knn` search with oversampling and using `rescore` to rerank the results: + +[source,console] +-------------------------------------------------- +POST /my-index/_search +{ + "size": 10, <1> + "knn": { + "query_vector": [0.04283529, 0.85670587, -0.51402352, 0], + "field": "my_int4_vector", + "k": 20, <2> + "num_candidates": 50 + }, + "rescore": { + "window_size": 20, <3> + "query": { + "rescore_query": { + "script_score": { + "query": { + "match_all": {} + }, + "script": { + "source": "(dotProduct(params.queryVector, 'my_int4_vector') + 1.0)", <4> + "params": { + "queryVector": [0.04283529, 0.85670587, -0.51402352, 0] + } + } + } + }, + "query_weight": 0, <5> + "rescore_query_weight": 1 <6> + } + } +} +-------------------------------------------------- +// TEST[skip: setup not provided] +<1> The number of results to return, note its only 10 and we will oversample by 2x, gathering 20 nearest neighbors. +<2> The number of results to return from the KNN search. This will do an approximate KNN search with 50 candidates +per HNSW graph and use the quantized vectors, returning the 20 most similar vectors +according to the quantized score. Additionally, since this is the top-level `knn` object, the global top 20 results +will from all shards will be gathered before rescoring. Combining with `rescore`, this is oversampling by `2x`, meaning +gathering 20 nearest neighbors according to quantized scoring and rescoring with higher fidelity float vectors. +<3> The number of results to rescore, if you want to rescore all results, set this to the same value as `k` +<4> The script to rescore the results. Script score will interact directly with the originally provided float32 vector. +<5> The weight of the original query, here we simply throw away the original score +<6> The weight of the rescore query, here we only use the rescore query + +The second way is to score per shard with the <> and <>. Generally, this means that there will be more rescoring per shard, but this +can increase overall recall at the cost of compute. + +[source,console] +-------------------------------------------------- +POST /my-index/_search +{ + "size": 10, <1> + "query": { + "script_score": { + "query": { + "knn": { <2> + "query_vector": [0.04283529, 0.85670587, -0.51402352, 0], + "field": "my_int4_vector", + "num_candidates": 20 <3> + } + }, + "script": { + "source": "(dotProduct(params.queryVector, 'my_int4_vector') + 1.0)", <4> + "params": { + "queryVector": [0.04283529, 0.85670587, -0.51402352, 0] + } + } + } + } +} +-------------------------------------------------- +// TEST[skip: setup not provided] +<1> The number of results to return +<2> The `knn` query to perform the initial search, this is executed per-shard +<3> The number of candidates to use for the initial approximate `knn` search. This will search using the quantized vectors +and return the top 20 candidates per shard to then be scored +<4> The script to score the results. Script score will interact directly with the originally provided float32 vector. diff --git a/docs/reference/search/search-your-data/retrieval-augmented-generation.asciidoc b/docs/reference/search/search-your-data/retrieval-augmented-generation.asciidoc new file mode 100644 index 0000000000000..2958999ede91d --- /dev/null +++ b/docs/reference/search/search-your-data/retrieval-augmented-generation.asciidoc @@ -0,0 +1,76 @@ +[rag-elasticsearch] +== Retrieval augmented generation + +.🍿 Prefer a video introduction? +*********************** +Check out https://www.youtube.com/watch?v=OS4ZefUPAks[this short video] from the Elastic Snackable Series. +*********************** + +Retrieval Augmented Generation (RAG) is a technique for improving language model responses by grounding the model with additional, verifiable sources of information. It works by first retrieving relevant context from an external datastore, which is then added to the model's context window. + +RAG is a form of https://arxiv.org/abs/2301.00234[in-context learning], where the model learns from information provided at inference time. +Compared to fine-tuning or continuous pre-training, RAG can be implemented more quickly and cheaply, and offers several advantages. + +image::images/search/rag-venn-diagram.svg[RAG sits at the intersection of information retrieval and generative AI, align=center, width=500] + +RAG sits at the intersection of https://www.elastic.co/what-is/information-retrieval[information retrieval] and generative AI. +{es} is an excellent tool for implementing RAG, because it offers various retrieval capabilities, such as full-text search, vector search, and hybrid search, as well as other tools like filtering, aggregations, and security features. + +[discrete] +[[rag-elasticsearch-advantages]] +=== Advantages of RAG + +Implementing RAG with {es} has several advantages: + +* *Improved context:* Enables grounding the language model with additional, up-to-date, and/or private data. +* *Reduced hallucination:* Helps minimize factual errors by enabling models to cite authoritative sources. +* *Cost efficiency:* Requires less maintenance compared to fine-tuning or continuously pre-training models. +* *Built-in security:* Controls data access by leveraging {es}'s <> features, such as role-based access control and field/document-level security. +* *Simplified response parsing:* Eliminates the need for custom parsing logic by letting the language model handle parsing {es} responses and formatting the retrieved context. +* *Flexible implementation:* Works with basic <>, and can be gradually updated to add more advanced and computationally intensive <> capabilities. + +[discrete] +[[rag-elasticsearch-components]] +=== RAG system overview + +The following diagram illustrates a simple RAG system using {es}. + +image::images/search/rag-schema.svg[Components of a simple RAG system using Elasticsearch, align=center, role="stretch"] + +The workflow is as follows: + +. The user submits a query. +. Elasticsearch retrieves relevant documents using full-text search, vector search, or hybrid search. +. The language model processes the context and generates a response, using custom instructions. Examples of custom instructions include "Cite a source" or "Provide a concise summary of the `content` field in markdown format." +. The model returns the final response to the user. + +[TIP] +==== +A more advanced setup might include query rewriting between steps 1 and 2. This intermediate step could use one or more additional language models with different instructions to reformulate queries for more specific and detailed responses. +==== + +[discrete] +[[rag-elasticsearch-getting-started]] +=== Getting started + +Start building RAG applications quickly with Playground, which seamlessly integrates {es} with language model providers. +The Playground UI enables you to build, test, and deploy RAG interfaces on top of your {es} indices. + +Playground automatically selects the best retrieval methods for your data, while providing full control over the final {es} queries and language model instructions. +You can also download the underlying Python code to integrate with your existing applications. + +Learn more in the {kibana-ref}/playground.html[Playground documentation] and +try the https://www.elastic.co/demo-gallery/ai-playground[interactive lab] for hands-on experience. + +[discrete] +[[rag-elasticsearch-learn-more]] +=== Learn more + +Learn more about building RAG systems using {es} in these blog posts: + +* https://www.elastic.co/blog/beyond-rag-basics-semantic-search-with-elasticsearch[Beyond RAG Basics: Advanced strategies for AI applications] +* https://www.elastic.co/search-labs/blog/building-a-rag-system-with-gemma-hugging-face-elasticsearch[Building a RAG system with Gemma, Hugging Face, and Elasticsearch] +* https://www.elastic.co/search-labs/blog/rag-agent-tool-elasticsearch-langchain[Building an agentic RAG tool with Elasticsearch and Langchain] + + + diff --git a/docs/reference/search/search-your-data/retrievers-examples.asciidoc b/docs/reference/search/search-your-data/retrievers-examples.asciidoc new file mode 100644 index 0000000000000..ad1cc32dcee01 --- /dev/null +++ b/docs/reference/search/search-your-data/retrievers-examples.asciidoc @@ -0,0 +1,1262 @@ +[[retrievers-examples]] + +Learn how to combine different retrievers in these hands-on examples. + +=== Retrievers examples + +[discrete] +[[retrievers-examples-setup]] +==== Add example data + +To begin with, lets create the `retrievers_example` index, and add some documents to it. + +[source,console] +---- +PUT retrievers_example +{ + "mappings": { + "properties": { + "vector": { + "type": "dense_vector", + "dims": 3, + "similarity": "l2_norm", + "index": true + }, + "text": { + "type": "text" + }, + "year": { + "type": "integer" + }, + "topic": { + "type": "keyword" + } + } + } +} + +POST /retrievers_example/_doc/1 +{ + "vector": [0.23, 0.67, 0.89], + "text": "Large language models are revolutionizing information retrieval by boosting search precision, deepening contextual understanding, and reshaping user experiences in data-rich environments.", + "year": 2024, + "topic": ["llm", "ai", "information_retrieval"] +} + +POST /retrievers_example/_doc/2 +{ + "vector": [0.12, 0.56, 0.78], + "text": "Artificial intelligence is transforming medicine, from advancing diagnostics and tailoring treatment plans to empowering predictive patient care for improved health outcomes.", + "year": 2023, + "topic": ["ai", "medicine"] +} + +POST /retrievers_example/_doc/3 +{ + "vector": [0.45, 0.32, 0.91], + "text": "AI is redefining security by enabling advanced threat detection, proactive risk analysis, and dynamic defenses against increasingly sophisticated cyber threats.", + "year": 2024, + "topic": ["ai", "security"] +} + +POST /retrievers_example/_doc/4 +{ + "vector": [0.34, 0.21, 0.98], + "text": "Elastic introduces Elastic AI Assistant, the open, generative AI sidekick powered by ESRE to democratize cybersecurity and enable users of every skill level.", + "year": 2023, + "topic": ["ai", "elastic", "assistant"] +} + +POST /retrievers_example/_doc/5 +{ + "vector": [0.11, 0.65, 0.47], + "text": "Learn how to spin up a deployment of our hosted Elasticsearch Service and use Elastic Observability to gain deeper insight into the behavior of your applications and systems.", + "year": 2024, + "topic": ["documentation", "observability", "elastic"] +} + +POST /retrievers_example/_refresh + +---- +// TESTSETUP + +Now that we have our documents in place, let's try to run some queries using retrievers. + +[discrete] +[[retrievers-examples-combining-standard-knn-retrievers-with-rrf]] +==== Example: Combining query and kNN with RRF + +First, let's examine how to combine two different types of queries: a `kNN` query and a +`query_string` query. While these queries may produce scores in different ranges, we can use +Reciprocal Rank Fusion (`rrf`) to combine the results and generate a merged final result +list. + +To implement this in the retriever framework, we start with the top-level element: our `rrf` +retriever. This retriever operates on top of two other retrievers: a `knn` retriever and a +`standard` retriever. Our query structure would look like this: + +[source,console] +---- +GET /retrievers_example/_search +{ + "retriever": { + "rrf": { + "retrievers": [ + { + "standard": { + "query": { + "query_string": { + "query": "(information retrieval) OR (artificial intelligence)", + "default_field": "text" + } + } + } + }, + { + "knn": { + "field": "vector", + "query_vector": [ + 0.23, + 0.67, + 0.89 + ], + "k": 3, + "num_candidates": 5 + } + } + ], + "rank_window_size": 10, + "rank_constant": 1 + } + }, + "_source": false +} +---- +// TEST + +This returns the following response based on the final rrf score for each result. + +.Example response +[%collapsible] +============== +[source,console-result] +---- +{ + "took": 42, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 3, + "relation": "eq" + }, + "max_score": 0.8333334, + "hits": [ + { + "_index": "retrievers_example", + "_id": "1", + "_score": 0.8333334 + }, + { + "_index": "retrievers_example", + "_id": "2", + "_score": 0.8333334 + }, + { + "_index": "retrievers_example", + "_id": "3", + "_score": 0.25 + } + ] + } +} +---- +// TESTRESPONSE[s/"took": 42/"took": $body.took/] +============== + +[discrete] +[[retrievers-examples-collapsing-retriever-results]] +==== Example: Grouping results by year with `collapse` + +In our result set, we have many documents with the same `year` value. We can clean this +up using the `collapse` parameter with our retriever. This, as with the standard <> feature, +enables grouping results by any field and returns only the highest-scoring document from each group. In this example +we'll collapse our results based on the `year` field. + +[source,console] +---- +GET /retrievers_example/_search +{ + "retriever": { + "rrf": { + "retrievers": [ + { + "standard": { + "query": { + "query_string": { + "query": "(information retrieval) OR (artificial intelligence)", + "default_field": "text" + } + } + } + }, + { + "knn": { + "field": "vector", + "query_vector": [ + 0.23, + 0.67, + 0.89 + ], + "k": 3, + "num_candidates": 5 + } + } + ], + "rank_window_size": 10, + "rank_constant": 1 + } + }, + "collapse": { + "field": "year", + "inner_hits": { + "name": "topic related documents", + "_source": [ + "year" + ] + } + }, + "_source": false +} +---- +// TEST[continued] + +This returns the following response with collapsed results. + +.Example response +[%collapsible] +============== +[source,console-result] +---- +{ + "took": 42, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 3, + "relation": "eq" + }, + "max_score": 0.8333334, + "hits": [ + { + "_index": "retrievers_example", + "_id": "1", + "_score": 0.8333334, + "fields": { + "year": [ + 2024 + ] + }, + "inner_hits": { + "topic related documents": { + "hits": { + "total": { + "value": 2, + "relation": "eq" + }, + "max_score": 0.8333334, + "hits": [ + { + "_index": "retrievers_example", + "_id": "1", + "_score": 0.8333334, + "_source": { + "year": 2024 + } + }, + { + "_index": "retrievers_example", + "_id": "3", + "_score": 0.25, + "_source": { + "year": 2024 + } + } + ] + } + } + } + }, + { + "_index": "retrievers_example", + "_id": "2", + "_score": 0.8333334, + "fields": { + "year": [ + 2023 + ] + }, + "inner_hits": { + "topic related documents": { + "hits": { + "total": { + "value": 1, + "relation": "eq" + }, + "max_score": 0.8333334, + "hits": [ + { + "_index": "retrievers_example", + "_id": "2", + "_score": 0.8333334, + "_source": { + "year": 2023 + } + } + ] + } + } + } + } + ] + } +} +---- +// TESTRESPONSE[s/"took": 42/"took": $body.took/] +============== + +[discrete] +[[retrievers-examples-highlighting-retriever-results]] +==== Example: Highlighting results based on nested sub-retrievers + +Highlighting is now also available for nested sub-retrievers matches. For example, consider the same +`rrf` retriever as above, with a `knn` and `standard` retriever as its sub-retrievers. We can specify a `highlight` +section, as defined in <> documentation, and compute highlights for the top results. + +[source,console] +---- +GET /retrievers_example/_search +{ + "retriever": { + "rrf": { + "retrievers": [ + { + "standard": { + "query": { + "query_string": { + "query": "(information retrieval) OR (artificial intelligence)", + "default_field": "text" + } + } + } + }, + { + "knn": { + "field": "vector", + "query_vector": [ + 0.23, + 0.67, + 0.89 + ], + "k": 3, + "num_candidates": 5 + } + } + ], + "rank_window_size": 10, + "rank_constant": 1 + } + }, + "highlight": { + "fields": { + "text": { + "fragment_size": 150, + "number_of_fragments": 3 + } + } + }, + "_source": false +} +---- +// TEST[continued] + +This would highlight the `text` field, based on the matches produced by the `standard` retriever. The highlighted snippets +would then be included in the response as usual, i.e. under each search hit. + +.Example response +[%collapsible] +============== +[source,console-result] +---- +{ + "took": 42, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 3, + "relation": "eq" + }, + "max_score": 0.8333334, + "hits": [ + { + "_index": "retrievers_example", + "_id": "1", + "_score": 0.8333334, + "highlight": { + "text": [ + "Large language models are revolutionizing information retrieval by boosting search precision, deepening contextual understanding, and reshaping user experiences" + ] + } + }, + { + "_index": "retrievers_example", + "_id": "2", + "_score": 0.8333334, + "highlight": { + "text": [ + "Artificial intelligence is transforming medicine, from advancing diagnostics and tailoring treatment plans to empowering predictive patient care for improved" + ] + } + }, + { + "_index": "retrievers_example", + "_id": "3", + "_score": 0.25 + } + ] + } +} +---- +// TESTRESPONSE[s/"took": 42/"took": $body.took/] +============== + +[discrete] +[[retrievers-examples-inner-hits-retriever-results]] +==== Example: Computing inner hits from nested sub-retrievers + +We can also define `inner_hits` to be computed on any of the sub-retrievers, and propagate those computations to the top +level compound retriever. For example, let's create a new index with a `knn` field, nested under the `nested_field` field, +and index a couple of documents. + +[source,console] +---- +PUT retrievers_example_nested +{ + "mappings": { + "properties": { + "nested_field": { + "type": "nested", + "properties": { + "paragraph_id": { + "type": "keyword" + }, + "nested_vector": { + "type": "dense_vector", + "dims": 3, + "similarity": "l2_norm", + "index": true + } + } + }, + "topic": { + "type": "keyword" + } + } + } +} + +POST /retrievers_example_nested/_doc/1 +{ + "nested_field": [ + { + "paragraph_id": "1a", + "nested_vector": [ + -1.12, + -0.59, + 0.78 + ] + }, + { + "paragraph_id": "1b", + "nested_vector": [ + -0.12, + 1.56, + 0.42 + ] + }, + { + "paragraph_id": "1c", + "nested_vector": [ + 1, + -1, + 0 + ] + } + ], + "topic": [ + "ai" + ] +} + +POST /retrievers_example_nested/_doc/2 +{ + "nested_field": [ + { + "paragraph_id": "2a", + "nested_vector": [ + 0.23, + 1.24, + 0.65 + ] + } + ], + "topic": [ + "information_retrieval" + ] +} + +POST /retrievers_example_nested/_doc/3 +{ + "topic": [ + "ai" + ] +} + +POST /retrievers_example_nested/_refresh +---- +// TEST[continued] + +Now we can run an `rrf` retriever query and also compute <> for the `nested_field.nested_vector` +field, based on the `knn` query specified. + +[source,console] +---- +GET /retrievers_example_nested/_search +{ + "retriever": { + "rrf": { + "retrievers": [ + { + "standard": { + "query": { + "nested": { + "path": "nested_field", + "inner_hits": { + "name": "nested_vector", + "_source": false, + "fields": [ + "nested_field.paragraph_id" + ] + }, + "query": { + "knn": { + "field": "nested_field.nested_vector", + "query_vector": [ + 1, + 0, + 0.5 + ], + "k": 10 + } + } + } + } + } + }, + { + "standard": { + "query": { + "term": { + "topic": "ai" + } + } + } + } + ], + "rank_window_size": 10, + "rank_constant": 1 + } + }, + "_source": [ + "topic" + ] +} +---- +// TEST[continued] + +This would propagate the `inner_hits` defined for the `knn` query to the `rrf` retriever, and compute inner hits for `rrf`'s top results. + +.Example response +[%collapsible] +============== +[source,console-result] +---- +{ + "took": 42, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 3, + "relation": "eq" + }, + "max_score": 1.0, + "hits": [ + { + "_index": "retrievers_example_nested", + "_id": "1", + "_score": 1.0, + "_source": { + "topic": [ + "ai" + ] + }, + "inner_hits": { + "nested_vector": { + "hits": { + "total": { + "value": 3, + "relation": "eq" + }, + "max_score": 0.44353113, + "hits": [ + { + "_index": "retrievers_example_nested", + "_id": "1", + "_nested": { + "field": "nested_field", + "offset": 2 + }, + "_score": 0.44353113, + "fields": { + "nested_field": [ + { + "paragraph_id": [ + "1c" + ] + } + ] + } + }, + { + "_index": "retrievers_example_nested", + "_id": "1", + "_nested": { + "field": "nested_field", + "offset": 1 + }, + "_score": 0.26567122, + "fields": { + "nested_field": [ + { + "paragraph_id": [ + "1b" + ] + } + ] + } + }, + { + "_index": "retrievers_example_nested", + "_id": "1", + "_nested": { + "field": "nested_field", + "offset": 0 + }, + "_score": 0.18478848, + "fields": { + "nested_field": [ + { + "paragraph_id": [ + "1a" + ] + } + ] + } + } + ] + } + } + } + }, + { + "_index": "retrievers_example_nested", + "_id": "2", + "_score": 0.33333334, + "_source": { + "topic": [ + "information_retrieval" + ] + }, + "inner_hits": { + "nested_vector": { + "hits": { + "total": { + "value": 1, + "relation": "eq" + }, + "max_score": 0.32002488, + "hits": [ + { + "_index": "retrievers_example_nested", + "_id": "2", + "_nested": { + "field": "nested_field", + "offset": 0 + }, + "_score": 0.32002488, + "fields": { + "nested_field": [ + { + "paragraph_id": [ + "2a" + ] + } + ] + } + } + ] + } + } + } + }, + { + "_index": "retrievers_example_nested", + "_id": "3", + "_score": 0.33333334, + "_source": { + "topic": [ + "ai" + ] + }, + "inner_hits": { + "nested_vector": { + "hits": { + "total": { + "value": 0, + "relation": "eq" + }, + "max_score": null, + "hits": [] + } + } + } + } + ] + } +} +---- +// TESTRESPONSE[s/"took": 42/"took": $body.took/] +============== + +Note: if using more than one `inner_hits` we need to provide custom names for each `inner_hits` so that they +are unique across all retrievers within the request. + +[discrete] +[[retrievers-examples-rrf-and-aggregations]] +==== Example: Combine RRF with aggregations + +Retrievers support both composability and most of the standard `_search` functionality. For instance, +we can compute aggregations with the `rrf` retriever. When using a compound retriever, +the aggregations are computed based on its nested retrievers. In the following example, +the `terms` aggregation for the `topic` field will include all results, not just the top `rank_window_size`, +from the 2 nested retrievers, i.e. all documents whose `year` field is greater than 2023, and whose `topic` field +matches the term `elastic`. + +[source,console] +---- +GET retrievers_example/_search +{ + "retriever": { + "rrf": { + "retrievers": [ + { + "standard": { + "query": { + "range": { + "year": { + "gt": 2023 + } + } + } + } + }, + { + "standard": { + "query": { + "term": { + "topic": "elastic" + } + } + } + } + ], + "rank_window_size": 10, + "rank_constant": 1 + } + }, + "_source": false, + "aggs": { + "topics": { + "terms": { + "field": "topic" + } + } + } +} +---- +// TEST[continued] + +.Example response +[%collapsible] +============== +[source, console-result] +---- +{ + "took": 42, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 4, + "relation": "eq" + }, + "max_score": 0.5833334, + "hits": [ + { + "_index": "retrievers_example", + "_id": "5", + "_score": 0.5833334 + }, + { + "_index": "retrievers_example", + "_id": "1", + "_score": 0.5 + }, + { + "_index": "retrievers_example", + "_id": "4", + "_score": 0.5 + }, + { + "_index": "retrievers_example", + "_id": "3", + "_score": 0.33333334 + } + ] + }, + "aggregations": { + "topics": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": "ai", + "doc_count": 3 + }, + { + "key": "elastic", + "doc_count": 2 + }, + { + "key": "assistant", + "doc_count": 1 + }, + { + "key": "documentation", + "doc_count": 1 + }, + { + "key": "information_retrieval", + "doc_count": 1 + }, + { + "key": "llm", + "doc_count": 1 + }, + { + "key": "observability", + "doc_count": 1 + }, + { + "key": "security", + "doc_count": 1 + } + ] + } + } +} +---- +// TESTRESPONSE[s/"took": 42/"took": $body.took/] +============== + +[discrete] +[[retrievers-examples-explain-multiple-rrf]] +==== Example: Explainability with multiple retrievers + +By adding `explain: true` to the request, each retriever will now provide a detailed explanation of all the steps +and calculations required to compute the final score. Composability is fully supported in the context of `explain`, and +each retriever will provide its own explanation, as shown in the example below. + +[source,console] +---- +GET /retrievers_example/_search +{ + "retriever": { + "rrf": { + "retrievers": [ + { + "standard": { + "query": { + "term": { + "topic": "elastic" + } + } + } + }, + { + "rrf": { + "retrievers": [ + { + "standard": { + "query": { + "query_string": { + "query": "(information retrieval) OR (artificial intelligence)", + "default_field": "text" + } + } + } + }, + { + "knn": { + "field": "vector", + "query_vector": [ + 0.23, + 0.67, + 0.89 + ], + "k": 3, + "num_candidates": 5 + } + } + ], + "rank_window_size": 10, + "rank_constant": 1 + } + } + ], + "rank_window_size": 10, + "rank_constant": 1 + } + }, + "_source": false, + "size": 1, + "explain": true +} +---- +// TEST[continued] + +The output of which, albeit a bit verbose, will provide all the necessary info to assist in debugging and reason with ranking. + +.Example response +[%collapsible] +============== +[source, console-result] +---- +{ + "took": 42, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 5, + "relation": "eq" + }, + "max_score": 0.5, + "hits": [ + { + "_shard": "[retrievers_example][0]", + "_node": "jnrdZFKS3abUgWVsVdj2Vg", + "_index": "retrievers_example", + "_id": "1", + "_score": 0.5, + "_explanation": { + "value": 0.5, + "description": "rrf score: [0.5] computed for initial ranks [0, 1] with rankConstant: [1] as sum of [1 / (rank + rankConstant)] for each query", + "details": [ + { + "value": 0.0, + "description": "rrf score: [0], result not found in query at index [0]", + "details": [] + }, + { + "value": 1, + "description": "rrf score: [0.5], for rank [1] in query at index [1] computed as [1 / (1 + 1)], for matching query with score", + "details": [ + { + "value": 0.8333334, + "description": "rrf score: [0.8333334] computed for initial ranks [2, 1] with rankConstant: [1] as sum of [1 / (rank + rankConstant)] for each query", + "details": [ + { + "value": 2, + "description": "rrf score: [0.33333334], for rank [2] in query at index [0] computed as [1 / (2 + 1)], for matching query with score", + "details": [ + { + "value": 2.8129659, + "description": "sum of:", + "details": [ + { + "value": 1.4064829, + "description": "weight(text:information in 0) [PerFieldSimilarity], result of:", + "details": [ + *** + ] + }, + { + "value": 1.4064829, + "description": "weight(text:retrieval in 0) [PerFieldSimilarity], result of:", + "details": [ + *** + ] + } + ] + } + ] + }, + { + "value": 1, + "description": "rrf score: [0.5], for rank [1] in query at index [1] computed as [1 / (1 + 1)], for matching query with score", + "details": [ + { + "value": 1, + "description": "doc [0] with an original score of [1.0] is at rank [1] from the following source queries.", + "details": [ + { + "value": 1.0, + "description": "found vector with calculated similarity: 1.0", + "details": [] + } + ] + } + ] + } + ] + } + ] + } + ] + } + } + ] + } +} +---- +// TESTRESPONSE[s/"took": 42/"took": $body.took/] +// TESTRESPONSE[s/\.\.\./$body.hits.hits.0._explanation.details.1.details.0.details.0.details.0.details.0.details.0/] +// TESTRESPONSE[s/\*\*\*/$body.hits.hits.0._explanation.details.1.details.0.details.0.details.0.details.1.details.0/] +// TESTRESPONSE[s/jnrdZFKS3abUgWVsVdj2Vg/$body.hits.hits.0._node/] +============== + +[discrete] +[[retrievers-examples-text-similarity-reranker-on-top-of-rrf]] +==== Example: Rerank results of an RRF retriever + +To demonstrate the full functionality of retrievers, the following examples also require access to a <> set up using the <>. + +In this example we'll set up a reranking service and use it with the `text_similarity_reranker` retriever to rerank our top results. + +[source,console] +---- +PUT _inference/rerank/my-rerank-model +{ + "service": "cohere", + "service_settings": { + "model_id": "rerank-english-v3.0", + "api_key": "{{COHERE_API_KEY}}" + } +} +---- +// TEST[skip: no_access_to_ml] + +Let's start by reranking the results of the `rrf` retriever in our previous example. + +[source,console] +---- +GET retrievers_example/_search +{ + "retriever": { + "text_similarity_reranker": { + "retriever": { + "rrf": { + "retrievers": [ + { + "standard": { + "query": { + "query_string": { + "query": "(information retrieval) OR (artificial intelligence)", + "default_field": "text" + } + } + } + }, + { + "knn": { + "field": "vector", + "query_vector": [ + 0.23, + 0.67, + 0.89 + ], + "k": 3, + "num_candidates": 5 + } + } + ], + "rank_window_size": 10, + "rank_constant": 1 + } + }, + "field": "text", + "inference_id": "my-rerank-model", + "inference_text": "What are the state of the art applications of AI in information retrieval?" + } + }, + "_source": false +} + +---- +// TEST[skip: no_access_to_ml] + +[discrete] +[[retrievers-examples-rrf-ranking-on-text-similarity-reranker-results]] +==== Example: RRF with semantic reranker + +For this example, we'll replace the rrf's `standard` retriever with the `text_similarity_reranker` retriever, using the +`my-rerank-model` reranker we previously configured. Since this is a reranker, it needs an initial pool of +documents to work with. In this case, we'll rerank the top `rank_window_size` documents matching the `ai` topic. + +[source,console] +---- +GET /retrievers_example/_search +{ + "retriever": { + "rrf": { + "retrievers": [ + { + "knn": { + "field": "vector", + "query_vector": [ + 0.23, + 0.67, + 0.89 + ], + "k": 3, + "num_candidates": 5 + } + }, + { + "text_similarity_reranker": { + "retriever": { + "standard": { + "query": { + "term": { + "topic": "ai" + } + } + } + }, + "field": "text", + "inference_id": "my-rerank-model", + "inference_text": "Can I use generative AI to identify user intent and improve search relevance?" + } + } + ], + "rank_window_size": 10, + "rank_constant": 1 + } + }, + "_source": false +} +---- +// TEST[skip: no_access_to_ml] + +[discrete] +[[retrievers-examples-chaining-text-similarity-reranker-retrievers]] +==== Example: Chaining multiple semantic rerankers + +Full composability means we can chain together multiple retrievers of the same type. For instance, +imagine we have a computationally expensive reranker that's specialized for AI content. We can rerank the results of a `text_similarity_reranker` using another `text_similarity_reranker` retriever. Each reranker can operate on different fields and/or use different inference services. + +[source,console] +---- +GET retrievers_example/_search +{ + "retriever": { + "text_similarity_reranker": { + "retriever": { + "text_similarity_reranker": { + "retriever": { + "knn": { + "field": "vector", + "query_vector": [ + 0.23, + 0.67, + 0.89 + ], + "k": 3, + "num_candidates": 5 + } + }, + "rank_window_size": 100, + "field": "text", + "inference_id": "my-rerank-model", + "inference_text": "What are the state of the art applications of AI in information retrieval?" + } + }, + "rank_window_size": 10, + "field": "text", + "inference_id": "my-other-more-expensive-rerank-model", + "inference_text": "Applications of Large Language Models in technology and their impact on user satisfaction" + } + }, + "_source": false +} +---- +// TEST[skip: no_access_to_ml] + +Note that our example applies two reranking steps. First, we rerank the top 100 +documents from the `knn` search using the `my-rerank-model` reranker. Then we +pick the top 10 results and rerank them using the more fine-grained +`my-other-more-expensive-rerank-model`. diff --git a/docs/reference/search/search-your-data/retrievers-overview.asciidoc b/docs/reference/search/search-your-data/retrievers-overview.asciidoc new file mode 100644 index 0000000000000..1771b5bb0d849 --- /dev/null +++ b/docs/reference/search/search-your-data/retrievers-overview.asciidoc @@ -0,0 +1,188 @@ +[[retrievers-overview]] +== Retrievers + +A retriever is an abstraction that was added to the Search API in *8.14.0* and was made generally available in *8.16.0*. +This abstraction enables the configuration of multi-stage retrieval pipelines within a single `_search` call. +This simplifies your search application logic, because you no longer need to configure complex searches via multiple {es} calls or implement additional client-side logic to combine results from different queries. + +This document provides a general overview of the retriever abstraction. +For implementation details, including notable restrictions, check out the +<> in the `_search` API docs. + +[discrete] +[[retrievers-overview-types]] +=== Retriever types + +Retrievers come in various types, each tailored for different search operations. +The following retrievers are currently available: + +* <>. +Returns top documents from a traditional https://www.elastic.co/guide/en/elasticsearch/reference/master/query-dsl.html[query]. +Mimics a traditional query but in the context of a retriever framework. +This ensures backward compatibility as existing `_search` requests remain supported. +That way you can transition to the new abstraction at your own pace without mixing syntaxes. +* <>. +Returns top documents from a <>, in the context of a retriever framework. +* <>. +Combines and ranks multiple first-stage retrievers using the reciprocal rank fusion (RRF) algorithm. +Allows you to combine multiple result sets with different relevance indicators into a single result set. +An RRF retriever is a *compound retriever*, where its `filter` element is propagated to its sub retrievers. +* <>. +Applies <> to the query before returning results. +* <>. +Used for <>. +Requires first creating a `rerank` task using the <>. + +[discrete] +[[retrievers-overview-why-are-they-useful]] +=== What makes retrievers useful? + +Here's an overview of what makes retrievers useful and how they differ from regular queries. + +. *Simplified user experience*. +Retrievers simplify the user experience by allowing entire retrieval pipelines to be configured in a single API call. +This maintains backward compatibility with traditional query elements by automatically translating them to the appropriate retriever. +. *Structured retrieval*. +Retrievers provide a more structured way to define search operations. +They allow searches to be described using a "retriever tree", a hierarchical structure that clarifies the sequence and logic of operations, making complex searches more understandable and manageable. +. *Composability and flexibility*. +Retrievers enable flexible composability, allowing you to build pipelines and seamlessly integrate different retrieval strategies into these pipelines. +Retrievers make it easy to test out different retrieval strategy combinations. +. *Compound operations*. +A retriever can have sub retrievers. +This allows complex nested searches where the results of one retriever feed into another, supporting sophisticated querying strategies that might involve multiple stages or criteria. +. *Retrieval as a first-class concept*. +Unlike traditional queries, where the query is a part of a larger search API call, retrievers are designed as standalone entities that can be combined or used in isolation. +This enables a more modular and flexible approach to constructing searches. +. *Enhanced control over document scoring and ranking*. +Retrievers allow for more explicit control over how documents are scored and filtered. +For instance, you can specify minimum score thresholds, apply complex filters without affecting scoring, and use parameters like `terminate_after` for performance optimizations. +. *Integration with existing {es} functionalities*. +Even though retrievers can be used instead of existing `_search` API syntax (like the +`query` and `knn`), they are designed to integrate seamlessly with things like pagination (`search_after`) and sorting. +They also maintain compatibility with aggregation operations by treating the combination of all leaf retrievers as +`should` clauses in a boolean query. +. *Cleaner separation of concerns*. +When using compound retrievers, only the query element is allowed, which enforces a cleaner separation of concerns and prevents the complexity that might arise from overly nested or interdependent configurations. + +[discrete] +[[retrievers-overview-example]] +=== Example + +The following example demonstrates how using retrievers simplify the composability of queries for RRF ranking. + +[source,js] +---- +GET example-index/_search +{ + "retriever": { + "rrf": { + "retrievers": [ + { + "standard": { + "query": { + "sparse_vector": { + "field": "vector.tokens", + "inference_id": "my-elser-endpoint", + "query": "What blue shoes are on sale?" + } + } + } + }, + { + "standard": { + "query": { + "match": { + "text": "blue shoes sale" + } + } + } + } + ] + } + } +} +---- +//NOTCONSOLE + +This example demonstrates how you can combine different retrieval strategies into a single `retriever` pipeline. + +Compare to `RRF` with `sub_searches` approach (which is deprecated as of 8.16.0): + +.*Expand* for example +[%collapsible] +============== + +[source,js] +---- +GET example-index/_search +{ + "sub_searches":[ + { + "query":{ + "match":{ + "text":"blue shoes sale" + } + } + }, + { + "query":{ + "sparse_vector": { + "field": "vector.tokens", + "inference_id": "my-elser-endoint", + "query": "What blue shoes are on sale?" + } + } + } + ], + "rank":{ + "rrf":{ + "rank_window_size":50, + "rank_constant":20 + } + } +} +---- +//NOTCONSOLE +============== + +For more examples on how to use retrievers, please refer to <>. + +[discrete] +[[retrievers-overview-glossary]] +=== Glossary + +Here are some important terms: + +* *Retrieval Pipeline*. +Defines the entire retrieval and ranking logic to produce top hits. +* *Retriever Tree*. +A hierarchical structure that defines how retrievers interact. +* *First-stage Retriever*. +Returns an initial set of candidate documents. +* *Compound Retriever*. +Builds on one or more retrievers, enhancing document retrieval and ranking logic. +* *Combiners*. +Compound retrievers that merge top hits from multiple sub-retrievers. +* *Rerankers*. +Special compound retrievers that reorder hits and may adjust the number of hits, with distinctions between first-stage and second-stage rerankers. + +[discrete] +[[retrievers-overview-play-in-search]] +=== Retrievers in action + +The Search Playground builds Elasticsearch queries using the retriever abstraction. +It automatically detects the fields and types in your index and builds a retriever tree based on your selections. + +You can use the Playground to experiment with different retriever configurations and see how they affect search results. + +Refer to the {kibana-ref}/playground.html[Playground documentation] for more information. + +[discrete] +[[retrievers-overview-api-reference]] +=== API reference + +For implementation details, including notable restrictions, check out the <> in the Search API docs. + + +include::retrievers-examples.asciidoc[] diff --git a/docs/reference/search/search-your-data/retrievers-reranking/index.asciidoc b/docs/reference/search/search-your-data/retrievers-reranking/index.asciidoc deleted file mode 100644 index 87ed52e365370..0000000000000 --- a/docs/reference/search/search-your-data/retrievers-reranking/index.asciidoc +++ /dev/null @@ -1,8 +0,0 @@ -[[retrievers-reranking-overview]] -== Retrievers and reranking - -* <> -* <> - -include::retrievers-overview.asciidoc[] -include::semantic-reranking.asciidoc[] diff --git a/docs/reference/search/search-your-data/retrievers-reranking/retrievers-overview.asciidoc b/docs/reference/search/search-your-data/retrievers-reranking/retrievers-overview.asciidoc deleted file mode 100644 index c0fe7471946f3..0000000000000 --- a/docs/reference/search/search-your-data/retrievers-reranking/retrievers-overview.asciidoc +++ /dev/null @@ -1,183 +0,0 @@ -[[retrievers-overview]] -=== Retrievers - -preview::[] - -A retriever is an abstraction that was added to the Search API in *8.14.0*. -This abstraction enables the configuration of multi-stage retrieval pipelines within a single `_search` call. -This simplifies your search application logic, because you no longer need to configure complex searches via multiple {es} calls or implement additional client-side logic to combine results from different queries. - -This document provides a general overview of the retriever abstraction. -For implementation details, including notable restrictions, check out the -<> in the `_search` API docs. - -[discrete] -[[retrievers-overview-types]] -==== Retriever types - -Retrievers come in various types, each tailored for different search operations. -The following retrievers are currently available: - -* <>. Returns top documents from a -traditional https://www.elastic.co/guide/en/elasticsearch/reference/master/query-dsl.html[query]. -Mimics a traditional query but in the context of a retriever framework. This -ensures backward compatibility as existing `_search` requests remain supported. -That way you can transition to the new abstraction at your own pace without -mixing syntaxes. -* <>. Returns top documents from a <>, -in the context of a retriever framework. -* <>. Combines and ranks multiple first-stage retrievers using -the reciprocal rank fusion (RRF) algorithm. Allows you to combine multiple result sets -with different relevance indicators into a single result set. -An RRF retriever is a *compound retriever*, where its `filter` element is -propagated to its sub retrievers. -+ -Sub retrievers may not use elements that are restricted by having a compound retriever as part of the retriever tree. -See the <> for detailed examples and information on how to use the RRF retriever. -* <>. Used for <>. -Requires first creating a `rerank` task using the <>. - -[discrete] -==== What makes retrievers useful? - -Here's an overview of what makes retrievers useful and how they differ from regular queries. - -. *Simplified user experience*. -Retrievers simplify the user experience by allowing entire retrieval pipelines to be configured in a single API call. -This maintains backward compatibility with traditional query elements by automatically translating them to the appropriate retriever. -. *Structured retrieval*. -Retrievers provide a more structured way to define search operations. -They allow searches to be described using a "retriever tree", a hierarchical structure that clarifies the sequence and logic of operations, making complex searches more understandable and manageable. -. *Composability and flexibility*. -Retrievers enable flexible composability, allowing you to build pipelines and seamlessly integrate different retrieval strategies into these pipelines. -Retrievers make it easy to test out different retrieval strategy combinations. -. *Compound operations*. -A retriever can have sub retrievers. -This allows complex nested searches where the results of one retriever feed into another, supporting sophisticated querying strategies that might involve multiple stages or criteria. -. *Retrieval as a first-class concept*. -Unlike traditional queries, where the query is a part of a larger search API call, retrievers are designed as standalone entities that can be combined or used in isolation. -This enables a more modular and flexible approach to constructing searches. -. *Enhanced control over document scoring and ranking*. -Retrievers allow for more explicit control over how documents are scored and filtered. -For instance, you can specify minimum score thresholds, apply complex filters without affecting scoring, and use parameters like `terminate_after` for performance optimizations. -. *Integration with existing {es} functionalities*. -Even though retrievers can be used instead of existing `_search` API syntax (like the -`query` and `knn`), they are designed to integrate seamlessly with things like pagination (`search_after`) and sorting. -They also maintain compatibility with aggregation operations by treating the combination of all leaf retrievers as -`should` clauses in a boolean query. -. *Cleaner separation of concerns*. -When using compound retrievers, only the query element is allowed, which enforces a cleaner separation of concerns and prevents the complexity that might arise from overly nested or interdependent configurations. - -[discrete] -[[retrievers-overview-example]] -==== Example - -The following example demonstrates how using retrievers simplify the composability of queries for RRF ranking. - -[source,js] ----- -GET example-index/_search -{ - "retriever": { - "rrf": { - "retrievers": [ - { - "standard": { - "query": { - "sparse_vector": { - "field": "vector.tokens", - "inference_id": "my-elser-endpoint", - "query": "What blue shoes are on sale?" - } - } - } - }, - { - "standard": { - "query": { - "match": { - "text": "blue shoes sale" - } - } - } - } - ] - } - } -} ----- -//NOTCONSOLE - -This example demonstrates how you can combine different retrieval strategies into a single `retriever` pipeline. - -Compare to `RRF` with `sub_searches` approach: - -.*Expand* for example -[%collapsible] -============== - -[source,js] ----- -GET example-index/_search -{ - "sub_searches":[ - { - "query":{ - "match":{ - "text":"blue shoes sale" - } - } - }, - { - "query":{ - "sparse_vector": { - "field": "vector.tokens", - "inference_id": "my-elser-endoint", - "query": "What blue shoes are on sale?" - } - } - } - ], - "rank":{ - "rrf":{ - "rank_window_size":50, - "rank_constant":20 - } - } -} ----- -//NOTCONSOLE -============== - -[discrete] -[[retrievers-overview-glossary]] -==== Glossary - -Here are some important terms: - -* *Retrieval Pipeline*. Defines the entire retrieval and ranking logic to -produce top hits. -* *Retriever Tree*. A hierarchical structure that defines how retrievers interact. -* *First-stage Retriever*. Returns an initial set of candidate documents. -* *Compound Retriever*. Builds on one or more retrievers, -enhancing document retrieval and ranking logic. -* *Combiners*. Compound retrievers that merge top hits -from multiple sub-retrievers. -* *Rerankers*. Special compound retrievers that reorder hits and may adjust the number of hits, with distinctions between first-stage and second-stage rerankers. - -[discrete] -[[retrievers-overview-play-in-search]] -==== Retrievers in action - -The Search Playground builds Elasticsearch queries using the retriever abstraction. -It automatically detects the fields and types in your index and builds a retriever tree based on your selections. - -You can use the Playground to experiment with different retriever configurations and see how they affect search results. - -Refer to the {kibana-ref}/playground.html[Playground documentation] for more information. - -[discrete] -[[retrievers-overview-api-reference]] -==== API reference - -For implementation details, including notable restrictions, check out the <> in the Search API docs. diff --git a/docs/reference/search/search-your-data/retrievers-reranking/semantic-reranking.asciidoc b/docs/reference/search/search-your-data/retrievers-reranking/semantic-reranking.asciidoc deleted file mode 100644 index add2d7455983e..0000000000000 --- a/docs/reference/search/search-your-data/retrievers-reranking/semantic-reranking.asciidoc +++ /dev/null @@ -1,142 +0,0 @@ -[[semantic-reranking]] -=== Semantic reranking - -preview::[] - -[TIP] -==== -This overview focuses more on the high-level concepts and use cases for semantic reranking. For full implementation details on how to set up and use semantic reranking in {es}, see the <> in the Search API docs. -==== - -Rerankers improve the relevance of results from earlier-stage retrieval mechanisms. -_Semantic_ rerankers use machine learning models to reorder search results based on their semantic similarity to a query. - -First-stage retrievers and rankers must be very fast and efficient because they process either the entire corpus, or all matching documents. -In a multi-stage pipeline, you can progressively use more computationally intensive ranking functions and techniques, as they will operate on smaller result sets at each step. -This helps avoid query latency degradation and keeps costs manageable. - -Semantic reranking requires relatively large and complex machine learning models and operates in real-time in response to queries. -This technique makes sense on a small _top-k_ result set, as one the of the final steps in a pipeline. -This is a powerful technique for improving search relevance that works equally well with keyword, semantic, or hybrid retrieval algorithms. - -The next sections provide more details on the benefits, use cases, and model types used for semantic reranking. -The final sections include a practical, high-level overview of how to implement <> and links to the full reference documentation. - -[discrete] -[[semantic-reranking-use-cases]] -==== Use cases - -Semantic reranking enables a variety of use cases: - -* *Lexical (BM25) retrieval results reranking* -** Out-of-the-box semantic search by adding a simple API call to any lexical/BM25 retrieval pipeline. -** Adds semantic search capabilities on top of existing indices without reindexing, perfect for quick improvements. -** Ideal for environments with complex existing indices. - -* *Semantic retrieval results reranking* -** Improves results from semantic retrievers using ELSER sparse vector embeddings or dense vector embeddings by using more powerful models. -** Adds a refinement layer on top of hybrid retrieval with <>. - -* *General applications* -** Supports automatic and transparent chunking, eliminating the need for pre-chunking at index time. -** Provides explicit control over document relevance in retrieval-augmented generation (RAG) uses cases or other scenarios involving language model (LLM) inputs. - -Now that we've outlined the value of semantic reranking, we'll explore the specific models that power this process and how they differ. - -[discrete] -[[semantic-reranking-models]] -==== Cross-encoder and bi-encoder models - -At a high level, two model types are used for semantic reranking: cross-encoders and bi-encoders. - -NOTE: In this version, {es} *only supports cross-encoders* for semantic reranking. - -* A *cross-encoder model* can be thought of as a more powerful, all-in-one solution, because it generates query-aware document representations. -It takes the query and document texts as a single, concatenated input. -* A *bi-encoder model* takes as input either document or query text. -Documents and query embeddings are computed separately, so they aren't aware of each other. -** To compute a ranking score, an external operation is required. This typically involves computing dot-product or cosine similarity between the query and document embeddings. - -In brief, cross-encoders provide high accuracy but are more resource-intensive. -Bi-encoders are faster and more cost-effective but less precise. - -In future versions, {es} will also support bi-encoders. -If you're interested in a more detailed analysis of the practical differences between cross-encoders and bi-encoders, untoggle the next section. - -.Comparisons between cross-encoder and bi-encoder -[%collapsible] -============== -The following is a non-exhaustive list of considerations when choosing between cross-encoders and bi-encoders for semantic reranking: - -* Because a cross-encoder model simultaneously processes both query and document texts, it can better infer their relevance, making it more effective as a reranker than a bi-encoder. -* Cross-encoder models are generally larger and more computationally intensive, resulting in higher latencies and increased computational costs. -* There are significantly fewer open-source cross-encoders, while bi-encoders offer a wide variety of sizes, languages, and other trade-offs. -* The effectiveness of cross-encoders can also improve the relevance of semantic retrievers. -For example, their ability to take word order into account can improve on dense or sparse embedding retrieval. -* When trained in tandem with specific retrievers (like lexical/BM25), cross-encoders can “correct” typical errors made by those retrievers. -* Cross-encoders output scores that are consistent across queries. -This enables you to maintain high relevance in result sets, by setting a minimum score threshold for all queries. -For example, this is important when using results in a RAG workflow or if you're otherwise feeding results to LLMs. -Note that similarity scores from bi-encoders/embedding similarities are _query-dependent_, meaning you cannot set universal cut-offs. -* Bi-encoders rerank using embeddings. You can improve your reranking latency by creating embeddings at ingest-time. These embeddings can be stored for reranking without being indexed for retrieval, reducing your memory footprint. -============== - -[discrete] -[[semantic-reranking-in-es]] -==== Semantic reranking in {es} - -In {es}, semantic rerankers are implemented using the {es} <> and a <>. - -To use semantic reranking in {es}, you need to: - -. *Choose a reranking model*. -Currently you can: - -** Integrate directly with the <> using the `rerank` task type -** Integrate directly with the <> using the `rerank` task type -** Upload a model to {es} from Hugging Face with {eland-docs}/machine-learning.html#ml-nlp-pytorch[Eland]. You'll need to use the `text_similarity` NLP task type when loading the model using Eland. Refer to {ml-docs}/ml-nlp-model-ref.html#ml-nlp-model-ref-text-similarity[the Elastic NLP model reference] for a list of third party text similarity models supported by {es} for semantic reranking. -*** Then set up an <> with the `rerank` task type -. *Create a `rerank` task using the <>*. -The Inference API creates an inference endpoint and configures your chosen machine learning model to perform the reranking task. -. *Define a `text_similarity_reranker` retriever in your search request*. -The retriever syntax makes it simple to configure both the retrieval and reranking of search results in a single API call. - -.*Example search request* with semantic reranker -[%collapsible] -============== -The following example shows a search request that uses a semantic reranker to reorder the top-k documents based on their semantic similarity to the query. -[source,console] ----- -POST _search -{ - "retriever": { - "text_similarity_reranker": { - "retriever": { - "standard": { - "query": { - "match": { - "text": "How often does the moon hide the sun?" - } - } - } - }, - "field": "text", - "inference_id": "my-cohere-rerank-model", - "inference_text": "How often does the moon hide the sun?", - "rank_window_size": 100, - "min_score": 0.5 - } - } -} ----- -// TEST[skip:TBD] -============== - -[discrete] -[[semantic-reranking-learn-more]] -==== Learn more - -* Read the <> for syntax and implementation details -* Learn more about the <> abstraction -* Learn more about the Elastic <> -* Check out our https://github.com/elastic/elasticsearch-labs/blob/main/notebooks/integrations/cohere/cohere-elasticsearch.ipynb[Python notebook] for using Cohere with {es} \ No newline at end of file diff --git a/docs/reference/search/search-your-data/search-api.asciidoc b/docs/reference/search/search-your-data/search-api.asciidoc index 98c5a48b7559b..a9e74d54dd9d9 100644 --- a/docs/reference/search/search-your-data/search-api.asciidoc +++ b/docs/reference/search/search-your-data/search-api.asciidoc @@ -530,3 +530,4 @@ include::retrieve-inner-hits.asciidoc[] include::search-shard-routing.asciidoc[] include::search-using-query-rules.asciidoc[] include::search-template.asciidoc[] + diff --git a/docs/reference/search/search-your-data/search-using-query-rules.asciidoc b/docs/reference/search/search-your-data/search-using-query-rules.asciidoc index 18be825d02376..7d9d14684beee 100644 --- a/docs/reference/search/search-your-data/search-using-query-rules.asciidoc +++ b/docs/reference/search/search-your-data/search-using-query-rules.asciidoc @@ -10,7 +10,7 @@ _Query rules_ allow customization of search results for queries that match speci This allows for more control over results, for example ensuring that promoted documents that match defined criteria are returned at the top of the result list. Metadata is defined in the query rule, and is matched against the query criteria. Query rules use metadata to match a query. -Metadata is provided as part of the <> as an object and can be anything that helps differentiate the query, for example: +Metadata is provided as part of the search request as an object and can be anything that helps differentiate the query, for example: * A user-entered query string * Personalized metadata about users (e.g. country, language, etc) @@ -18,13 +18,13 @@ Metadata is provided as part of the <> as an o * A referring site * etc. -Query rules define a metadata key that will be used to match the metadata provided in the <> with the criteria specified in the rule. +Query rules define a metadata key that will be used to match the metadata provided in the <> with the criteria specified in the rule. -When a query rule matches the <> metadata according to its defined criteria, the query rule action is applied to the underlying `organic` query. +When a query rule matches the rule metadata according to its defined criteria, the query rule action is applied to the underlying `organic` query. For example, a query rule could be defined to match a user-entered query string of `pugs` and a country `us` and promote adoptable shelter dogs if the rule query met both criteria. -Rules are defined using the <> and searched using the <>. +Rules are defined using the <> and searched using the <> or the <>. [discrete] [[query-rule-definition]] @@ -189,9 +189,11 @@ You can use the <> call to retrieve the ruleset you just crea [discrete] [[rule-query-search]] -==== Perform a rule query +==== Search using query rules + +Once you have defined one or more query rulesets, you can search using these rulesets using the <> or the <>. +Retrievers are the recommended way to use rule queries, as they will work out of the box with other reranking retrievers such as <>. -Once you have defined one or more query rulesets, you can search these rulesets using the <> query. Rulesets are evaluated in order, so rules in the first ruleset you specify will be applied before any subsequent rulesets. An example query for the `my-ruleset` defined above is: @@ -200,18 +202,22 @@ An example query for the `my-ruleset` defined above is: ---- GET /my-index-000001/_search { - "query": { + "retriever": { "rule": { - "organic": { - "query_string": { - "query": "puggles" + "retriever": { + "standard": { + "query": { + "query_string": { + "query": "puggles" + } + } } }, "match_criteria": { "query_string": "puggles", "user_country": "us" }, - "ruleset_ids": ["my-ruleset"] + "ruleset_ids": [ "my-ruleset" ] } } } @@ -227,3 +233,51 @@ In this case, the rules are applied in the following order: - Where the matching rule appears in the ruleset - If multiple documents are specified in a single rule, in the order they are specified - If a document is matched by both a `pinned` rule and an `exclude` rule, the `exclude` rule will take precedence + +You can specify reranking retrievers such as <> or <> in the rule query to apply query rules on already-reranked results. +Here is an example: + +[source,console] +---- +GET my-index-000001/_search +{ + "retriever": { + "rule": { + "match_criteria": { + "query_string": "puggles", + "user_country": "us" + }, + "ruleset_ids": [ + "my-ruleset" + ], + "retriever": { + "rrf": { + "retrievers": [ + { + "standard": { + "query": { + "query_string": { + "query": "pugs" + } + } + } + }, + { + "standard": { + "query": { + "query_string": { + "query": "puggles" + } + } + } + } + ] + } + } + } + } +} +---- +// TEST[continued] + +This will apply pinned and excluded query rules on top of the content that was reranked by RRF. diff --git a/docs/reference/search/search-your-data/search-your-data.asciidoc b/docs/reference/search/search-your-data/search-your-data.asciidoc index a885df2f2179e..0828462fd1850 100644 --- a/docs/reference/search/search-your-data/search-your-data.asciidoc +++ b/docs/reference/search/search-your-data/search-your-data.asciidoc @@ -18,7 +18,7 @@ Search for exact values:: Search for <> of numbers, dates, IPs, or strings. -Full-text search:: +<>:: Use <> to query <> and find documents that best match query terms. @@ -43,11 +43,14 @@ DSL, with a simplified user experience. Create search applications based on your results directly in the Kibana Search UI. include::search-api.asciidoc[] +include::full-text-search.asciidoc[] +include::../../how-to/recipes.asciidoc[] +// ☝️ search relevance recipes +include::retrievers-overview.asciidoc[] include::knn-search.asciidoc[] include::semantic-search.asciidoc[] -include::retrievers-reranking/index.asciidoc[] -include::learning-to-rank.asciidoc[] +include::retrieval-augmented-generation.asciidoc[] include::search-across-clusters.asciidoc[] include::search-with-synonyms.asciidoc[] include::search-application-overview.asciidoc[] -include::behavioral-analytics/behavioral-analytics-overview.asciidoc[] +include::behavioral-analytics/behavioral-analytics-overview.asciidoc[] \ No newline at end of file diff --git a/docs/reference/search/search-your-data/semantic-search-deploy-model.asciidoc b/docs/reference/search/search-your-data/semantic-search-deploy-model.asciidoc new file mode 100644 index 0000000000000..6c610159ae0b9 --- /dev/null +++ b/docs/reference/search/search-your-data/semantic-search-deploy-model.asciidoc @@ -0,0 +1,97 @@ +[[semantic-search-deployed-nlp-model]] +=== Tutorial: semantic search with a deployed model + +++++ +Semantic search with deployed model +++++ + +[IMPORTANT] +==== +* For the easiest way to perform semantic search in the {stack}, refer to the <> end-to-end tutorial. +* This tutorial was written before the <> and <> was introduced. +Today we have simpler options for performing semantic search. +==== + +This guide shows you how to implement semantic search with models deployed in {es}: from selecting an NLP model, to writing queries. + + +[discrete] +[[deployed-select-nlp-model]] +==== Select an NLP model + +{es} offers the usage of a {ml-docs}/ml-nlp-model-ref.html#ml-nlp-model-ref-text-embedding[wide range of NLP models], including both dense and sparse vector models. +Your choice of the language model is critical for implementing semantic search successfully. + +While it is possible to bring your own text embedding model, achieving good search results through model tuning is challenging. +Selecting an appropriate model from our third-party model list is the first step. +Training the model on your own data is essential to ensure better search results than using only BM25. +However, the model training process requires a team of data scientists and ML experts, making it expensive and time-consuming. + +To address this issue, Elastic provides a pre-trained representational model called {ml-docs}/ml-nlp-elser.html[Elastic Learned Sparse EncodeR (ELSER)]. +ELSER, currently available only for English, is an out-of-domain sparse vector model that does not require fine-tuning. +This adaptability makes it suitable for various NLP use cases out of the box. +Unless you have a team of ML specialists, it is highly recommended to use the ELSER model. + +In the case of sparse vector representation, the vectors mostly consist of zero values, with only a small subset containing non-zero values. +This representation is commonly used for textual data. +In the case of ELSER, each document in an index and the query text itself are represented by high-dimensional sparse vectors. +Each non-zero element of the vector corresponds to a term in the model vocabulary. +The ELSER vocabulary contains around 30000 terms, so the sparse vectors created by ELSER contain about 30000 values, the majority of which are zero. +Effectively the ELSER model is replacing the terms in the original query with other terms that have been learnt to exist in the documents that best match the original search terms in a training dataset, and weights to control how important each is. + + +[discrete] +[[deployed-deploy-nlp-model]] +==== Deploy the model + +After you decide which model you want to use for implementing semantic search, you need to deploy the model in {es}. + +include::{es-ref-dir}/tab-widgets/semantic-search/deploy-nlp-model-widget.asciidoc[] + + +[discrete] +[[deployed-field-mappings]] +==== Map a field for the text embeddings + +Before you start using the deployed model to generate embeddings based on your input text, you need to prepare your index mapping first. +The mapping of the index depends on the type of model. + +include::{es-ref-dir}/tab-widgets/semantic-search/field-mappings-widget.asciidoc[] + + +[discrete] +[[deployed-generate-embeddings]] +==== Generate text embeddings + +Once you have created the mappings for the index, you can generate text embeddings from your input text. +This can be done by using an +<> with an <>. +The ingest pipeline processes the input data and indexes it into the destination index. +At index time, the inference ingest processor uses the trained model to infer against the data ingested through the pipeline. +After you created the ingest pipeline with the inference processor, you can ingest your data through it to generate the model output. + +include::{es-ref-dir}/tab-widgets/semantic-search/generate-embeddings-widget.asciidoc[] + +Now it is time to perform semantic search! + + +[discrete] +[[deployed-search]] +==== Search the data + +Depending on the type of model you have deployed, you can query rank features with a <> query, or dense vectors with a kNN search. + +include::{es-ref-dir}/tab-widgets/semantic-search/search-widget.asciidoc[] + + +[discrete] +[[deployed-hybrid-search]] +==== Beyond semantic search with hybrid search + +In some situations, lexical search may perform better than semantic search. +For example, when searching for single words or IDs, like product numbers. + +Combining semantic and lexical search into one hybrid search request using <> provides the best of both worlds. +Not only that, but hybrid search using reciprocal rank fusion {blog-ref}improving-information-retrieval-elastic-stack-hybrid[has been shown to perform better in general]. + +include::{es-ref-dir}/tab-widgets/semantic-search/hybrid-search-widget.asciidoc[] \ No newline at end of file diff --git a/docs/reference/search/search-your-data/semantic-search-elser.asciidoc b/docs/reference/search/search-your-data/semantic-search-elser.asciidoc index 11aec59a00b30..5309b24fa37c9 100644 --- a/docs/reference/search/search-your-data/semantic-search-elser.asciidoc +++ b/docs/reference/search/search-your-data/semantic-search-elser.asciidoc @@ -117,15 +117,15 @@ All unique passages, along with their IDs, have been extracted from that data se https://github.com/elastic/stack-docs/blob/main/docs/en/stack/ml/nlp/data/msmarco-passagetest2019-unique.tsv[tsv file]. IMPORTANT: The `msmarco-passagetest2019-top1000` dataset was not utilized to train the model. -It is only used in this tutorial as a sample dataset that is easily accessible for demonstration purposes. +We use this sample dataset in the tutorial because is easily accessible for demonstration purposes. You can use a different data set to test the workflow and become familiar with it. -Download the file and upload it to your cluster using the -{kibana-ref}/connect-to-elasticsearch.html#upload-data-kibana[Data Visualizer] -in the {ml-app} UI. -Assign the name `id` to the first column and `content` to the second column. -The index name is `test-data`. -Once the upload is complete, you can see an index named `test-data` with 182469 documents. +Download the file and upload it to your cluster using the {kibana-ref}/connect-to-elasticsearch.html#upload-data-kibana[File Uploader] in the UI. +After your data is analyzed, click **Override settings**. +Under **Edit field names**, assign `id` to the first column and `content` to the second. +Click **Apply**, then **Import**. +Name the index `test-data`, and click **Import**. +After the upload is complete, you will see an index named `test-data` with 182,469 documents. [discrete] [[reindexing-data-elser]] @@ -161,6 +161,18 @@ GET _tasks/ You can also open the Trained Models UI, select the Pipelines tab under ELSER to follow the progress. +Reindexing large datasets can take a long time. +You can test this workflow using only a subset of the dataset. +Do this by cancelling the reindexing process, and only generating embeddings for the subset that was reindexed. +The following API request will cancel the reindexing task: + +[source,console] +---- +POST _tasks//_cancel +---- +// TEST[skip:TBD] + + [discrete] [[text-expansion-query]] ==== Semantic search by using the `sparse_vector` query diff --git a/docs/reference/search/search-your-data/semantic-search-inference.asciidoc b/docs/reference/search/search-your-data/semantic-search-inference.asciidoc index dee91a6aa4ec4..c2fcb88380f53 100644 --- a/docs/reference/search/search-your-data/semantic-search-inference.asciidoc +++ b/docs/reference/search/search-your-data/semantic-search-inference.asciidoc @@ -39,12 +39,13 @@ Create an {infer} endpoint by using the <>: include::{es-ref-dir}/tab-widgets/inference-api/infer-api-task-widget.asciidoc[] + [discrete] [[infer-service-mappings]] ==== Create the index mapping The mapping of the destination index - the index that contains the embeddings that the model will create based on your input text - must be created. -The destination index must have a field with the <> field type for most models and the <> field type for the sparse vector models like in the case of the `elser` service to index the output of the used model. +The destination index must have a field with the <> field type for most models and the <> field type for the sparse vector models like in the case of the `elasticsearch` service to index the output of the used model. include::{es-ref-dir}/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc[] @@ -67,12 +68,12 @@ It consists of 200 queries, each accompanied by a list of relevant text passages All unique passages, along with their IDs, have been extracted from that data set and compiled into a https://github.com/elastic/stack-docs/blob/main/docs/en/stack/ml/nlp/data/msmarco-passagetest2019-unique.tsv[tsv file]. -Download the file and upload it to your cluster using the -{kibana-ref}/connect-to-elasticsearch.html#upload-data-kibana[Data Visualizer] -in the {ml-app} UI. -Assign the name `id` to the first column and `content` to the second column. -The index name is `test-data`. -Once the upload is complete, you can see an index named `test-data` with 182469 documents. +Download the file and upload it to your cluster using the {kibana-ref}/connect-to-elasticsearch.html#upload-data-kibana[Data Visualizer] in the {ml-app} UI. +After your data is analyzed, click **Override settings**. +Under **Edit field names**, assign `id` to the first column and `content` to the second. +Click **Apply**, then **Import**. +Name the index `test-data`, and click **Import**. +After the upload is complete, you will see an index named `test-data` with 182,469 documents. [discrete] [[reindexing-data-infer]] @@ -91,7 +92,10 @@ GET _tasks/ ---- // TEST[skip:TBD] -You can also cancel the reindexing process if you don't want to wait until the reindexing process is fully complete which might take hours for large data sets: +Reindexing large datasets can take a long time. +You can test this workflow using only a subset of the dataset. +Do this by cancelling the reindexing process, and only generating embeddings for the subset that was reindexed. +The following API request will cancel the reindexing task: [source,console] ---- diff --git a/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc b/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc index 2b8b6c9c25afe..50a9da4af2fba 100644 --- a/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc +++ b/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc @@ -14,55 +14,23 @@ You don't need to define model related settings and parameters, or create {infer The recommended way to use <> in the {stack} is following the `semantic_text` workflow. When you need more control over indexing and query settings, you can still use the complete {infer} workflow (refer to <> to review the process). -This tutorial uses the <> for demonstration, but you can use any service and their supported models offered by the {infer-cap} API. +This tutorial uses the <> for demonstration, but you can use any service and their supported models offered by the {infer-cap} API. [discrete] [[semantic-text-requirements]] ==== Requirements -To use the `semantic_text` field type, you must have an {infer} endpoint deployed in -your cluster using the <>. +This tutorial uses the <> for demonstration, which is created automatically as needed. +To use the `semantic_text` field type with an {infer} service other than `elasticsearch` service, you must create an inference endpoint using the <>. -[discrete] -[[semantic-text-infer-endpoint]] -==== Create the {infer} endpoint - -Create an inference endpoint by using the <>: - -[source,console] ------------------------------------------------------------- -PUT _inference/sparse_embedding/my-elser-endpoint <1> -{ - "service": "elser", <2> - "service_settings": { - "num_allocations": 1, - "num_threads": 1 - } -} ------------------------------------------------------------- -// TEST[skip:TBD] -<1> The task type is `sparse_embedding` in the path as the `elser` service will -be used and ELSER creates sparse vectors. The `inference_id` is -`my-elser-endpoint`. -<2> The `elser` service is used in this example. - -[NOTE] -==== -You might see a 502 bad gateway error in the response when using the {kib} Console. -This error usually just reflects a timeout, while the model downloads in the background. -You can check the download progress in the {ml-app} UI. -If using the Python client, you can set the `timeout` parameter to a higher value. -==== [discrete] [[semantic-text-index-mapping]] ==== Create the index mapping -The mapping of the destination index - the index that contains the embeddings -that the inference endpoint will generate based on your input text - must be created. The -destination index must have a field with the <> -field type to index the output of the used inference endpoint. +The mapping of the destination index - the index that contains the embeddings that the inference endpoint will generate based on your input text - must be created. +The destination index must have a field with the <> field type to index the output of the used inference endpoint. [source,console] ------------------------------------------------------------ @@ -70,13 +38,8 @@ PUT semantic-embeddings { "mappings": { "properties": { - "semantic_text": { <1> - "type": "semantic_text", <2> - "inference_id": "my-elser-endpoint" <3> - }, - "content": { <4> - "type": "text", - "copy_to": "semantic_text" <5> + "content": { <1> + "type": "semantic_text" <2> } } } @@ -85,12 +48,16 @@ PUT semantic-embeddings // TEST[skip:TBD] <1> The name of the field to contain the generated embeddings. <2> The field to contain the embeddings is a `semantic_text` field. -<3> The `inference_id` is the inference endpoint you created in the previous step. -It will be used to generate the embeddings based on the input text. -Every time you ingest data into the related `semantic_text` field, this endpoint will be used for creating the vector representation of the text. -<4> The field to store the text reindexed from a source index in the <> step. -<5> The textual data stored in the `content` field will be copied to `semantic_text` and processed by the {infer} endpoint. -The `semantic_text` field will store the embeddings generated based on the input data. +Since no `inference_id` is provided, the default endpoint `.elser-2-elasticsearch` for the <> is used. +To use a different {infer} service, you must create an {infer} endpoint first using the <> and then specify it in the `semantic_text` field mapping using the `inference_id` parameter. + + +[NOTE] +==== +If you're using web crawlers or connectors to generate indices, you have to <> for these indices to include the `semantic_text` field. +Once the mapping is updated, you'll need to run a full web crawl or a full connector sync. +This ensures that all existing documents are reprocessed and updated with the new semantic embeddings, enabling semantic search on the updated data. +==== [discrete] @@ -105,24 +72,21 @@ a list of relevant text passages. All unique passages, along with their IDs, have been extracted from that data set and compiled into a https://github.com/elastic/stack-docs/blob/main/docs/en/stack/ml/nlp/data/msmarco-passagetest2019-unique.tsv[tsv file]. -Download the file and upload it to your cluster using the -{kibana-ref}/connect-to-elasticsearch.html#upload-data-kibana[Data Visualizer] -in the {ml-app} UI. Assign the name `id` to the first column and `content` to -the second column. The index name is `test-data`. Once the upload is complete, -you can see an index named `test-data` with 182469 documents. +Download the file and upload it to your cluster using the {kibana-ref}/connect-to-elasticsearch.html#upload-data-kibana[Data Visualizer] in the {ml-app} UI. +After your data is analyzed, click **Override settings**. +Under **Edit field names**, assign `id` to the first column and `content` to the second. +Click **Apply**, then **Import**. +Name the index `test-data`, and click **Import**. +After the upload is complete, you will see an index named `test-data` with 182,469 documents. [discrete] [[semantic-text-reindex-data]] ==== Reindex the data -Create the embeddings from the text by reindexing the data from the `test-data` -index to the `semantic-embeddings` index. The data in the `content` field will -be reindexed into the `content` field of the destination index. -The `content` field data will be copied to the `semantic_text` field as a result of the `copy_to` -parameter set in the index mapping creation step. The copied data will be -processed by the {infer} endpoint associated with the `semantic_text` semantic text -field. +Create the embeddings from the text by reindexing the data from the `test-data` index to the `semantic-embeddings` index. +The data in the `content` field will be reindexed into the `content` semantic text field of the destination index. +The reindexed data will be processed by the {infer} endpoint associated with the `content` semantic text field. [source,console] ------------------------------------------------------------ @@ -150,8 +114,10 @@ GET _tasks/ ------------------------------------------------------------ // TEST[skip:TBD] -It is recommended to cancel the reindexing process if you don't want to wait -until it is fully complete which might take a long time for an inference endpoint with few assigned resources: +Reindexing large datasets can take a long time. +You can test this workflow using only a subset of the dataset. +Do this by cancelling the reindexing process, and only generating embeddings for the subset that was reindexed. +The following API request will cancel the reindexing task: [source,console] ------------------------------------------------------------ @@ -164,10 +130,9 @@ POST _tasks//_cancel [[semantic-text-semantic-search]] ==== Semantic search -After the data set has been enriched with the embeddings, you can query the data -using semantic search. Provide the `semantic_text` field name and the query text -in a `semantic` query type. The {infer} endpoint used to generate the embeddings -for the `semantic_text` field will be used to process the query text. +After the data set has been enriched with the embeddings, you can query the data using semantic search. +Provide the `semantic_text` field name and the query text in a `semantic` query type. +The {infer} endpoint used to generate the embeddings for the `semantic_text` field will be used to process the query text. [source,console] ------------------------------------------------------------ @@ -175,7 +140,7 @@ GET semantic-embeddings/_search { "query": { "semantic": { - "field": "semantic_text", <1> + "field": "content", <1> "query": "How to avoid muscle soreness while running?" <2> } } @@ -186,88 +151,12 @@ GET semantic-embeddings/_search <2> The query text. As a result, you receive the top 10 documents that are closest in meaning to the -query from the `semantic-embedding` index: +query from the `semantic-embedding` index. -[source,console-result] ------------------------------------------------------------- -"hits": [ - { - "_index": "semantic-embeddings", - "_id": "6DdEuo8B0vYIvzmhoEtt", - "_score": 24.972616, - "_source": { - "semantic_text": { - "inference": { - "inference_id": "my-elser-endpoint", - "model_settings": { - "task_type": "sparse_embedding" - }, - "chunks": [ - { - "text": "There are a few foods and food groups that will help to fight inflammation and delayed onset muscle soreness (both things that are inevitable after a long, hard workout) when you incorporate them into your postworkout eats, whether immediately after your run or at a meal later in the day. Advertisement. Advertisement.", - "embeddings": { - (...) - } - } - ] - } - }, - "id": 1713868, - "content": "There are a few foods and food groups that will help to fight inflammation and delayed onset muscle soreness (both things that are inevitable after a long, hard workout) when you incorporate them into your postworkout eats, whether immediately after your run or at a meal later in the day. Advertisement. Advertisement." - } - }, - { - "_index": "semantic-embeddings", - "_id": "-zdEuo8B0vYIvzmhplLX", - "_score": 22.143118, - "_source": { - "semantic_text": { - "inference": { - "inference_id": "my-elser-endpoint", - "model_settings": { - "task_type": "sparse_embedding" - }, - "chunks": [ - { - "text": "During Your Workout. There are a few things you can do during your workout to help prevent muscle injury and soreness. According to personal trainer and writer for Iron Magazine, Marc David, doing warm-ups and cool-downs between sets can help keep muscle soreness to a minimum.", - "embeddings": { - (...) - } - } - ] - } - }, - "id": 3389244, - "content": "During Your Workout. There are a few things you can do during your workout to help prevent muscle injury and soreness. According to personal trainer and writer for Iron Magazine, Marc David, doing warm-ups and cool-downs between sets can help keep muscle soreness to a minimum." - } - }, - { - "_index": "semantic-embeddings", - "_id": "77JEuo8BdmhTuQdXtQWt", - "_score": 21.506052, - "_source": { - "semantic_text": { - "inference": { - "inference_id": "my-elser-endpoint", - "model_settings": { - "task_type": "sparse_embedding" - }, - "chunks": [ - { - "text": "This is especially important if the soreness is due to a weightlifting routine. For this time period, do not exert more than around 50% of the level of effort (weight, distance and speed) that caused the muscle groups to be sore.", - "embeddings": { - (...) - } - } - ] - } - }, - "id": 363742, - "content": "This is especially important if the soreness is due to a weightlifting routine. For this time period, do not exert more than around 50% of the level of effort (weight, distance and speed) that caused the muscle groups to be sore." - } - }, - (...) -] ------------------------------------------------------------- -// NOTCONSOLE +[discrete] +[[semantic-text-further-examples]] +==== Further examples and reading +* If you want to use `semantic_text` in hybrid search, refer to https://colab.research.google.com/github/elastic/elasticsearch-labs/blob/main/notebooks/search/09-semantic-text.ipynb[this notebook] for a step-by-step guide. +* For more information on how to optimize your ELSER endpoints, refer to {ml-docs}/ml-nlp-elser.html#elser-recommendations[the ELSER recommendations] section in the model documentation. +* To learn more about model autoscaling, refer to the {ml-docs}/ml-nlp-auto-scale.html[trained model autoscaling] page. diff --git a/docs/reference/search/search-your-data/semantic-search.asciidoc b/docs/reference/search/search-your-data/semantic-search.asciidoc index fa84c3848b78c..e0fb8415fee18 100644 --- a/docs/reference/search/search-your-data/semantic-search.asciidoc +++ b/docs/reference/search/search-your-data/semantic-search.asciidoc @@ -7,109 +7,95 @@ Semantic search is a search method that helps you find data based on the intent Using an NLP model enables you to extract text embeddings out of text. Embeddings are vectors that provide a numeric representation of a text. Pieces of content with similar meaning have similar representations. -NLP models can be used in the {stack} various ways, you can: -* deploy models in {es} -* use the <> (recommended) -* use the <> +image::images/semantic-options.svg[Overview of semantic search workflows in {es}] +You have several options for using NLP models in the {stack}: -[[semantic-search-diagram]] -.A simplified representation of encoding textual concepts as vectors -image::images/search/vector-search-oversimplification.png[A simplified representation of encoding textual concepts as vectors,align="center"] +* use the `semantic_text` workflow (recommended) +* use the {infer} API workflow +* deploy models directly in {es} -At query time, {es} can use the same NLP model to convert a query into embeddings, enabling you to find documents with similar text embeddings. +Refer to <> to choose your workflow. -This guide shows you how to implement semantic search with {es}: From selecting an NLP model, to writing queries. +You can also store your own embeddings in {es} as vectors. +Refer to <> for guidance on which query type to use for semantic search. + +At query time, {es} can use the same NLP model to convert a query into embeddings, enabling you to find documents with similar text embeddings. -IMPORTANT: For the easiest way to perform semantic search in the {stack}, refer to the <> end-to-end tutorial. [discrete] -[[semantic-search-select-nlp-model]] -=== Select an NLP model - -{es} offers the usage of a -{ml-docs}/ml-nlp-model-ref.html#ml-nlp-model-ref-text-embedding[wide range of NLP models], including both dense and sparse vector models. -Your choice of the language model is critical for implementing semantic search successfully. - -While it is possible to bring your own text embedding model, achieving good search results through model tuning is challenging. -Selecting an appropriate model from our third-party model list is the first step. -Training the model on your own data is essential to ensure better search results than using only BM25. -However, the model training process requires a team of data scientists and ML experts, making it expensive and time-consuming. - -To address this issue, Elastic provides a pre-trained representational model called {ml-docs}/ml-nlp-elser.html[Elastic Learned Sparse EncodeR (ELSER)]. -ELSER, currently available only for English, is an out-of-domain sparse vector model that does not require fine-tuning. -This adaptability makes it suitable for various NLP use cases out of the box. -Unless you have a team of ML specialists, it is highly recommended to use the ELSER model. - -In the case of sparse vector representation, the vectors mostly consist of zero values, with only a small subset containing non-zero values. -This representation is commonly used for textual data. -In the case of ELSER, each document in an index and the query text itself are represented by high-dimensional sparse vectors. -Each non-zero element of the vector corresponds to a term in the model vocabulary. -The ELSER vocabulary contains around 30000 terms, so the sparse vectors created by ELSER contain about 30000 values, the majority of which are zero. -Effectively the ELSER model is replacing the terms in the original query with other terms that have been learnt to exist in the documents that best match the original search terms in a training dataset, and weights to control how important each is. +[[using-nlp-models]] +=== Choose a semantic search workflow [discrete] -[[semantic-search-deploy-nlp-model]] -=== Deploy the model +==== `semantic_text` workflow -After you decide which model you want to use for implementing semantic search, you need to deploy the model in {es}. +The simplest way to use NLP models in the {stack} is through the <>. +We recommend using this approach because it abstracts away a lot of manual work. +All you need to do is create an {infer} endpoint and an index mapping to start ingesting, embedding, and querying data. +There is no need to define model-related settings and parameters, or to create {infer} ingest pipelines. +Refer to the <> documentation for a list of supported services. -include::{es-ref-dir}/tab-widgets/semantic-search/deploy-nlp-model-widget.asciidoc[] +The <> tutorial shows you the process end-to-end. [discrete] -[[semantic-search-field-mappings]] -=== Map a field for the text embeddings +==== {infer} API workflow -Before you start using the deployed model to generate embeddings based on your input text, you need to prepare your index mapping first. -The mapping of the index depends on the type of model. +The <> is more complex but offers greater control over the {infer} endpoint configuration. +You need to create an {infer} endpoint, provide various model-related settings and parameters, define an index mapping, and set up an {infer} ingest pipeline with the appropriate settings. -include::{es-ref-dir}/tab-widgets/semantic-search/field-mappings-widget.asciidoc[] +The <> tutorial shows you the process end-to-end. [discrete] -[[semantic-search-generate-embeddings]] -=== Generate text embeddings +==== Model deployment workflow -Once you have created the mappings for the index, you can generate text embeddings from your input text. -This can be done by using an -<> with an <>. -The ingest pipeline processes the input data and indexes it into the destination index. -At index time, the inference ingest processor uses the trained model to infer against the data ingested through the pipeline. -After you created the ingest pipeline with the inference processor, you can ingest your data through it to generate the model output. +You can also deploy NLP in {es} manually, without using an {infer} endpoint. +This is the most complex and labor intensive workflow for performing semantic search in the {stack}. +You need to select an NLP model from the {ml-docs}/ml-nlp-model-ref.html#ml-nlp-model-ref-text-embedding[list of supported dense and sparse vector models], deploy it using the Eland client, create an index mapping, and set up a suitable ingest pipeline to start ingesting and querying data. -include::{es-ref-dir}/tab-widgets/semantic-search/generate-embeddings-widget.asciidoc[] +The <> tutorial shows you the process end-to-end. -Now it is time to perform semantic search! [discrete] -[[semantic-search-search]] -=== Search the data +[[using-query]] +=== Using the right query -Depending on the type of model you have deployed, you can query rank features with a <> query, or dense vectors with a kNN search. +Crafting the right query is crucial for semantic search. +Which query you use and which field you target in your queries depends on your chosen workflow. +If you're using the `semantic_text` workflow it's quite simple. +If not, it depends on which type of embeddings you're working with. -include::{es-ref-dir}/tab-widgets/semantic-search/search-widget.asciidoc[] +[cols="30%, 30%, 40%", options="header"] +|======================================================================================================================================================================================================= +| Field type to query | Query to use | Notes +| <> | <> | The `semantic_text` field handles generating embeddings for you at index time and query time. +| <> | <> | The `sparse_vector` query can generate query embeddings for you, but you can also provide your own. You must provide embeddings at index time. +| <> | <> | The `knn` query can generate query embeddings for you, but you can also provide your own. You must provide embeddings at index time. +|======================================================================================================================================================================================================= -[discrete] -[[semantic-search-hybrid-search]] -=== Beyond semantic search with hybrid search - -In some situations, lexical search may perform better than semantic search. -For example, when searching for single words or IDs, like product numbers. +If you want {es} to generate embeddings at both index and query time, use the `semantic_text` field and the `semantic` query. +If you want to bring your own embeddings, use the `sparse_vector` or `dense_vector` field type and the associated query depending on the NLP model you used to generate the embeddings. -Combining semantic and lexical search into one hybrid search request using -<> provides the best of both worlds. -Not only that, but hybrid search using reciprocal rank fusion {blog-ref}improving-information-retrieval-elastic-stack-hybrid[has been shown to perform better in general]. +IMPORTANT: For the easiest way to perform semantic search in the {stack}, refer to the <> end-to-end tutorial. -include::{es-ref-dir}/tab-widgets/semantic-search/hybrid-search-widget.asciidoc[] [discrete] [[semantic-search-read-more]] === Read more * Tutorials: -** <> +** <> +** <> +** <> using the model deployment workflow +** <> ** {ml-docs}/ml-nlp-text-emb-vector-search-example.html[Semantic search with the msmarco-MiniLM-L-12-v3 sentence-transformer model] +* Interactive examples: +** The https://github.com/elastic/elasticsearch-labs[`elasticsearch-labs`] repo contains a number of interactive semantic search examples in the form of executable Python notebooks, using the {es} Python client +** https://github.com/elastic/elasticsearch-labs/blob/main/notebooks/search/03-ELSER.ipynb[Semantic search with ELSER using the model deployment workflow] +** https://github.com/elastic/elasticsearch-labs/blob/main/notebooks/search/09-semantic-text.ipynb[Semantic search with `semantic_text`] * Blogs: +** https://www.elastic.co/search-labs/blog/semantic-search-simplified-semantic-text[{es} new semantic_text mapping: Simplifying semantic search] ** {blog-ref}may-2023-launch-sparse-encoder-ai-model[Introducing Elastic Learned Sparse Encoder: Elastic's AI model for semantic search] ** {blog-ref}lexical-ai-powered-search-elastic-vector-database[How to get the best of lexical and AI-powered search with Elastic's vector database] ** Information retrieval blog series: @@ -117,10 +103,12 @@ include::{es-ref-dir}/tab-widgets/semantic-search/hybrid-search-widget.asciidoc[ *** {blog-ref}improving-information-retrieval-elastic-stack-benchmarking-passage-retrieval[Part 2: Benchmarking passage retrieval] *** {blog-ref}may-2023-launch-information-retrieval-elasticsearch-ai-model[Part 3: Introducing Elastic Learned Sparse Encoder, our new retrieval model] *** {blog-ref}improving-information-retrieval-elastic-stack-hybrid[Part 4: Hybrid retrieval] -* Interactive examples: -** The https://github.com/elastic/elasticsearch-labs[`elasticsearch-labs`] repo contains a number of interactive semantic search examples in the form of executable Python notebooks, using the {es} Python client -include::semantic-search-elser.asciidoc[] + include::semantic-search-semantic-text.asciidoc[] +include::semantic-text-hybrid-search[] include::semantic-search-inference.asciidoc[] +include::semantic-search-elser.asciidoc[] include::cohere-es.asciidoc[] +include::semantic-search-deploy-model.asciidoc[] +include::ingest-vectors.asciidoc[] diff --git a/docs/reference/search/search-your-data/semantic-text-hybrid-search b/docs/reference/search/search-your-data/semantic-text-hybrid-search new file mode 100644 index 0000000000000..4b49a7c3155db --- /dev/null +++ b/docs/reference/search/search-your-data/semantic-text-hybrid-search @@ -0,0 +1,217 @@ +[[semantic-text-hybrid-search]] +=== Tutorial: hybrid search with `semantic_text` +++++ +Hybrid search with `semantic_text` +++++ + +This tutorial demonstrates how to perform hybrid search, combining semantic search with traditional full-text search. + +In hybrid search, semantic search retrieves results based on the meaning of the text, while full-text search focuses on exact word matches. By combining both methods, hybrid search delivers more relevant results, particularly in cases where relying on a single approach may not be sufficient. + +The recommended way to use hybrid search in the {stack} is following the `semantic_text` workflow. +This tutorial uses the <> for demonstration, but you can use any service and their supported models offered by the {infer-cap} API. + +[discrete] +[[hybrid-search-create-index-mapping]] +==== Create an index mapping + +The destination index will contain both the embeddings for semantic search and the original text field for full-text search. This structure enables the combination of semantic search and full-text search. + +[source,console] +------------------------------------------------------------ +PUT semantic-embeddings +{ + "mappings": { + "properties": { + "semantic_text": { <1> + "type": "semantic_text", + }, + "content": { <2> + "type": "text", + "copy_to": "semantic_text" <3> + } + } + } +} +------------------------------------------------------------ +// TEST[skip:TBD] +<1> The name of the field to contain the generated embeddings for semantic search. +<2> The name of the field to contain the original text for lexical search. +<3> The textual data stored in the `content` field will be copied to `semantic_text` and processed by the {infer} endpoint. + +[NOTE] +==== +If you want to run a search on indices that were populated by web crawlers or connectors, you have to +<> for these indices to +include the `semantic_text` field. Once the mapping is updated, you'll need to run a full web crawl or a full connector sync. This ensures that all existing +documents are reprocessed and updated with the new semantic embeddings, enabling hybrid search on the updated data. +==== + +[discrete] +[[semantic-text-hybrid-load-data]] +==== Load data + +In this step, you load the data that you later use to create embeddings from. + +Use the `msmarco-passagetest2019-top1000` data set, which is a subset of the MS MARCO Passage Ranking data set. It consists of 200 queries, each accompanied by a list of relevant text passages. All unique passages, along with their IDs, have been extracted from that data set and compiled into a https://github.com/elastic/stack-docs/blob/main/docs/en/stack/ml/nlp/data/msmarco-passagetest2019-unique.tsv[tsv file]. + +Download the file and upload it to your cluster using the {kibana-ref}/connect-to-elasticsearch.html#upload-data-kibana[Data Visualizer] in the {ml-app} UI. After your data is analyzed, click **Override settings**. Under **Edit field names**, assign `id` to the first column and `content` to the second. Click **Apply**, then **Import**. Name the index `test-data`, and click **Import**. After the upload is complete, you will see an index named `test-data` with 182,469 documents. + +[discrete] +[[hybrid-search-reindex-data]] +==== Reindex the data for hybrid search + +Reindex the data from the `test-data` index into the `semantic-embeddings` index. +The data in the `content` field of the source index is copied into the `content` field of the destination index. +The `copy_to` parameter set in the index mapping creation ensures that the content is copied into the `semantic_text` field. The data is processed by the {infer} endpoint at ingest time to generate embeddings. + +[NOTE] +==== +This step uses the reindex API to simulate data ingestion. If you are working with data that has already been indexed, +rather than using the `test-data` set, reindexing is still required to ensure that the data is processed by the {infer} endpoint +and the necessary embeddings are generated. +==== + +[source,console] +------------------------------------------------------------ +POST _reindex?wait_for_completion=false +{ + "source": { + "index": "test-data", + "size": 10 <1> + }, + "dest": { + "index": "semantic-embeddings" + } +} +------------------------------------------------------------ +// TEST[skip:TBD] +<1> The default batch size for reindexing is 1000. Reducing size to a smaller +number makes the update of the reindexing process quicker which enables you to +follow the progress closely and detect errors early. + +The call returns a task ID to monitor the progress: + +[source,console] +------------------------------------------------------------ +GET _tasks/ +------------------------------------------------------------ +// TEST[skip:TBD] + +Reindexing large datasets can take a long time. You can test this workflow using only a subset of the dataset. + +To cancel the reindexing process and generate embeddings for the subset that was reindexed: + +[source,console] +------------------------------------------------------------ +POST _tasks//_cancel +------------------------------------------------------------ +// TEST[skip:TBD] + +[discrete] +[[hybrid-search-perform-search]] +==== Perform hybrid search + +After reindexing the data into the `semantic-embeddings` index, you can perform hybrid search by using <>. RRF is a technique that merges the rankings from both semantic and lexical queries, giving more weight to results that rank high in either search. This ensures that the final results are balanced and relevant. + +[source,console] +------------------------------------------------------------ +GET semantic-embeddings/_search +{ + "retriever": { + "rrf": { + "retrievers": [ + { + "standard": { <1> + "query": { + "match": { + "content": "How to avoid muscle soreness while running?" <2> + } + } + } + }, + { + "standard": { <3> + "query": { + "semantic": { + "field": "semantic_text", <4> + "query": "How to avoid muscle soreness while running?" + } + } + } + } + ] + } + } +} +------------------------------------------------------------ +// TEST[skip:TBD] +<1> The first `standard` retriever represents the traditional lexical search. +<2> Lexical search is performed on the `content` field using the specified phrase. +<3> The second `standard` retriever refers to the semantic search. +<4> The `semantic_text` field is used to perform the semantic search. + + +After performing the hybrid search, the query will return the top 10 documents that match both semantic and lexical search criteria. The results include detailed information about each document: + +[source,console-result] +------------------------------------------------------------ +{ + "took": 107, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 473, + "relation": "eq" + }, + "max_score": null, + "hits": [ + { + "_index": "semantic-embeddings", + "_id": "wv65epIBEMBRnhfTsOFM", + "_score": 0.032786883, + "_rank": 1, + "_source": { + "semantic_text": { + "inference": { + "inference_id": "my-elser-endpoint", + "model_settings": { + "task_type": "sparse_embedding" + }, + "chunks": [ + { + "text": "What so many out there do not realize is the importance of what you do after you work out. You may have done the majority of the work, but how you treat your body in the minutes and hours after you exercise has a direct effect on muscle soreness, muscle strength and growth, and staying hydrated. Cool Down. After your last exercise, your workout is not over. The first thing you need to do is cool down. Even if running was all that you did, you still should do light cardio for a few minutes. This brings your heart rate down at a slow and steady pace, which helps you avoid feeling sick after a workout.", + "embeddings": { + "exercise": 1.571044, + "after": 1.3603843, + "sick": 1.3281639, + "cool": 1.3227621, + "muscle": 1.2645415, + "sore": 1.2561599, + "cooling": 1.2335974, + "running": 1.1750668, + "hours": 1.1104802, + "out": 1.0991782, + "##io": 1.0794281, + "last": 1.0474665, + (...) + } + } + ] + } + }, + "id": 8408852, + "content": "What so many out there do not realize is the importance of (...)" + } + } + ] + } +} +------------------------------------------------------------ +// NOTCONSOLE diff --git a/docs/reference/search/search.asciidoc b/docs/reference/search/search.asciidoc index 501d645665a02..d022605db22b1 100644 --- a/docs/reference/search/search.asciidoc +++ b/docs/reference/search/search.asciidoc @@ -4,6 +4,12 @@ Search ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs]. +-- + Returns search hits that match the query defined in the request. [source,console] @@ -38,7 +44,7 @@ must have the `read` index privilege for the alias's data streams or indices. Allows you to execute a search query and get back search hits that match the query. You can provide search queries using the <> or <>. +query string parameter>> or <>. [[search-search-api-path-params]] ==== {api-path-parms-title} diff --git a/docs/reference/search/suggesters.asciidoc b/docs/reference/search/suggesters.asciidoc index c5cdbc3bd6660..22f427477b905 100644 --- a/docs/reference/search/suggesters.asciidoc +++ b/docs/reference/search/suggesters.asciidoc @@ -3,6 +3,12 @@ Suggests similar looking terms based on a provided text by using a suggester. +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs]. +-- + [source,console] -------------------------------------------------- POST my-index-000001/_search diff --git a/docs/reference/search/terms-enum.asciidoc b/docs/reference/search/terms-enum.asciidoc index 4f34deb985abe..46f6d3560ecdb 100644 --- a/docs/reference/search/terms-enum.asciidoc +++ b/docs/reference/search/terms-enum.asciidoc @@ -4,6 +4,12 @@ Terms enum ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-search[Search APIs]. +-- + The terms enum API can be used to discover terms in the index that match a partial string. Supported field types are <>, <>, <>, diff --git a/docs/reference/search/validate.asciidoc b/docs/reference/search/validate.asciidoc index ce682e485cd27..ab943cf72b463 100644 --- a/docs/reference/search/validate.asciidoc +++ b/docs/reference/search/validate.asciidoc @@ -4,6 +4,12 @@ Validate ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/operation/operation-indices-validate-query[Validate a query]. +-- + Validates a potentially expensive query without executing it. [source,console] diff --git a/docs/reference/searchable-snapshots/apis/clear-cache.asciidoc b/docs/reference/searchable-snapshots/apis/clear-cache.asciidoc index ff67be02e6d00..ac357e29d1d17 100644 --- a/docs/reference/searchable-snapshots/apis/clear-cache.asciidoc +++ b/docs/reference/searchable-snapshots/apis/clear-cache.asciidoc @@ -5,6 +5,12 @@ Clear cache ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-searchable_snapshots[Searchable snapshots APIs]. +-- + experimental::[] Clears indices and data streams from the shared cache for diff --git a/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc b/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc index b47bc2370ab10..f1613ea62492d 100644 --- a/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc +++ b/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc @@ -5,6 +5,12 @@ Mount snapshot ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-searchable_snapshots[Searchable snapshots APIs]. +-- + Mount a snapshot as a searchable snapshot index. [[searchable-snapshots-api-mount-request]] diff --git a/docs/reference/searchable-snapshots/apis/node-cache-stats.asciidoc b/docs/reference/searchable-snapshots/apis/node-cache-stats.asciidoc index 62faceb99d4fc..d42ba02876941 100644 --- a/docs/reference/searchable-snapshots/apis/node-cache-stats.asciidoc +++ b/docs/reference/searchable-snapshots/apis/node-cache-stats.asciidoc @@ -5,6 +5,12 @@ Cache stats ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-searchable_snapshots[Searchable snapshots APIs]. +-- + Retrieves statistics about the shared cache for <>. diff --git a/docs/reference/searchable-snapshots/apis/searchable-snapshots-apis.asciidoc b/docs/reference/searchable-snapshots/apis/searchable-snapshots-apis.asciidoc index f90e27ea63224..1a1856198d286 100644 --- a/docs/reference/searchable-snapshots/apis/searchable-snapshots-apis.asciidoc +++ b/docs/reference/searchable-snapshots/apis/searchable-snapshots-apis.asciidoc @@ -2,6 +2,12 @@ [[searchable-snapshots-apis]] == Searchable snapshots APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-searchable_snapshots[Searchable snapshots APIs]. +-- + You can use the following APIs to perform searchable snapshots operations. * <> diff --git a/docs/reference/searchable-snapshots/apis/shard-stats.asciidoc b/docs/reference/searchable-snapshots/apis/shard-stats.asciidoc index 369d19da9ae56..f1eb9ac8d92b2 100644 --- a/docs/reference/searchable-snapshots/apis/shard-stats.asciidoc +++ b/docs/reference/searchable-snapshots/apis/shard-stats.asciidoc @@ -5,6 +5,12 @@ Searchable snapshot statistics ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-searchable_snapshots[Searchable snapshots APIs]. +-- + experimental::[] Retrieves statistics about searchable snapshots. diff --git a/docs/reference/searchable-snapshots/index.asciidoc b/docs/reference/searchable-snapshots/index.asciidoc index a38971a0bae6a..8b0b3dc57686e 100644 --- a/docs/reference/searchable-snapshots/index.asciidoc +++ b/docs/reference/searchable-snapshots/index.asciidoc @@ -176,10 +176,10 @@ nodes that have a shared cache. ==== Manually mounting snapshots captured by an Index Lifecycle Management ({ilm-init}) policy can interfere with {ilm-init}'s automatic management. This may lead to issues such as data loss -or complications with snapshot handling. +or complications with snapshot handling. For optimal results, allow {ilm-init} to manage -snapshots automatically. +snapshots automatically. <>. ==== @@ -293,6 +293,14 @@ repository. If you wish to search data across multiple regions, configure multiple clusters and use <> or <> instead of {search-snaps}. +It's worth noting that if a searchable snapshot index has no replicas, then when the node +hosting it is shut down, allocation will immediately try to relocate the index to a new node +in order to maximize availability. For fully mounted indices this will result in the new node +downloading the entire index snapshot from the cloud repository. Under a rolling cluster restart, +this may happen multiple times for each searchable snapshot index. Temporarily +disabling allocation during planned node restart will prevent this, as described in +the <>. + [discrete] [[back-up-restore-searchable-snapshots]] === Back up and restore {search-snaps} diff --git a/docs/reference/security/authentication/remote-clusters-privileges-api-key.asciidoc b/docs/reference/security/authentication/remote-clusters-privileges-api-key.asciidoc index 1d31c7b6b9345..9b51a58725f39 100644 --- a/docs/reference/security/authentication/remote-clusters-privileges-api-key.asciidoc +++ b/docs/reference/security/authentication/remote-clusters-privileges-api-key.asciidoc @@ -2,7 +2,8 @@ === Configure roles and users To use a remote cluster for {ccr} or {ccs}, you need to create user roles with -<> on the local cluster. +<> or +<> on the local cluster. You can manage users and roles from Stack Management in {kib} by selecting *Security > Roles* from the side navigation. You can also use the @@ -80,7 +81,7 @@ POST /_security/role/remote-search "privileges": [ "read", "read_cross_cluster", - "view_index_metadata" + "view_index_metadata" ] } ] diff --git a/docs/reference/security/authentication/saml-guide.asciidoc b/docs/reference/security/authentication/saml-guide.asciidoc index cf91e11b7f18f..45d153a071ffd 100644 --- a/docs/reference/security/authentication/saml-guide.asciidoc +++ b/docs/reference/security/authentication/saml-guide.asciidoc @@ -32,7 +32,7 @@ that supports at least the SAML 2.0 _Web Browser SSO Profile_. It has been tested with a number of popular IdP implementations, such as https://www.elastic.co/blog/how-to-configure-elasticsearch-saml-authentication-with-adfs[Microsoft Active Directory Federation Services (ADFS)], https://www.elastic.co/blog/saml-based-single-sign-on-with-elasticsearch-and-azure-active-directory[Azure Active Directory (AAD)], -and https://www.elastic.co/blog/setting-up-saml-for-elastic-enterprise-search-okta-edition[Okta]. +and https://www.elastic.co/blog/how-to-set-up-okta-saml-login-kibana-elastic-cloud[Okta]. This guide assumes that you have an existing IdP and wish to add {kib} as a Service Provider. diff --git a/docs/reference/security/authorization/managing-roles.asciidoc b/docs/reference/security/authorization/managing-roles.asciidoc index 535d70cbc5e9c..0c3f520605f07 100644 --- a/docs/reference/security/authorization/managing-roles.asciidoc +++ b/docs/reference/security/authorization/managing-roles.asciidoc @@ -249,12 +249,10 @@ The following describes the structure of a remote cluster permissions entry: <> and <>. This field is required. <2> The cluster level privileges for the remote cluster. The allowed values here are a subset of the -<>. This field is required. +<>. +The <> can be used to determine +which privileges are allowed here. This field is required. -The `monitor_enrich` privilege for remote clusters was introduced in version -8.15.0. Currently, this is the only privilege available for remote clusters and -is required to enable users to use the `ENRICH` keyword in ES|QL queries across -clusters. ==== Example diff --git a/docs/reference/security/authorization/privileges.asciidoc b/docs/reference/security/authorization/privileges.asciidoc index 747b1eef40441..3b69e5c1ba984 100644 --- a/docs/reference/security/authorization/privileges.asciidoc +++ b/docs/reference/security/authorization/privileges.asciidoc @@ -250,6 +250,11 @@ Privileges to list and view details on existing repositories and snapshots. + This privilege is not available in {serverless-full}. +`monitor_stats`:: +Privileges to list and view details of stats. ++ +This privilege is not available in {serverless-full}. + `monitor_text_structure`:: All read-only operations related to the <>. + diff --git a/docs/reference/security/ccs-clients-integrations/index.asciidoc b/docs/reference/security/ccs-clients-integrations/index.asciidoc index 11e58bb2aaf57..414e71d71b46e 100644 --- a/docs/reference/security/ccs-clients-integrations/index.asciidoc +++ b/docs/reference/security/ccs-clients-integrations/index.asciidoc @@ -13,6 +13,7 @@ be secured as well, or at least communicate with the cluster in a secured way: * <> * {auditbeat-ref}/securing-auditbeat.html[Auditbeat] * {filebeat-ref}/securing-filebeat.html[Filebeat] +* {fleet-guide}/secure.html[{fleet} & {agent}] * {heartbeat-ref}/securing-heartbeat.html[Heartbeat] * {kibana-ref}/using-kibana-with-security.html[{kib}] * {logstash-ref}/ls-security.html[Logstash] diff --git a/docs/reference/security/fips-140-compliance.asciidoc b/docs/reference/security/fips-140-compliance.asciidoc index 5bf73d43541d6..dec17927e62b8 100644 --- a/docs/reference/security/fips-140-compliance.asciidoc +++ b/docs/reference/security/fips-140-compliance.asciidoc @@ -53,8 +53,8 @@ https://docs.oracle.com/en/java/javase/17/security/java-cryptography-architectur https://docs.oracle.com/en/java/javase/17/security/java-secure-socket-extension-jsse-reference-guide.html[JSSE] implementation is required so that the JVM uses FIPS validated implementations of NIST recommended cryptographic algorithms. -Elasticsearch has been tested with Bouncy Castle's https://repo1.maven.org/maven2/org/bouncycastle/bc-fips/1.0.2.4/bc-fips-1.0.2.4.jar[bc-fips 1.0.2.4] -and https://repo1.maven.org/maven2/org/bouncycastle/bctls-fips/1.0.17/bctls-fips-1.0.17.jar[bctls-fips 1.0.17]. +Elasticsearch has been tested with Bouncy Castle's https://repo1.maven.org/maven2/org/bouncycastle/bc-fips/1.0.2.5/bc-fips-1.0.2.5.jar[bc-fips 1.0.2.5] +and https://repo1.maven.org/maven2/org/bouncycastle/bctls-fips/1.0.19/bctls-fips-1.0.19.jar[bctls-fips 1.0.19]. Please refer to the {es} https://www.elastic.co/support/matrix#matrix_jvm[JVM support matrix] for details on which combinations of JVM and security provider are supported in FIPS mode. Elasticsearch does not ship with a FIPS certified provider. It is the responsibility of the user to install and configure the security provider to ensure compliance with FIPS 140-2. Using a FIPS certified provider will ensure that only diff --git a/docs/reference/security/limitations.asciidoc b/docs/reference/security/limitations.asciidoc index 96af0e01c8075..b1bdd8cbbf5d5 100644 --- a/docs/reference/security/limitations.asciidoc +++ b/docs/reference/security/limitations.asciidoc @@ -81,12 +81,13 @@ including the following queries: * A search request cannot be profiled if document level security is enabled. * The <> does not return terms if document level security is enabled. +* The <> query does not support specifying fields using wildcards. NOTE: While document-level security prevents users from viewing restricted documents, it's still possible to write search requests that return aggregate information about the entire index. A user whose access is restricted to specific documents in an index could still learn about field names and terms that only exist in inaccessible -documents, and count how many inaccessible documents contain a given term. +documents, and count how many inaccessible documents contain a given term. [discrete] [[alias-limitations]] diff --git a/docs/reference/settings/notification-settings.asciidoc b/docs/reference/settings/notification-settings.asciidoc index 145112ef4d27c..c375ddf076a66 100644 --- a/docs/reference/settings/notification-settings.asciidoc +++ b/docs/reference/settings/notification-settings.asciidoc @@ -118,6 +118,17 @@ If you configure multiple email accounts, you must either configure this setting or specify the email account to use in the <> action. See <>. +`xpack.notification.email.recipient_allowlist`:: +(<>) +Specifies addresses to which emails are allowed to be sent. +Emails with recipients (`To:`, `Cc:`, or `Bcc:`) outside of these patterns will be rejected and an +error thrown. This setting defaults to `["*"]` which means all recipients are allowed. +Simple globbing is supported, such as `list-*@company.com` in the list of allowed recipients. + +NOTE: This setting can't be used at the same time as `xpack.notification.email.account.domain_allowlist` +and an error will be thrown if both are set at the same time. This setting can be used to specify domains +to allow by using a wildcard pattern such as `*@company.com`. + `xpack.notification.email.account`:: Specifies account information for sending notifications via email. You can specify the following email account attributes: @@ -129,6 +140,10 @@ Specifies domains to which emails are allowed to be sent. Emails with recipients `Bcc:`) outside of these domains will be rejected and an error thrown. This setting defaults to `["*"]` which means all domains are allowed. Simple globbing is supported, such as `*.company.com` in the list of allowed domains. + +NOTE: This setting can't be used at the same time as `xpack.notification.email.recipient_allowlist` +and an error will be thrown if both are set at the same time. + -- [[email-account-attributes]] diff --git a/docs/reference/settings/security-settings.asciidoc b/docs/reference/settings/security-settings.asciidoc index 7dd9d0574638c..0fc4d59e72350 100644 --- a/docs/reference/settings/security-settings.asciidoc +++ b/docs/reference/settings/security-settings.asciidoc @@ -1990,7 +1990,7 @@ idle for more than the specified timeout. The server can also set the `Keep-Alive` HTTP response header. The effective time-to-live value is the smaller value between this setting and the `Keep-Alive` -reponse header. Configure this setting to `-1` to let the server dictate the value. +response header. Configure this setting to `-1` to let the server dictate the value. If the header is not set by the server and the setting has value of `-1`, the time-to-live is infinite and connections never expire. // end::oidc-http-connection-pool-ttl-tag[] diff --git a/docs/reference/setup.asciidoc b/docs/reference/setup.asciidoc index b346fddc5e5a1..80828fdbfbb02 100644 --- a/docs/reference/setup.asciidoc +++ b/docs/reference/setup.asciidoc @@ -27,6 +27,8 @@ the only resource-intensive application on the host or container. For example, you might run {metricbeat} alongside {es} for cluster statistics, but a resource-heavy {ls} deployment should be on its own host. +include::run-elasticsearch-locally.asciidoc[] + include::setup/install.asciidoc[] include::setup/configuration.asciidoc[] @@ -81,8 +83,6 @@ include::modules/indices/search-settings.asciidoc[] include::settings/security-settings.asciidoc[] -include::modules/shard-ops.asciidoc[] - include::modules/indices/request_cache.asciidoc[] include::settings/snapshot-settings.asciidoc[] diff --git a/docs/reference/setup/bootstrap-checks-xes.asciidoc b/docs/reference/setup/bootstrap-checks-xes.asciidoc index bc187e668d0c0..926afe6463f46 100644 --- a/docs/reference/setup/bootstrap-checks-xes.asciidoc +++ b/docs/reference/setup/bootstrap-checks-xes.asciidoc @@ -6,6 +6,7 @@ In addition to the <>, there are checks that are specific to {xpack} features. [discrete] +[[bootstrap-checks-xpack-encrypt-sensitive-data]] === Encrypt sensitive data check //See EncryptSensitiveDAtaBootstrapCheck.java @@ -18,6 +19,7 @@ on each node in the cluster. For more information, see <>. [discrete] +[[bootstrap-checks-xpack-pki-realm]] === PKI realm check //See PkiRealmBootstrapCheckTests.java @@ -31,6 +33,7 @@ To pass this bootstrap check, if a PKI realm is enabled, you must configure TLS and enable client authentication on at least one network communication layer. [discrete] +[[bootstrap-checks-xpack-role-mappings]] === Role mappings check If you authenticate users with realms other than `native` or `file` realms, you @@ -65,6 +68,7 @@ To pass this bootstrap check, you must [discrete] +[[bootstrap-checks-xpack-token-ssl]] === Token SSL check //See TokenSSLBootstrapCheckTests.java diff --git a/docs/reference/setup/bootstrap-checks.asciidoc b/docs/reference/setup/bootstrap-checks.asciidoc index 20f93496934f7..7793e3e3c3359 100644 --- a/docs/reference/setup/bootstrap-checks.asciidoc +++ b/docs/reference/setup/bootstrap-checks.asciidoc @@ -60,6 +60,7 @@ in the <>. We strongly encourage you to do this if you are in this specific situation. This system property can be used to force execution of the bootstrap checks independent of the node configuration. +[[bootstrap-checks-heap-size]] === Heap size check By default, {es} automatically sizes JVM heap based on a node's @@ -71,6 +72,7 @@ size on startup. If the initial heap size is not equal to the maximum heap size, some JVM heap may not be locked after a resize. To avoid these issues, start the JVM with an initial heap size equal to the maximum heap size. +[[bootstrap-checks-file-descriptor]] === File descriptor check File descriptors are a Unix construct for tracking open "files". In Unix @@ -83,6 +85,7 @@ bootstrap check is enforced on OS X and Linux. To pass the file descriptor check, you might have to configure <>. +[[bootstrap-checks-memory-lock]] === Memory lock check When the JVM does a major garbage collection it touches every page of @@ -116,6 +119,7 @@ least 4096 threads. This can be done via `/etc/security/limits.conf` using the `nproc` setting (note that you might have to increase the limits for the `root` user too). +[[bootstrap-checks-max-file-size]] === Max file size check The segment files that are the components of individual shards and the translog @@ -144,6 +148,7 @@ address space. This can be done via adding ` - as unlimited` to `/etc/security/limits.conf`. This may require you to increase the limits for the `root` user too. +[[bootstrap-checks-max-map-count]] === Maximum map count check Continuing from the previous <>, to @@ -158,6 +163,7 @@ Alternatively, the maximum map count check is only needed if you are using indices. If you <> the use of `mmap` then this bootstrap check will not be enforced. +[[bootstrap-checks-client-jvm]] === Client JVM check There are two different JVMs provided by OpenJDK-derived JVMs: the @@ -171,6 +177,7 @@ JVM check, you must start Elasticsearch with the server VM. On modern systems and operating systems, the server VM is the default. +[[bootstrap-checks-serial-collector]] === Use serial collector check There are various garbage collectors for the OpenJDK-derived JVMs @@ -187,6 +194,7 @@ configuration that ships with Elasticsearch configures Elasticsearch to use the G1GC garbage collector with JDK14 and later versions. For earlier JDK versions, the configuration defaults to the CMS collector. +[[bootstrap-checks-syscall-filter]] === System call filter check Elasticsearch installs system call filters of various flavors depending on the operating system (e.g., seccomp on Linux). These system call @@ -198,6 +206,7 @@ installed. To pass the system call filter check you must fix any configuration errors on your system that prevented system call filters from installing (check your logs). +[[bootstrap-checks-onerror]] === OnError and OnOutOfMemoryError checks The JVM options `OnError` and `OnOutOfMemoryError` enable executing @@ -214,6 +223,7 @@ use the JVM flag `ExitOnOutOfMemoryError`. While this does not have the full capabilities of `OnError` nor `OnOutOfMemoryError`, arbitrary forking will not be supported with seccomp enabled. +[[bootstrap-checks-early-access]] === Early-access check The OpenJDK project provides early-access snapshots of upcoming releases. These @@ -221,12 +231,14 @@ releases are not suitable for production. The early-access check detects these early-access snapshots. To pass this check, you must start Elasticsearch on a release build of the JVM. +[[bootstrap-checks-all-permission]] === All permission check The all permission check ensures that the security policy used during bootstrap does not grant the `java.security.AllPermission` to Elasticsearch. Running with the all permission granted is equivalent to disabling the security manager. +[[bootstrap-checks-discovery-configuration]] === Discovery configuration check By default, when Elasticsearch first starts up it will try and discover other @@ -245,10 +257,9 @@ properties: - `discovery.seed_providers` - `cluster.initial_master_nodes` -Note that you should remove `cluster.initial_master_nodes` from the -configuration after the cluster has started for the first time. Do not use this -setting when restarting nodes or when adding new nodes to an existing cluster. -Instead, configure `discovery.seed_hosts` or `discovery.seed_providers`. If you -do not need any discovery configuration, for instance if running a single-node -cluster, set `discovery.seed_hosts: []` to disable discovery and satisfy this -bootstrap check. +Note that you must <> after the cluster has started for the +first time. Instead, configure `discovery.seed_hosts` or +`discovery.seed_providers`. If you do not need any discovery configuration, for +instance if running a single-node cluster, set `discovery.seed_hosts: []` to +disable discovery and satisfy this bootstrap check. diff --git a/docs/reference/setup/important-settings/discovery-settings.asciidoc b/docs/reference/setup/important-settings/discovery-settings.asciidoc index 180121b93206a..acd10a045c6c0 100644 --- a/docs/reference/setup/important-settings/discovery-settings.asciidoc +++ b/docs/reference/setup/important-settings/discovery-settings.asciidoc @@ -53,16 +53,18 @@ first election. In <>, with no discovery settings configured, this step is performed automatically by the nodes themselves. -Because auto-bootstrapping is <>, when starting a new cluster in production -mode, you must explicitly list the master-eligible nodes whose votes should be -counted in the very first election. You set this list using the -`cluster.initial_master_nodes` setting. +Because auto-bootstrapping is <>, +when starting a new cluster in production mode, you must explicitly list the +master-eligible nodes whose votes should be counted in the very first election. +You set this list using the `cluster.initial_master_nodes` setting on every +master-eligible node. Do not configure this setting on master-ineligible nodes. IMPORTANT: After the cluster forms successfully for the first time, remove the -`cluster.initial_master_nodes` setting from each node's configuration. Do not -use this setting when restarting a cluster or adding a new node to an existing -cluster. +`cluster.initial_master_nodes` setting from each node's configuration and never +set it again for this cluster. Do not configure this setting on nodes joining +an existing cluster. Do not configure this setting on nodes which are +restarting. Do not configure this setting when performing a full-cluster +restart. See <>. [source,yaml] -------------------------------------------------- diff --git a/docs/reference/setup/install.asciidoc b/docs/reference/setup/install.asciidoc index 89373d0ce8d44..277846cb050bd 100644 --- a/docs/reference/setup/install.asciidoc +++ b/docs/reference/setup/install.asciidoc @@ -5,8 +5,8 @@ [[hosted-elasticsearch-service]] === Hosted Elasticsearch Service -{ecloud} offers all of the features of {es}, {kib}, and Elastic’s {observability}, {ents}, and {elastic-sec} solutions as a hosted service -available on AWS, GCP, and Azure. +{ecloud} offers all of the features of {es}, {kib}, and Elastic’s {observability}, {ents}, and {elastic-sec} solutions as a hosted service +available on AWS, GCP, and Azure. To set up Elasticsearch in {ecloud}, sign up for a {ess-trial}[free {ecloud} trial]. @@ -17,7 +17,7 @@ To set up Elasticsearch in {ecloud}, sign up for a {ess-trial}[free {ecloud} tri If you want to install and manage {es} yourself, you can: * Run {es} using a <>. -* Run {es} in a <>. +* Run {es} in a <>. * Set up and manage {es}, {kib}, {agent}, and the rest of the Elastic Stack on Kubernetes with {eck-ref}[{eck}]. TIP: To try out Elasticsearch on your own machine, we recommend using Docker and running both Elasticsearch and Kibana. For more information, see <>. Please note that this setup is *not suitable for production use*. @@ -76,27 +76,42 @@ Docker container images may be downloaded from the Elastic Docker Registry. [[jvm-version]] === Java (JVM) Version -{es} is built using Java, and includes a bundled version of -https://openjdk.java.net[OpenJDK] from the JDK maintainers (GPLv2+CE) within -each distribution. The bundled JVM is the recommended JVM. - -To use your own version of Java, set the `ES_JAVA_HOME` environment variable. -If you must use a version of Java that is different from the bundled JVM, it is -best to use the latest release of a link:/support/matrix[supported] -https://www.oracle.com/technetwork/java/eol-135779.html[LTS version of Java]. -{es} is closely coupled to certain OpenJDK-specific features, so it may not -work correctly with other JVMs. {es} will refuse to start if a known-bad -version of Java is used. - -If you use a JVM other than the bundled one, you are responsible for reacting -to announcements related to its security issues and bug fixes, and must -yourself determine whether each update is necessary or not. In contrast, the -bundled JVM is treated as an integral part of {es}, which means that Elastic -takes responsibility for keeping it up to date. Security issues and bugs within -the bundled JVM are treated as if they were within {es} itself. - -The bundled JVM is located within the `jdk` subdirectory of the {es} home -directory. You may remove this directory if using your own JVM. +{es} is built using Java, and includes a bundled version of https://openjdk.java.net[OpenJDK] within each distribution. We strongly +recommend using the bundled JVM in all installations of {es}. + +The bundled JVM is treated the same as any other dependency of {es} in terms of support and maintenance. This means that Elastic takes +responsibility for keeping it up to date, and reacts to security issues and bug reports as needed to address vulnerabilities and other bugs +in {es}. Elastic's support of the bundled JVM is subject to Elastic's https://www.elastic.co/support_policy[support policy] and +https://www.elastic.co/support/eol[end-of-life schedule] and is independent of the support policy and end-of-life schedule offered by the +original supplier of the JVM. Elastic does not support using the bundled JVM for purposes other than running {es}. + +TIP: {es} uses only a subset of the features offered by the JVM. Bugs and security issues in the bundled JVM often relate to features that +{es} does not use. Such issues do not apply to {es}. Elastic analyzes reports of security vulnerabilities in all its dependencies, including +in the bundled JVM, and will issue an https://www.elastic.co/community/security[Elastic Security Advisory] if such an advisory is needed. + +If you decide to run {es} using a version of Java that is different from the bundled one, prefer to use the latest release of a +https://www.oracle.com/technetwork/java/eol-135779.html[LTS version of Java] which is link:/support/matrix[listed in the support matrix]. +Although such a configuration is supported, if you encounter a security issue or other bug in your chosen JVM then Elastic may not be able +to help unless the issue is also present in the bundled JVM. Instead, you must seek assistance directly from the supplier of your chosen +JVM. You must also take responsibility for reacting to security and bug announcements from the supplier of your chosen JVM. {es} may not +perform optimally if using a JVM other than the bundled one. {es} is closely coupled to certain OpenJDK-specific features, so it may not +work correctly with JVMs that are not OpenJDK. {es} will refuse to start if you attempt to use a known-bad JVM version. + +To use your own version of Java, set the `ES_JAVA_HOME` environment variable to the path to your own JVM installation. The bundled JVM is +located within the `jdk` subdirectory of the {es} home directory. You may remove this directory if using your own JVM. + +[discrete] +[[jdk-locale]] +=== JDK locale database + +The locale database used by {es}, used to map from various date formats to +the underlying date storage format, depends on the version of the JDK +that {es} is running on. On JDK version 23 and above, {es} will use the +_CLDR_ database. On JDK version 22 and below, {es} will use the _COMPAT_ +database. This may mean that the strings used for textual date formats, +and the output of custom week-date formats, may change when moving from +a previous JDK version to JDK 23 or above. For more information, see +<>. [discrete] [[jvm-agents]] diff --git a/docs/reference/setup/install/docker.asciidoc b/docs/reference/setup/install/docker.asciidoc index 370fc5c4ccf7e..f3576db0c786c 100644 --- a/docs/reference/setup/install/docker.asciidoc +++ b/docs/reference/setup/install/docker.asciidoc @@ -25,6 +25,21 @@ TIP: This setup doesn't run multiple {es} nodes or {kib} by default. To create a multi-node cluster with {kib}, use Docker Compose instead. See <>. +[[docker-wolfi-hardened-image]] +===== Hardened Docker images + +You can also use the hardened https://wolfi.dev/[Wolfi] image for additional security. +Using Wolfi images requires Docker version 20.10.10 or higher. + +To use the Wolfi image, append `-wolfi` to the image tag in the Docker command. + +For example: + +[source,sh,subs="attributes"] +---- +docker pull {docker-wolfi-image} +---- + ===== Start a single-node cluster . Install Docker. Visit https://docs.docker.com/get-docker/[Get Docker] to @@ -39,7 +54,7 @@ adjust memory usage in Docker Desktop by going to **Settings > Resources**. ---- docker network create elastic ---- - +// REVIEWED[DEC.10.24] . Pull the {es} Docker image. + -- @@ -52,10 +67,11 @@ endif::[] ---- docker pull {docker-image} ---- +// REVIEWED[DEC.10.24] -- . Optional: Install -https://docs.sigstore.dev/system_config/installation/[Cosign] for your +https://docs.sigstore.dev/cosign/system_config/installation/[Cosign] for your environment. Then use Cosign to verify the {es} image's signature. + [[docker-verify-signature]] @@ -64,6 +80,7 @@ environment. Then use Cosign to verify the {es} image's signature. wget https://artifacts.elastic.co/cosign.pub cosign verify --key cosign.pub {docker-image} ---- +// REVIEWED[DEC.10.24] + The `cosign` command prints the check results and the signature payload in JSON format: + @@ -75,6 +92,7 @@ The following checks were performed on each of these signatures: - Existence of the claims in the transparency log was verified offline - The signatures were verified against the specified public key ---- +// REVIEWED[DEC.10.24] . Start an {es} container. + @@ -82,10 +100,21 @@ The following checks were performed on each of these signatures: ---- docker run --name es01 --net elastic -p 9200:9200 -it -m 1GB {docker-image} ---- +// REVIEWED[DEC.10.24] + TIP: Use the `-m` flag to set a memory limit for the container. This removes the need to <>. + + +{ml-cap} features such as <> +require a larger container with more than 1GB of memory. +If you intend to use the {ml} capabilities, then start the container with this command: ++ +[source,sh,subs="attributes"] +---- +docker run --name es01 --net elastic -p 9200:9200 -it -m 6GB -e "xpack.ml.use_auto_machine_memory_percent=true" {docker-image} +---- +// REVIEWED[DEC.10.24] The command prints the `elastic` user password and an enrollment token for {kib}. . Copy the generated `elastic` password and enrollment token. These credentials @@ -97,6 +126,7 @@ credentials using the following commands. docker exec -it es01 /usr/share/elasticsearch/bin/elasticsearch-reset-password -u elastic docker exec -it es01 /usr/share/elasticsearch/bin/elasticsearch-create-enrollment-token -s kibana ---- +// REVIEWED[DEC.10.24] + We recommend storing the `elastic` password as an environment variable in your shell. Example: + @@ -104,6 +134,7 @@ We recommend storing the `elastic` password as an environment variable in your s ---- export ELASTIC_PASSWORD="your_password" ---- +// REVIEWED[DEC.10.24] . Copy the `http_ca.crt` SSL certificate from the container to your local machine. + @@ -111,6 +142,7 @@ export ELASTIC_PASSWORD="your_password" ---- docker cp es01:/usr/share/elasticsearch/config/certs/http_ca.crt . ---- +// REVIEWED[DEC.10.24] . Make a REST API call to {es} to ensure the {es} container is running. + @@ -119,6 +151,7 @@ docker cp es01:/usr/share/elasticsearch/config/certs/http_ca.crt . curl --cacert http_ca.crt -u elastic:$ELASTIC_PASSWORD https://localhost:9200 ---- // NOTCONSOLE +// REVIEWED[DEC.10.24] ===== Add more nodes @@ -128,6 +161,7 @@ curl --cacert http_ca.crt -u elastic:$ELASTIC_PASSWORD https://localhost:9200 ---- docker exec -it es01 /usr/share/elasticsearch/bin/elasticsearch-create-enrollment-token -s node ---- +// REVIEWED[DEC.10.24] + The enrollment token is valid for 30 minutes. @@ -137,6 +171,7 @@ The enrollment token is valid for 30 minutes. ---- docker run -e ENROLLMENT_TOKEN="" --name es02 --net elastic -it -m 1GB {docker-image} ---- +// REVIEWED[DEC.10.24] . Call the <> to verify the node was added to the cluster. + @@ -145,6 +180,7 @@ docker run -e ENROLLMENT_TOKEN="" --name es02 --net elastic -it -m 1GB {d curl --cacert http_ca.crt -u elastic:$ELASTIC_PASSWORD https://localhost:9200/_cat/nodes ---- // NOTCONSOLE +// REVIEWED[DEC.10.24] [[run-kibana-docker]] ===== Run {kib} @@ -161,6 +197,7 @@ endif::[] ---- docker pull {kib-docker-image} ---- +// REVIEWED[DEC.10.24] -- . Optional: Verify the {kib} image's signature. @@ -170,6 +207,7 @@ docker pull {kib-docker-image} wget https://artifacts.elastic.co/cosign.pub cosign verify --key cosign.pub {kib-docker-image} ---- +// REVIEWED[DEC.10.24] . Start a {kib} container. + @@ -177,6 +215,7 @@ cosign verify --key cosign.pub {kib-docker-image} ---- docker run --name kib01 --net elastic -p 5601:5601 {kib-docker-image} ---- +// REVIEWED[DEC.10.24] . When {kib} starts, it outputs a unique generated link to the terminal. To access {kib}, open this link in a web browser. @@ -189,6 +228,7 @@ To regenerate the token, run: ---- docker exec -it es01 /usr/share/elasticsearch/bin/elasticsearch-create-enrollment-token -s kibana ---- +// REVIEWED[DEC.10.24] . Log in to {kib} as the `elastic` user with the password that was generated when you started {es}. @@ -199,6 +239,7 @@ To regenerate the password, run: ---- docker exec -it es01 /usr/share/elasticsearch/bin/elasticsearch-reset-password -u elastic ---- +// REVIEWED[DEC.10.24] [[remove-containers-docker]] ===== Remove containers @@ -217,6 +258,7 @@ docker rm es02 # Remove the {kib} container docker rm kib01 ---- +// REVIEWED[DEC.10.24] ===== Next steps @@ -297,6 +339,7 @@ ES_PORT=127.0.0.1:9200 ---- docker-compose up -d ---- +// REVIEWED[DEC.10.24] . After the cluster has started, open http://localhost:5601 in a web browser to access {kib}. @@ -312,6 +355,7 @@ is preserved and loaded when you restart the cluster with `docker-compose up`. ---- docker-compose down ---- +// REVIEWED[DEC.10.24] To delete the network, containers, and volumes when you stop the cluster, specify the `-v` option: @@ -320,6 +364,7 @@ specify the `-v` option: ---- docker-compose down -v ---- +// REVIEWED[DEC.10.24] ===== Next steps @@ -368,6 +413,7 @@ The `vm.max_map_count` setting must be set within the xhyve virtual machine: -------------------------------------------- screen ~/Library/Containers/com.docker.docker/Data/vms/0/tty -------------------------------------------- +// REVIEWED[DEC.10.24] . Press enter and use `sysctl` to configure `vm.max_map_count`: + @@ -485,6 +531,7 @@ To check the Docker daemon defaults for ulimits, run: -------------------------------------------- docker run --rm {docker-image} /bin/bash -c 'ulimit -Hn && ulimit -Sn && ulimit -Hu && ulimit -Su' -------------------------------------------- +// REVIEWED[DEC.10.24] If needed, adjust them in the Daemon or override them per container. For example, when using `docker run`, set: @@ -493,6 +540,7 @@ For example, when using `docker run`, set: -------------------------------------------- --ulimit nofile=65535:65535 -------------------------------------------- +// REVIEWED[DEC.10.24] ===== Disable swapping @@ -509,6 +557,7 @@ When using `docker run`, you can specify: ---- -e "bootstrap.memory_lock=true" --ulimit memlock=-1:-1 ---- +// REVIEWED[DEC.10.24] ===== Randomize published ports @@ -536,6 +585,7 @@ environment variable. For example, to use 1GB, use the following command. ---- docker run -e ES_JAVA_OPTS="-Xms1g -Xmx1g" -e ENROLLMENT_TOKEN="" --name es01 -p 9200:9200 --net elastic -it {docker-image} ---- +// REVIEWED[DEC.10.24] The `ES_JAVA_OPTS` variable overrides all other JVM options. We do not recommend using `ES_JAVA_OPTS` in production. @@ -607,6 +657,7 @@ If you mount the password file to `/run/secrets/bootstrapPassword.txt`, specify: -------------------------------------------- -e ELASTIC_PASSWORD_FILE=/run/secrets/bootstrapPassword.txt -------------------------------------------- +// REVIEWED[DEC.10.24] You can override the default command for the image to pass {es} configuration parameters as command line options. For example: diff --git a/docs/reference/setup/install/docker/docker-compose.yml b/docs/reference/setup/install/docker/docker-compose.yml index 15d8c11e2f12f..db5b6e6c91b49 100644 --- a/docs/reference/setup/install/docker/docker-compose.yml +++ b/docs/reference/setup/install/docker/docker-compose.yml @@ -90,6 +90,7 @@ services: - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt - xpack.security.transport.ssl.verification_mode=certificate - xpack.license.self_generated.type=${LICENSE} + - xpack.ml.use_auto_machine_memory_percent=true mem_limit: ${MEM_LIMIT} ulimits: memlock: @@ -130,6 +131,7 @@ services: - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt - xpack.security.transport.ssl.verification_mode=certificate - xpack.license.self_generated.type=${LICENSE} + - xpack.ml.use_auto_machine_memory_percent=true mem_limit: ${MEM_LIMIT} ulimits: memlock: @@ -170,6 +172,7 @@ services: - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt - xpack.security.transport.ssl.verification_mode=certificate - xpack.license.self_generated.type=${LICENSE} + - xpack.ml.use_auto_machine_memory_percent=true mem_limit: ${MEM_LIMIT} ulimits: memlock: diff --git a/docs/reference/setup/stopping.asciidoc b/docs/reference/setup/stopping.asciidoc index 8c3a8d40fa1d2..f80812f026936 100644 --- a/docs/reference/setup/stopping.asciidoc +++ b/docs/reference/setup/stopping.asciidoc @@ -50,9 +50,14 @@ such a shutdown, it does not go through an orderly shutdown as described above. process will also return with a special status code indicating the nature of the error. [horizontal] +Killed by jvmkiller agent:: 158 +User or kernel SIGTERM:: 143 +Slain by kernel oom-killer:: 137 +Segmentation fault:: 134 JVM internal error:: 128 Out of memory error:: 127 Stack overflow error:: 126 Unknown virtual machine error:: 125 Serious I/O error:: 124 +Bootstrap check failure:: 78 Unknown fatal error:: 1 diff --git a/docs/reference/shutdown/apis/shutdown-api.asciidoc b/docs/reference/shutdown/apis/shutdown-api.asciidoc index 24cbca720d160..b950cd3d19c56 100644 --- a/docs/reference/shutdown/apis/shutdown-api.asciidoc +++ b/docs/reference/shutdown/apis/shutdown-api.asciidoc @@ -4,6 +4,12 @@ NOTE: {cloud-only} +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-shutdown[Node lifecycle APIs]. +-- + You use the shutdown APIs to prepare nodes for temporary or permanent shutdown, monitor the shutdown status, and enable a previously shut-down node to resume normal operations. [discrete] diff --git a/docs/reference/shutdown/apis/shutdown-delete.asciidoc b/docs/reference/shutdown/apis/shutdown-delete.asciidoc index 4d7f30c3a1e48..225d88c63a016 100644 --- a/docs/reference/shutdown/apis/shutdown-delete.asciidoc +++ b/docs/reference/shutdown/apis/shutdown-delete.asciidoc @@ -3,6 +3,12 @@ NOTE: {cloud-only} +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-shutdown[Node lifecycle APIs]. +-- + Cancels shutdown preparations or clears a shutdown request so a node can resume normal operations. diff --git a/docs/reference/shutdown/apis/shutdown-get.asciidoc b/docs/reference/shutdown/apis/shutdown-get.asciidoc index 5feac28353ab5..b0097eb0caf7f 100644 --- a/docs/reference/shutdown/apis/shutdown-get.asciidoc +++ b/docs/reference/shutdown/apis/shutdown-get.asciidoc @@ -3,6 +3,12 @@ NOTE: {cloud-only} +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-shutdown[Node lifecycle APIs]. +-- + Retrieves the status of a node that's being prepared for shutdown. [[get-shutdown-api-request]] @@ -72,6 +78,7 @@ including the status of shard migration, task migration, and plugin cleanup: "nodes": [ { "node_id": "USpTGYaBSIKbgSUJR2Z9lg", + "node_ephemeral_id": null, "type": "RESTART", "reason": "Demonstrating how the node shutdown API works", "shutdown_startedmillis": 1624406108685, diff --git a/docs/reference/shutdown/apis/shutdown-put.asciidoc b/docs/reference/shutdown/apis/shutdown-put.asciidoc index 344dd8fa36717..5eef4763a9c3a 100644 --- a/docs/reference/shutdown/apis/shutdown-put.asciidoc +++ b/docs/reference/shutdown/apis/shutdown-put.asciidoc @@ -3,6 +3,12 @@ NOTE: {cloud-only} +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-shutdown[Node lifecycle APIs]. +-- + Prepares a node to be shut down. [[put-shutdown-api-request]] diff --git a/docs/reference/slm/apis/slm-api.asciidoc b/docs/reference/slm/apis/slm-api.asciidoc index d061ff6b0aaf7..ee624a70b00f5 100644 --- a/docs/reference/slm/apis/slm-api.asciidoc +++ b/docs/reference/slm/apis/slm-api.asciidoc @@ -2,6 +2,12 @@ [[snapshot-lifecycle-management-api]] == {slm-cap} APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-slm[{slm-cap} APIs]. +-- + You use the following APIs to set up policies to automatically take snapshots and control how long they are retained. diff --git a/docs/reference/slm/apis/slm-delete.asciidoc b/docs/reference/slm/apis/slm-delete.asciidoc index 650ee68e24fb7..9c63d7326421f 100644 --- a/docs/reference/slm/apis/slm-delete.asciidoc +++ b/docs/reference/slm/apis/slm-delete.asciidoc @@ -5,6 +5,12 @@ Delete policy ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-slm[{slm-cap} APIs]. +-- + Deletes an existing snapshot lifecycle policy. [[slm-api-delete-lifecycle-request]] diff --git a/docs/reference/slm/apis/slm-execute-retention.asciidoc b/docs/reference/slm/apis/slm-execute-retention.asciidoc index 75bcdb3143668..ad8d7b15a43d7 100644 --- a/docs/reference/slm/apis/slm-execute-retention.asciidoc +++ b/docs/reference/slm/apis/slm-execute-retention.asciidoc @@ -5,6 +5,12 @@ Execute snapshot retention policy ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-slm[{slm-cap} APIs]. +-- + Deletes any snapshots that are expired according to the policy's retention rules. [[slm-api-execute-retention-request]] diff --git a/docs/reference/slm/apis/slm-execute.asciidoc b/docs/reference/slm/apis/slm-execute.asciidoc index f3977d6aed2fd..9ea35602de4e2 100644 --- a/docs/reference/slm/apis/slm-execute.asciidoc +++ b/docs/reference/slm/apis/slm-execute.asciidoc @@ -5,6 +5,12 @@ Execute snapshot lifecycle policy ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-slm[{slm-cap} APIs]. +-- + Immediately creates a snapshot according to the lifecycle policy, without waiting for the scheduled time. diff --git a/docs/reference/slm/apis/slm-get-status.asciidoc b/docs/reference/slm/apis/slm-get-status.asciidoc index d4afbaddb1beb..181927ac35a1c 100644 --- a/docs/reference/slm/apis/slm-get-status.asciidoc +++ b/docs/reference/slm/apis/slm-get-status.asciidoc @@ -7,6 +7,12 @@ Get {slm} status ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-slm[{slm-cap} APIs]. +-- + Retrieves the status of {slm} ({slm-init}). [[slm-api-get-status-request]] diff --git a/docs/reference/slm/apis/slm-get.asciidoc b/docs/reference/slm/apis/slm-get.asciidoc index f7c847d06dc4b..723d8b374b914 100644 --- a/docs/reference/slm/apis/slm-get.asciidoc +++ b/docs/reference/slm/apis/slm-get.asciidoc @@ -5,6 +5,12 @@ Get policy ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-slm[{slm-cap} APIs]. +-- + Retrieves one or more snapshot lifecycle policy definitions and information about the latest snapshot attempts. diff --git a/docs/reference/slm/apis/slm-put.asciidoc b/docs/reference/slm/apis/slm-put.asciidoc index 51ad571ee12e7..a6ee29bfd1bc9 100644 --- a/docs/reference/slm/apis/slm-put.asciidoc +++ b/docs/reference/slm/apis/slm-put.asciidoc @@ -4,8 +4,13 @@ Create or update policy ++++ -Creates or updates a snapshot lifecycle policy. +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-slm[{slm-cap} APIs]. +-- +Creates or updates a snapshot lifecycle policy. [[slm-api-put-request]] ==== {api-request-title} diff --git a/docs/reference/slm/apis/slm-start.asciidoc b/docs/reference/slm/apis/slm-start.asciidoc index 9d9b8108cb57b..87c19ec600b61 100644 --- a/docs/reference/slm/apis/slm-start.asciidoc +++ b/docs/reference/slm/apis/slm-start.asciidoc @@ -1,12 +1,17 @@ [role="xpack"] [[slm-api-start]] === Start {slm} API - [subs="attributes"] ++++ Start {slm} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-slm[{slm-cap} APIs]. +-- + Turns on {slm} ({slm-init}). [[slm-api-start-request]] diff --git a/docs/reference/slm/apis/slm-stats.asciidoc b/docs/reference/slm/apis/slm-stats.asciidoc index 340631e9c5601..d6b9cd960e45f 100644 --- a/docs/reference/slm/apis/slm-stats.asciidoc +++ b/docs/reference/slm/apis/slm-stats.asciidoc @@ -5,6 +5,12 @@ Get snapshot lifecycle stats ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-slm[{slm-cap} APIs]. +-- + Returns global and policy-level statistics about actions taken by {slm}. [[slm-api-stats-request]] diff --git a/docs/reference/slm/apis/slm-stop.asciidoc b/docs/reference/slm/apis/slm-stop.asciidoc index 253abec7b4d11..2bfe9646bcffd 100644 --- a/docs/reference/slm/apis/slm-stop.asciidoc +++ b/docs/reference/slm/apis/slm-stop.asciidoc @@ -1,12 +1,17 @@ [role="xpack"] [[slm-api-stop]] === Stop {slm} API - [subs="attributes"] ++++ Stop {slm} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-slm[{slm-cap} APIs]. +-- + Turn off {slm} ({slm-init}). [[slm-api-stop-request]] diff --git a/docs/reference/snapshot-restore/apis/clean-up-repo-api.asciidoc b/docs/reference/snapshot-restore/apis/clean-up-repo-api.asciidoc index 249e192c0c587..dbb7541693492 100644 --- a/docs/reference/snapshot-restore/apis/clean-up-repo-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/clean-up-repo-api.asciidoc @@ -4,6 +4,12 @@ Clean up snapshot repository ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-snapshot[Snapshot and restore APIs]. +-- + Triggers the review of a snapshot repository's contents and deletes any stale data that is not referenced by existing snapshots. See <>. diff --git a/docs/reference/snapshot-restore/apis/clone-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/clone-snapshot-api.asciidoc index 590bc7e7410f5..cf432d807d9c5 100644 --- a/docs/reference/snapshot-restore/apis/clone-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/clone-snapshot-api.asciidoc @@ -4,6 +4,12 @@ Clone snapshot ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-snapshot[Snapshot and restore APIs]. +-- + Clones part or all of a snapshot into a new snapshot. [source,console] diff --git a/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc index baa28bb7b0a53..a6c39abbda734 100644 --- a/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc @@ -4,6 +4,12 @@ Create snapshot ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-snapshot[Snapshot and restore APIs]. +-- + <> of a cluster or specified data streams and indices. diff --git a/docs/reference/snapshot-restore/apis/delete-repo-api.asciidoc b/docs/reference/snapshot-restore/apis/delete-repo-api.asciidoc index 4301fea642523..e0df427da745c 100644 --- a/docs/reference/snapshot-restore/apis/delete-repo-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/delete-repo-api.asciidoc @@ -4,6 +4,12 @@ Delete snapshot repository ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-snapshot[Snapshot and restore APIs]. +-- + Unregisters one or more <>. When a repository is unregistered, {es} only removes the reference to the diff --git a/docs/reference/snapshot-restore/apis/delete-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/delete-snapshot-api.asciidoc index 8824977d660e4..74db60c1970c0 100644 --- a/docs/reference/snapshot-restore/apis/delete-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/delete-snapshot-api.asciidoc @@ -4,6 +4,12 @@ Delete snapshot ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-snapshot[Snapshot and restore APIs]. +-- + Deletes a <>. //// diff --git a/docs/reference/snapshot-restore/apis/get-repo-api.asciidoc b/docs/reference/snapshot-restore/apis/get-repo-api.asciidoc index cf1b9813c519e..5f6b6485c5ee4 100644 --- a/docs/reference/snapshot-restore/apis/get-repo-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/get-repo-api.asciidoc @@ -4,6 +4,12 @@ Get snapshot repository ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-snapshot[Snapshot and restore APIs]. +-- + Gets information about one or more registered <>. diff --git a/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc index 622e1ade024b0..f9eb6a27df039 100644 --- a/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc @@ -4,6 +4,12 @@ Get snapshot ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-snapshot[Snapshot and restore APIs]. +-- + Retrieves information about one or more snapshots. //// diff --git a/docs/reference/snapshot-restore/apis/get-snapshot-status-api.asciidoc b/docs/reference/snapshot-restore/apis/get-snapshot-status-api.asciidoc index e677408da3f25..dbbf547528f46 100644 --- a/docs/reference/snapshot-restore/apis/get-snapshot-status-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/get-snapshot-status-api.asciidoc @@ -4,6 +4,12 @@ Get snapshot status ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-snapshot[Snapshot and restore APIs]. +-- + Retrieves a detailed description of the current state for each shard participating in the snapshot. Note that this API should only be used to obtain detailed shard-level information for ongoing snapshots. If this detail is not needed, or you want to obtain information about one or more existing snapshots, use the <>. //// diff --git a/docs/reference/snapshot-restore/apis/put-repo-api.asciidoc b/docs/reference/snapshot-restore/apis/put-repo-api.asciidoc index 0d3b5586da869..55b61cc321ed3 100644 --- a/docs/reference/snapshot-restore/apis/put-repo-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/put-repo-api.asciidoc @@ -4,6 +4,12 @@ Create or update snapshot repository ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-snapshot[Snapshot and restore APIs]. +-- + Registers or updates a <>. [source,console] diff --git a/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc b/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc index f18ef1ee6e826..ca46ba1fb2b57 100644 --- a/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc @@ -5,6 +5,12 @@ Repository analysis ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-snapshot[Snapshot and restore APIs]. +-- + Analyzes a repository, reporting its performance characteristics and any incorrect behaviour found. diff --git a/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc index 9fe06d73f1a63..2f2c7fcd8ebd2 100644 --- a/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc @@ -4,6 +4,12 @@ Restore snapshot ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-snapshot[Snapshot and restore APIs]. +-- + Restores a <> of a cluster or specified data streams and indices. //// diff --git a/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc b/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc index b8bb6a2cd7d13..715687f02edea 100644 --- a/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc +++ b/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc @@ -1,6 +1,12 @@ [[snapshot-restore-apis]] == Snapshot and restore APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-snapshot[Snapshot and restore APIs]. +-- + You can use the following APIs to set up snapshot repositories, manage snapshot backups, and restore snapshots to a running cluster. diff --git a/docs/reference/snapshot-restore/apis/verify-repo-api.asciidoc b/docs/reference/snapshot-restore/apis/verify-repo-api.asciidoc index dd845663be8d7..333a12ff49ac0 100644 --- a/docs/reference/snapshot-restore/apis/verify-repo-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/verify-repo-api.asciidoc @@ -4,6 +4,12 @@ Verify snapshot repository ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-snapshot[Snapshot and restore APIs]. +-- + Checks for common misconfigurations in a snapshot repository. See <>. diff --git a/docs/reference/snapshot-restore/apis/verify-repo-integrity-api.asciidoc b/docs/reference/snapshot-restore/apis/verify-repo-integrity-api.asciidoc index 99ae126b401f5..22a4051a546cd 100644 --- a/docs/reference/snapshot-restore/apis/verify-repo-integrity-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/verify-repo-integrity-api.asciidoc @@ -5,6 +5,12 @@ Verify repository integrity ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-snapshot[Snapshot and restore APIs]. +-- + Verifies the integrity of the contents of a snapshot repository. //// diff --git a/docs/reference/snapshot-restore/register-repository.asciidoc b/docs/reference/snapshot-restore/register-repository.asciidoc index 2147ad3c684f3..6c1319c2c71b1 100644 --- a/docs/reference/snapshot-restore/register-repository.asciidoc +++ b/docs/reference/snapshot-restore/register-repository.asciidoc @@ -248,10 +248,11 @@ that you have an archive copy of its contents that you can use to recreate the repository in its current state at a later date. You must ensure that {es} does not write to the repository while you are taking -the backup of its contents. You can do this by unregistering it, or registering -it with `readonly: true`, on all your clusters. If {es} writes any data to the -repository during the backup then the contents of the backup may not be -consistent and it may not be possible to recover any data from it in future. +the backup of its contents. If {es} writes any data to the repository during +the backup then the contents of the backup may not be consistent and it may not +be possible to recover any data from it in future. Prevent writes to the +repository by unregistering the repository from the cluster which has write +access to it. Alternatively, if your repository supports it, you may take an atomic snapshot of the underlying filesystem and then take a backup of this filesystem diff --git a/docs/reference/snapshot-restore/repository-azure.asciidoc b/docs/reference/snapshot-restore/repository-azure.asciidoc index c361414052e14..eadda1d22fe22 100644 --- a/docs/reference/snapshot-restore/repository-azure.asciidoc +++ b/docs/reference/snapshot-restore/repository-azure.asciidoc @@ -181,7 +181,7 @@ is running. When running {es} in https://azure.microsoft.com/en-gb/products/kubernetes-service[Azure Kubernetes -Service], for instance using {eck-ref}[{eck}], you should use +Service], for instance using {eck-ref}/k8s-snapshots.html#k8s-azure-workload-identity[{eck}], you should use https://azure.github.io/azure-workload-identity/docs/introduction.html[Azure Workload Identity] to provide credentials to {es}. To use Azure Workload Identity, mount the `azure-identity-token` volume as a subdirectory of the diff --git a/docs/reference/snapshot-restore/repository-s3.asciidoc b/docs/reference/snapshot-restore/repository-s3.asciidoc index 3a9c12caebad9..f53708acc40ba 100644 --- a/docs/reference/snapshot-restore/repository-s3.asciidoc +++ b/docs/reference/snapshot-restore/repository-s3.asciidoc @@ -6,6 +6,9 @@ You can use AWS S3 as a repository for {ref}/snapshot-restore.html[Snapshot/Rest *If you are looking for a hosted solution of Elasticsearch on AWS, please visit https://www.elastic.co/cloud/.* +See https://www.youtube.com/watch?v=ACqfyzWf-xs[this video] +for a walkthrough of connecting an AWS S3 repository. + [[repository-s3-usage]] ==== Getting started @@ -35,7 +38,8 @@ PUT _snapshot/my_s3_repository The client that you use to connect to S3 has a number of settings available. The settings have the form `s3.client.CLIENT_NAME.SETTING_NAME`. By default, `s3` repositories use a client named `default`, but this can be modified using -the <> `client`. For example: +the <> `client`. For example, to +use a client named `my-alternate-client`, register the repository as follows: [source,console] ---- @@ -66,10 +70,19 @@ bin/elasticsearch-keystore add s3.client.default.secret_key bin/elasticsearch-keystore add s3.client.default.session_token ---- -If instead you want to use the instance role or container role to access S3 -then you should leave these settings unset. You can switch from using specific -credentials back to the default of using the instance role or container role by -removing these settings from the keystore as follows: +If you do not configure these settings then {es} will attempt to automatically +obtain credentials from the environment in which it is running: + +* Nodes running on an instance in AWS EC2 will attempt to use the EC2 Instance + Metadata Service (IMDS) to obtain instance role credentials. {es} supports + both IMDS version 1 and IMDS version 2. + +* Nodes running in a container in AWS ECS and AWS EKS will attempt to obtain + container role credentials similarly. + +You can switch from using specific credentials back to the default of using the +instance role or container role by removing these settings from the keystore as +follows: [source,sh] ---- @@ -79,20 +92,14 @@ bin/elasticsearch-keystore remove s3.client.default.secret_key bin/elasticsearch-keystore remove s3.client.default.session_token ---- -*All* client secure settings of this repository type are -{ref}/secure-settings.html#reloadable-secure-settings[reloadable]. -You can define these settings before the node is started, -or call the <> -after the settings are defined to apply them to a running node. - -After you reload the settings, the internal `s3` clients, used to transfer the snapshot -contents, will utilize the latest settings from the keystore. Any existing `s3` -repositories, as well as any newly created ones, will pick up the new values -stored in the keystore. - -NOTE: In-progress snapshot/restore tasks will not be preempted by a *reload* of -the client's secure settings. The task will complete using the client as it was -built when the operation started. +Define the relevant secure settings in each node's keystore before starting the +node. The secure settings described here are all +{ref}/secure-settings.html#reloadable-secure-settings[reloadable] so you may +update the keystore contents on each node while the node is running and then +call the <> to apply the updated settings to the nodes in the cluster. After this API +completes, {es} will use the updated setting values for all future snapshot +operations, but ongoing operations may continue to use older setting values. The following list contains the available client settings. Those that must be stored in the keystore are marked as "secure" and are *reloadable*; the other @@ -261,9 +268,11 @@ multiple deployments may share the same bucket. `chunk_size`:: - (<>) Big files can be broken down into chunks during snapshotting if needed. - Specify the chunk size as a value and unit, for example: - `1TB`, `1GB`, `10MB`. Defaults to the maximum size of a blob in the S3 which is `5TB`. + (<>) The maximum size of object that {es} will write to the repository + when creating a snapshot. Files which are larger than `chunk_size` will be chunked into several + smaller objects. {es} may also split a file across multiple objects to satisfy other constraints + such as the `max_multipart_parts` limit. Defaults to `5TB` which is the + https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html[maximum size of an object in AWS S3]. `compress`:: @@ -292,6 +301,14 @@ include::repository-shared-settings.asciidoc[] size allowed by S3. Defaults to `100mb` or `5%` of JVM heap, whichever is smaller. +`max_multipart_parts` :: + + (integer) The maximum number of parts that {es} will write during a multipart upload of a single + object. Files which are larger than `buffer_size × max_multipart_parts` will be chunked into + several smaller objects. {es} may also split a file across multiple objects to satisfy other + constraints such as the `chunk_size` limit. Defaults to `10000` which is the + https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html[maximum number of parts in a multipart upload in AWS S3]. + `canned_acl`:: The S3 repository supports all @@ -311,20 +328,14 @@ include::repository-shared-settings.asciidoc[] `delete_objects_max_size`:: - (<>) Sets the maxmimum batch size, betewen 1 and 1000, used - for `DeleteObjects` requests. Defaults to 1000 which is the maximum number - supported by the - https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html[AWS - DeleteObjects API]. + (integer) Sets the maxmimum batch size, betewen 1 and 1000, used for `DeleteObjects` requests. Defaults to 1000 which is the maximum + number supported by the https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html[AWS DeleteObjects API]. `max_multipart_upload_cleanup_size`:: - (<>) Sets the maximum number of possibly-dangling multipart - uploads to clean up in each batch of snapshot deletions. Defaults to `1000` - which is the maximum number supported by the - https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html[AWS - ListMultipartUploads API]. If set to `0`, {es} will not attempt to clean up - dangling multipart uploads. + (integer) Sets the maximum number of possibly-dangling multipart uploads to clean up in each batch of snapshot deletions. Defaults to + `1000` which is the maximum number supported by the https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html[AWS + ListMultipartUploads API]. If set to `0`, {es} will not attempt to clean up dangling multipart uploads. NOTE: The option of defining client settings in the repository settings as documented below is considered deprecated, and will be removed in a future diff --git a/docs/reference/snapshot-restore/repository-source-only.asciidoc b/docs/reference/snapshot-restore/repository-source-only.asciidoc index 07ddedd197931..3c11d6ca6e59c 100644 --- a/docs/reference/snapshot-restore/repository-source-only.asciidoc +++ b/docs/reference/snapshot-restore/repository-source-only.asciidoc @@ -18,7 +18,7 @@ stream or index. ================================================== Source-only snapshots are only supported if the `_source` field is enabled and no source-filtering is applied. -When you restore a source-only snapshot: +As a result, indices adopting synthetic source cannot be restored. When you restore a source-only snapshot: * The restored index is read-only and can only serve `match_all` search or scroll requests to enable reindexing. @@ -27,6 +27,9 @@ When you restore a source-only snapshot: * The mapping of the restored index is empty, but the original mapping is available from the types top level `meta` element. + * Information such as document count, deleted document count and store size are not available for such indices + since these indices do not contain the relevant data structures to retrieve this information from. Therefore, + this information is not shown for such indices in APIs such as the <>. ================================================== Before registering a source-only repository, use {kib} or the diff --git a/docs/reference/sql/apis/clear-sql-cursor-api.asciidoc b/docs/reference/sql/apis/clear-sql-cursor-api.asciidoc index 48663ca0d75aa..a16d31c4b12d7 100644 --- a/docs/reference/sql/apis/clear-sql-cursor-api.asciidoc +++ b/docs/reference/sql/apis/clear-sql-cursor-api.asciidoc @@ -5,6 +5,12 @@ Clear SQL cursor ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-sql[SQL APIs]. +-- + Clears an <>. //// diff --git a/docs/reference/sql/apis/delete-async-sql-search-api.asciidoc b/docs/reference/sql/apis/delete-async-sql-search-api.asciidoc index 1737a39401dba..e8e4ff68cef07 100644 --- a/docs/reference/sql/apis/delete-async-sql-search-api.asciidoc +++ b/docs/reference/sql/apis/delete-async-sql-search-api.asciidoc @@ -5,6 +5,12 @@ Delete async SQL search ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-sql[SQL APIs]. +-- + Deletes an <> or a <>. If the search is still running, the API cancels it. diff --git a/docs/reference/sql/apis/get-async-sql-search-api.asciidoc b/docs/reference/sql/apis/get-async-sql-search-api.asciidoc index 8ae575176dd3a..10ca4c648ebaa 100644 --- a/docs/reference/sql/apis/get-async-sql-search-api.asciidoc +++ b/docs/reference/sql/apis/get-async-sql-search-api.asciidoc @@ -5,6 +5,12 @@ Get async SQL search ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-sql[SQL APIs]. +-- + Returns results for an <> or a <>. diff --git a/docs/reference/sql/apis/get-async-sql-search-status-api.asciidoc b/docs/reference/sql/apis/get-async-sql-search-status-api.asciidoc index 61505bab7c454..7a0d68cd120cc 100644 --- a/docs/reference/sql/apis/get-async-sql-search-status-api.asciidoc +++ b/docs/reference/sql/apis/get-async-sql-search-status-api.asciidoc @@ -5,6 +5,12 @@ Get async SQL search status ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-sql[SQL APIs]. +-- + Returns the current status of an <> or a <>. diff --git a/docs/reference/sql/apis/sql-apis.asciidoc b/docs/reference/sql/apis/sql-apis.asciidoc index 08300522c3288..60f3c85c55d6c 100644 --- a/docs/reference/sql/apis/sql-apis.asciidoc +++ b/docs/reference/sql/apis/sql-apis.asciidoc @@ -2,6 +2,12 @@ [[sql-apis]] == SQL APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-sql[SQL APIs]. +-- + {es}'s SQL APIs let you run SQL queries on {es} indices and data streams. For an overview of {es}'s SQL features and related tutorials, see <>. diff --git a/docs/reference/sql/apis/sql-search-api.asciidoc b/docs/reference/sql/apis/sql-search-api.asciidoc index 118d7975aefd9..b1d002d343fb8 100644 --- a/docs/reference/sql/apis/sql-search-api.asciidoc +++ b/docs/reference/sql/apis/sql-search-api.asciidoc @@ -5,6 +5,12 @@ SQL search ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-sql[SQL APIs]. +-- + Returns results for an <>. [source,console] diff --git a/docs/reference/sql/apis/sql-translate-api.asciidoc b/docs/reference/sql/apis/sql-translate-api.asciidoc index 15e52d118800e..9183423eefd0f 100644 --- a/docs/reference/sql/apis/sql-translate-api.asciidoc +++ b/docs/reference/sql/apis/sql-translate-api.asciidoc @@ -5,6 +5,12 @@ SQL translate ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-sql[SQL APIs]. +-- + Translates an <> into a <> request containing <>. See <>. diff --git a/docs/reference/synonyms/apis/delete-synonym-rule.asciidoc b/docs/reference/synonyms/apis/delete-synonym-rule.asciidoc index 74cbab8c0b4a2..11f0708bafcda 100644 --- a/docs/reference/synonyms/apis/delete-synonym-rule.asciidoc +++ b/docs/reference/synonyms/apis/delete-synonym-rule.asciidoc @@ -1,10 +1,15 @@ [[delete-synonym-rule]] === Delete synonym rule - ++++ Delete synonym rule ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-synonyms[Synonyms APIs]. +-- + Deletes an individual synonym rule from a synonyms set. [[delete-synonym-rule-request]] diff --git a/docs/reference/synonyms/apis/delete-synonyms-set.asciidoc b/docs/reference/synonyms/apis/delete-synonyms-set.asciidoc index 9ba33ff3a5c75..62162e5c45675 100644 --- a/docs/reference/synonyms/apis/delete-synonyms-set.asciidoc +++ b/docs/reference/synonyms/apis/delete-synonyms-set.asciidoc @@ -1,10 +1,15 @@ [[delete-synonyms-set]] === Delete synonyms set - ++++ Delete synonyms set ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-synonyms[Synonyms APIs]. +-- + Deletes a synonyms set. [[delete-synonyms-set-request]] diff --git a/docs/reference/synonyms/apis/get-synonym-rule.asciidoc b/docs/reference/synonyms/apis/get-synonym-rule.asciidoc index c6c35e0efecca..3f0ee3f173245 100644 --- a/docs/reference/synonyms/apis/get-synonym-rule.asciidoc +++ b/docs/reference/synonyms/apis/get-synonym-rule.asciidoc @@ -1,10 +1,15 @@ [[get-synonym-rule]] === Get synonym rule - ++++ Get synonym rule ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-synonyms[Synonyms APIs]. +-- + Retrieves a synonym rule from a synonyms set. [[get-synonym-rule-request]] diff --git a/docs/reference/synonyms/apis/get-synonyms-set.asciidoc b/docs/reference/synonyms/apis/get-synonyms-set.asciidoc index 70bb5fb69526d..1bb31081712e5 100644 --- a/docs/reference/synonyms/apis/get-synonyms-set.asciidoc +++ b/docs/reference/synonyms/apis/get-synonyms-set.asciidoc @@ -5,6 +5,12 @@ Get synonyms set ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-synonyms[Synonyms APIs]. +-- + Retrieves a synonyms set. [[get-synonyms-set-request]] diff --git a/docs/reference/synonyms/apis/list-synonyms-sets.asciidoc b/docs/reference/synonyms/apis/list-synonyms-sets.asciidoc index 705a24c809e99..33ef220036b7a 100644 --- a/docs/reference/synonyms/apis/list-synonyms-sets.asciidoc +++ b/docs/reference/synonyms/apis/list-synonyms-sets.asciidoc @@ -5,6 +5,12 @@ List synonyms sets ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-synonyms[Synonyms APIs]. +-- + Retrieves a summary of all defined synonyms sets. This API allows to retrieve the total number of synonyms sets defined. diff --git a/docs/reference/synonyms/apis/put-synonym-rule.asciidoc b/docs/reference/synonyms/apis/put-synonym-rule.asciidoc index de2865632d55e..5bb561f0f923f 100644 --- a/docs/reference/synonyms/apis/put-synonym-rule.asciidoc +++ b/docs/reference/synonyms/apis/put-synonym-rule.asciidoc @@ -5,6 +5,12 @@ Create or update synonym rule ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-synonyms[Synonyms APIs]. +-- + Creates or updates a synonym rule for a synonym set. [[put-synonym-rule-request]] diff --git a/docs/reference/synonyms/apis/put-synonyms-set.asciidoc b/docs/reference/synonyms/apis/put-synonyms-set.asciidoc index ca75885921456..3af85638d022d 100644 --- a/docs/reference/synonyms/apis/put-synonyms-set.asciidoc +++ b/docs/reference/synonyms/apis/put-synonyms-set.asciidoc @@ -5,6 +5,12 @@ Create or update synonyms set ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-synonyms[Synonyms APIs]. +-- + Creates or updates a synonyms set. NOTE: Synonyms sets are limited to a maximum of 10,000 synonym rules per set. diff --git a/docs/reference/synonyms/apis/synonyms-apis.asciidoc b/docs/reference/synonyms/apis/synonyms-apis.asciidoc index dbbc26c36d3df..95fc0aae8c14d 100644 --- a/docs/reference/synonyms/apis/synonyms-apis.asciidoc +++ b/docs/reference/synonyms/apis/synonyms-apis.asciidoc @@ -7,6 +7,12 @@ --- +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-synonyms[Synonyms APIs]. +-- + The synonyms management API provides a convenient way to define and manage synonyms in an internal system index. Related synonyms can be grouped in a "synonyms set". Create as many synonym sets as you need. diff --git a/docs/reference/tab-widgets/api-call.asciidoc b/docs/reference/tab-widgets/api-call.asciidoc index bb6b89374075d..5e70d73684436 100644 --- a/docs/reference/tab-widgets/api-call.asciidoc +++ b/docs/reference/tab-widgets/api-call.asciidoc @@ -1,5 +1,5 @@ // tag::cloud[] -**Use {kib}** +**Option 1: Use {kib}** //tag::kibana-api-ex[] . Open {kib}'s main menu ("*☰*" near Elastic logo) and go to **Dev Tools > Console**. @@ -16,9 +16,9 @@ GET / //end::kibana-api-ex[] -**Use curl** +**Option 2: Use `curl`** -To communicate with {es} using curl or another client, you need your cluster's +To communicate with {es} using `curl` or another client, you need your cluster's endpoint. . Open {kib}'s main menu and click **Manage this deployment**. @@ -26,7 +26,7 @@ endpoint. . From your deployment menu, go to the **Elasticsearch** page. Click **Copy endpoint**. -. To submit an example API request, run the following curl command in a new +. To submit an example API request, run the following `curl` command in a new terminal session. Replace `` with the password for the `elastic` user. Replace `` with your endpoint. + diff --git a/docs/reference/tab-widgets/cpu-usage.asciidoc b/docs/reference/tab-widgets/cpu-usage.asciidoc index 575cf459ee5be..c6272228965eb 100644 --- a/docs/reference/tab-widgets/cpu-usage.asciidoc +++ b/docs/reference/tab-widgets/cpu-usage.asciidoc @@ -1,30 +1,20 @@ // tag::cloud[] -From your deployment menu, click **Performance**. The page's **CPU Usage** chart -shows your deployment's CPU usage as a percentage. +* (Recommended) Enable {cloud}/ec-monitoring-setup.html[logs and metrics]. When logs and metrics are enabled, monitoring information is visible on {kib}'s {kibana-ref}/xpack-monitoring.html[Stack Monitoring] page. ++ +You can also enable the {kibana-ref}/kibana-alerts.html[CPU usage threshold alert] to be notified about potential issues through email. -High CPU usage can also deplete your CPU credits. CPU credits let {ess} provide -smaller clusters with a performance boost when needed. The **CPU credits** -chart shows your remaining CPU credits, measured in seconds of CPU time. +* From your deployment menu, view the {cloud}/ec-saas-metrics-accessing.html[**Performance**] page. On this page, you can view two key metrics: +** **CPU usage**: Your deployment's CPU usage, represented as a percentage. +** **CPU credits**: Your remaining CPU credits, measured in seconds of CPU time. -You can also use the <> to get the current CPU usage -for each node. - -// tag::cpu-usage-cat-nodes[] -[source,console] ----- -GET _cat/nodes?v=true&s=cpu:desc ----- - -The response's `cpu` column contains the current CPU usage as a percentage. The -`name` column contains the node's name. -// end::cpu-usage-cat-nodes[] +{ess} grants {cloud}/ec-vcpu-boost-instance.html[CPU credits] per deployment +to provide smaller clusters with performance boosts when needed. High CPU +usage can deplete these credits, which might lead to {cloud}/ec-scenario_why_is_performance_degrading_over_time.html[performance degradation] and {cloud}/ec-scenario_why_are_my_cluster_response_times_suddenly_so_much_worse.html[increased cluster response times]. // end::cloud[] // tag::self-managed[] - -Use the <> to get the current CPU usage for each node. - -include::cpu-usage.asciidoc[tag=cpu-usage-cat-nodes] - +* Enable <>. When logs and metrics are enabled, monitoring information is visible on {kib}'s {kibana-ref}/xpack-monitoring.html[Stack Monitoring] page. ++ +You can also enable the {kibana-ref}/kibana-alerts.html[CPU usage threshold alert] to be notified about potential issues through email. // end::self-managed[] diff --git a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline.asciidoc index f1652c1d8aff8..66fa1b0739364 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline.asciidoc @@ -2,7 +2,7 @@ [source,console] ---- -DELETE _ingest/pipeline/*_embeddings +DELETE _ingest/pipeline/*_embeddings_pipeline ---- // TEST // TEARDOWN @@ -13,7 +13,7 @@ DELETE _ingest/pipeline/*_embeddings [source,console] -------------------------------------------------- -PUT _ingest/pipeline/cohere_embeddings +PUT _ingest/pipeline/cohere_embeddings_pipeline { "processors": [ { @@ -39,7 +39,7 @@ and the `output_field` that will contain the {infer} results. [source,console] -------------------------------------------------- -PUT _ingest/pipeline/elser_embeddings +PUT _ingest/pipeline/elser_embeddings_pipeline { "processors": [ { @@ -65,7 +65,7 @@ and the `output_field` that will contain the {infer} results. [source,console] -------------------------------------------------- -PUT _ingest/pipeline/hugging_face_embeddings +PUT _ingest/pipeline/hugging_face_embeddings_pipeline { "processors": [ { @@ -91,7 +91,7 @@ and the `output_field` that will contain the {infer} results. [source,console] -------------------------------------------------- -PUT _ingest/pipeline/openai_embeddings +PUT _ingest/pipeline/openai_embeddings_pipeline { "processors": [ { @@ -117,7 +117,7 @@ and the `output_field` that will contain the {infer} results. [source,console] -------------------------------------------------- -PUT _ingest/pipeline/azure_openai_embeddings +PUT _ingest/pipeline/azure_openai_embeddings_pipeline { "processors": [ { @@ -143,7 +143,7 @@ and the `output_field` that will contain the {infer} results. [source,console] -------------------------------------------------- -PUT _ingest/pipeline/azure_ai_studio_embeddings +PUT _ingest/pipeline/azure_ai_studio_embeddings_pipeline { "processors": [ { @@ -169,7 +169,7 @@ and the `output_field` that will contain the {infer} results. [source,console] -------------------------------------------------- -PUT _ingest/pipeline/google_vertex_ai_embeddings +PUT _ingest/pipeline/google_vertex_ai_embeddings_pipeline { "processors": [ { @@ -195,7 +195,7 @@ and the `output_field` that will contain the {infer} results. [source,console] -------------------------------------------------- -PUT _ingest/pipeline/mistral_embeddings +PUT _ingest/pipeline/mistral_embeddings_pipeline { "processors": [ { @@ -221,7 +221,7 @@ and the `output_field` that will contain the {infer} results. [source,console] -------------------------------------------------- -PUT _ingest/pipeline/amazon_bedrock_embeddings +PUT _ingest/pipeline/amazon_bedrock_embeddings_pipeline { "processors": [ { @@ -247,7 +247,7 @@ and the `output_field` that will contain the {infer} results. [source,console] -------------------------------------------------- -PUT _ingest/pipeline/alibabacloud_ai_search_embeddings +PUT _ingest/pipeline/alibabacloud_ai_search_embeddings_pipeline { "processors": [ { diff --git a/docs/reference/tab-widgets/inference-api/infer-api-reindex.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-reindex.asciidoc index 0b0ab9a9cfe0e..759d9e8da86b5 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-reindex.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-reindex.asciidoc @@ -10,7 +10,7 @@ POST _reindex?wait_for_completion=false }, "dest": { "index": "cohere-embeddings", - "pipeline": "cohere_embeddings" + "pipeline": "cohere_embeddings_pipeline" } } ---- @@ -37,7 +37,7 @@ POST _reindex?wait_for_completion=false }, "dest": { "index": "elser-embeddings", - "pipeline": "elser_embeddings" + "pipeline": "elser_embeddings_pipeline" } } ---- @@ -60,7 +60,7 @@ POST _reindex?wait_for_completion=false }, "dest": { "index": "hugging-face-embeddings", - "pipeline": "hugging_face_embeddings" + "pipeline": "hugging_face_embeddings_pipeline" } } ---- @@ -84,7 +84,7 @@ POST _reindex?wait_for_completion=false }, "dest": { "index": "openai-embeddings", - "pipeline": "openai_embeddings" + "pipeline": "openai_embeddings_pipeline" } } ---- @@ -112,7 +112,7 @@ POST _reindex?wait_for_completion=false }, "dest": { "index": "azure-openai-embeddings", - "pipeline": "azure_openai_embeddings" + "pipeline": "azure_openai_embeddings_pipeline" } } ---- @@ -140,7 +140,7 @@ POST _reindex?wait_for_completion=false }, "dest": { "index": "azure-ai-studio-embeddings", - "pipeline": "azure_ai_studio_embeddings" + "pipeline": "azure_ai_studio_embeddings_pipeline" } } ---- @@ -167,7 +167,7 @@ POST _reindex?wait_for_completion=false }, "dest": { "index": "google-vertex-ai-embeddings", - "pipeline": "google_vertex_ai_embeddings" + "pipeline": "google_vertex_ai_embeddings_pipeline" } } ---- @@ -189,7 +189,7 @@ POST _reindex?wait_for_completion=false }, "dest": { "index": "mistral-embeddings", - "pipeline": "mistral_embeddings" + "pipeline": "mistral_embeddings_pipeline" } } ---- @@ -212,7 +212,7 @@ POST _reindex?wait_for_completion=false }, "dest": { "index": "amazon-bedrock-embeddings", - "pipeline": "amazon_bedrock_embeddings" + "pipeline": "amazon_bedrock_embeddings_pipeline" } } ---- @@ -235,7 +235,7 @@ POST _reindex?wait_for_completion=false }, "dest": { "index": "alibabacloud-ai-search-embeddings", - "pipeline": "alibabacloud_ai_search_embeddings" + "pipeline": "alibabacloud_ai_search_embeddings_pipeline" } } ---- diff --git a/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc index eeecb4718658a..9e935f79aa0ac 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc @@ -8,7 +8,7 @@ the Cohere service. // tag::elser[] ELSER is a model trained by Elastic. If you have an {es} deployment, there is no -further requirement for using the {infer} API with the `elser` service. +further requirement for using the {infer} API with the `elasticsearch` service. // end::elser[] diff --git a/docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc index 6a2bdbb5e79a4..227eb774a4a9f 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc @@ -36,7 +36,7 @@ the `cosine` measures are equivalent. ------------------------------------------------------------ PUT _inference/sparse_embedding/elser_embeddings <1> { - "service": "elser", + "service": "elasticsearch", "service_settings": { "num_allocations": 1, "num_threads": 1 @@ -50,6 +50,14 @@ is the unique identifier of the {infer} endpoint is `elser_embeddings`. You don't need to download and deploy the ELSER model upfront, the API request above will download the model if it's not downloaded yet and then deploy it. +[NOTE] +==== +You might see a 502 bad gateway error in the response when using the {kib} Console. +This error usually just reflects a timeout, while the model downloads in the background. +You can check the download progress in the {ml-app} UI. +If using the Python client, you can set the `timeout` parameter to a higher value. +==== + // end::elser[] // tag::hugging-face[] @@ -198,7 +206,7 @@ PUT _inference/text_embedding/google_vertex_ai_embeddings <1> <2> A valid service account in JSON format for the Google Vertex AI API. <3> For the list of the available models, refer to the https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api[Text embeddings API] page. <4> The name of the location to use for the {infer} task. Refer to https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations[Generative AI on Vertex AI locations] for available locations. -<5> The name of the project to use for the {infer} task. +<5> The name of the project to use for the {infer} task. // end::google-vertex-ai[] diff --git a/docs/reference/tab-widgets/troubleshooting/snapshot/corrupt-repository.asciidoc b/docs/reference/tab-widgets/troubleshooting/snapshot/corrupt-repository.asciidoc index b2e864aab6db9..942b0f6ba21a6 100644 --- a/docs/reference/tab-widgets/troubleshooting/snapshot/corrupt-repository.asciidoc +++ b/docs/reference/tab-widgets/troubleshooting/snapshot/corrupt-repository.asciidoc @@ -71,7 +71,7 @@ GET _snapshot/my-repo ---- // TEST[skip:we're not setting up repos in these tests] + -The reponse will look like this: +The response will look like this: + [source,console-result] ---- diff --git a/docs/reference/text-structure/apis/find-field-structure.asciidoc b/docs/reference/text-structure/apis/find-field-structure.asciidoc index 4fa108e92d4cb..c4b289e6c30a9 100644 --- a/docs/reference/text-structure/apis/find-field-structure.asciidoc +++ b/docs/reference/text-structure/apis/find-field-structure.asciidoc @@ -2,6 +2,12 @@ [[find-field-structure]] = Find field structure API +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-text_structure[Text structure APIs]. +-- + Finds the structure of a field in an Elasticsearch index. [discrete] diff --git a/docs/reference/text-structure/apis/find-message-structure.asciidoc b/docs/reference/text-structure/apis/find-message-structure.asciidoc index 6c1bf5089bed2..18b85069559e3 100644 --- a/docs/reference/text-structure/apis/find-message-structure.asciidoc +++ b/docs/reference/text-structure/apis/find-message-structure.asciidoc @@ -2,6 +2,12 @@ [[find-message-structure]] = Find messages structure API +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-text_structure[Text structure APIs]. +-- + Finds the structure of a list of text messages. [discrete] diff --git a/docs/reference/text-structure/apis/find-structure.asciidoc b/docs/reference/text-structure/apis/find-structure.asciidoc index 361560bace4ed..fef0584222e13 100644 --- a/docs/reference/text-structure/apis/find-structure.asciidoc +++ b/docs/reference/text-structure/apis/find-structure.asciidoc @@ -2,6 +2,12 @@ [[find-structure]] = Find text structure API +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-text_structure[Text structure APIs]. +-- + Finds the structure of text. The text must contain data that is suitable to be ingested into the {stack}. diff --git a/docs/reference/text-structure/apis/index.asciidoc b/docs/reference/text-structure/apis/index.asciidoc index 9f4af120690f7..68607ddf708be 100644 --- a/docs/reference/text-structure/apis/index.asciidoc +++ b/docs/reference/text-structure/apis/index.asciidoc @@ -2,6 +2,12 @@ [[text-structure-apis]] == Text structure APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-text_structure[Text structure APIs]. +-- + You can use the following APIs to find text structures: * <> diff --git a/docs/reference/text-structure/apis/test-grok-pattern.asciidoc b/docs/reference/text-structure/apis/test-grok-pattern.asciidoc index 4034a24cf0a19..7c1cfb41608e9 100644 --- a/docs/reference/text-structure/apis/test-grok-pattern.asciidoc +++ b/docs/reference/text-structure/apis/test-grok-pattern.asciidoc @@ -6,6 +6,12 @@ Test Grok pattern ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-text_structure[Text structure APIs]. +-- + Tests a Grok pattern on lines of text, see also <>. [discrete] diff --git a/docs/reference/transform/apis/delete-transform.asciidoc b/docs/reference/transform/apis/delete-transform.asciidoc index 111dda23690b6..0b386f946d6c3 100644 --- a/docs/reference/transform/apis/delete-transform.asciidoc +++ b/docs/reference/transform/apis/delete-transform.asciidoc @@ -7,6 +7,12 @@ Delete {transform} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-transform[{transform-cap} APIs]. +-- + Deletes an existing {transform}. [[delete-transform-request]] diff --git a/docs/reference/transform/apis/get-transform-stats.asciidoc b/docs/reference/transform/apis/get-transform-stats.asciidoc index 273b1d094979e..e8d12a994faa3 100644 --- a/docs/reference/transform/apis/get-transform-stats.asciidoc +++ b/docs/reference/transform/apis/get-transform-stats.asciidoc @@ -7,6 +7,12 @@ Get {transform} statistics ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-transform[{transform-cap} APIs]. +-- + Retrieves usage information for {transforms}. diff --git a/docs/reference/transform/apis/get-transform.asciidoc b/docs/reference/transform/apis/get-transform.asciidoc index ece59138e2893..3eabf4ba26988 100644 --- a/docs/reference/transform/apis/get-transform.asciidoc +++ b/docs/reference/transform/apis/get-transform.asciidoc @@ -6,6 +6,12 @@ Get {transforms} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-transform[{transform-cap} APIs]. +-- + Retrieves configuration information for {transforms}. [[get-transform-request]] diff --git a/docs/reference/transform/apis/preview-transform.asciidoc b/docs/reference/transform/apis/preview-transform.asciidoc index fa9ad0c0fc8f9..c280bef52e0ee 100644 --- a/docs/reference/transform/apis/preview-transform.asciidoc +++ b/docs/reference/transform/apis/preview-transform.asciidoc @@ -7,6 +7,12 @@ Preview {transform} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-transform[{transform-cap} APIs]. +-- + Previews a {transform}. [[preview-transform-request]] diff --git a/docs/reference/transform/apis/put-transform.asciidoc b/docs/reference/transform/apis/put-transform.asciidoc index ed2ceba0a7a59..fc9abc8c6470c 100644 --- a/docs/reference/transform/apis/put-transform.asciidoc +++ b/docs/reference/transform/apis/put-transform.asciidoc @@ -7,6 +7,12 @@ Create {transform} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-transform[{transform-cap} APIs]. +-- + Instantiates a {transform}. [[put-transform-request]] diff --git a/docs/reference/transform/apis/reset-transform.asciidoc b/docs/reference/transform/apis/reset-transform.asciidoc index 1194d3589275d..3d9fd5db180b2 100644 --- a/docs/reference/transform/apis/reset-transform.asciidoc +++ b/docs/reference/transform/apis/reset-transform.asciidoc @@ -8,6 +8,12 @@ Reset {transform} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-transform[{transform-cap} APIs]. +-- + Resets a {transform}. [[reset-transform-request]] diff --git a/docs/reference/transform/apis/schedule-now-transform.asciidoc b/docs/reference/transform/apis/schedule-now-transform.asciidoc index 7a276edf08819..3349e14b02caf 100644 --- a/docs/reference/transform/apis/schedule-now-transform.asciidoc +++ b/docs/reference/transform/apis/schedule-now-transform.asciidoc @@ -8,6 +8,12 @@ Schedule now {transform} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-transform[{transform-cap} APIs]. +-- + Instantly runs a {transform} to process data. [[schedule-now-transform-request]] diff --git a/docs/reference/transform/apis/start-transform.asciidoc b/docs/reference/transform/apis/start-transform.asciidoc index f4f99f0f3457a..4bcb951f4c6b3 100644 --- a/docs/reference/transform/apis/start-transform.asciidoc +++ b/docs/reference/transform/apis/start-transform.asciidoc @@ -7,6 +7,12 @@ Start {transform} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-transform[{transform-cap} APIs]. +-- + Starts a {transform}. [[start-transform-request]] diff --git a/docs/reference/transform/apis/stop-transform.asciidoc b/docs/reference/transform/apis/stop-transform.asciidoc index e99fcbd413eba..d87784e036ae4 100644 --- a/docs/reference/transform/apis/stop-transform.asciidoc +++ b/docs/reference/transform/apis/stop-transform.asciidoc @@ -7,6 +7,12 @@ Stop {transforms} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-transform[{transform-cap} APIs]. +-- + Stops one or more {transforms}. diff --git a/docs/reference/transform/apis/transform-apis.asciidoc b/docs/reference/transform/apis/transform-apis.asciidoc index 20e5960e5bb18..45131f0214f9d 100644 --- a/docs/reference/transform/apis/transform-apis.asciidoc +++ b/docs/reference/transform/apis/transform-apis.asciidoc @@ -2,6 +2,12 @@ [[transform-apis]] = {transform-cap} APIs +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-transform[{transform-cap} APIs]. +-- + * <> * <> * <> diff --git a/docs/reference/transform/apis/update-transform.asciidoc b/docs/reference/transform/apis/update-transform.asciidoc index 1ac7d6d5410d2..c473ca0f83b8a 100644 --- a/docs/reference/transform/apis/update-transform.asciidoc +++ b/docs/reference/transform/apis/update-transform.asciidoc @@ -7,6 +7,12 @@ Update {transform} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-transform[{transform-cap} APIs]. +-- + Updates certain properties of a {transform}. [[update-transform-request]] diff --git a/docs/reference/transform/apis/upgrade-transforms.asciidoc b/docs/reference/transform/apis/upgrade-transforms.asciidoc index a1b01a6fd146a..826243938a9f4 100644 --- a/docs/reference/transform/apis/upgrade-transforms.asciidoc +++ b/docs/reference/transform/apis/upgrade-transforms.asciidoc @@ -7,6 +7,12 @@ Upgrade {transforms} ++++ +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-transform[{transform-cap} APIs]. +-- + Upgrades all {transforms}. [[upgrade-transforms-request]] diff --git a/docs/reference/transform/painless-examples.asciidoc b/docs/reference/transform/painless-examples.asciidoc index 8eb50964f4d5b..4b0802c79a340 100644 --- a/docs/reference/transform/painless-examples.asciidoc +++ b/docs/reference/transform/painless-examples.asciidoc @@ -5,6 +5,9 @@ Painless examples ++++ + +IMPORTANT: The examples that use the `scripted_metric` aggregation are not supported on {es} Serverless. + These examples demonstrate how to use Painless in {transforms}. You can learn more about the Painless scripting language in the {painless}/painless-guide.html[Painless guide]. @@ -37,6 +40,8 @@ with the latest timestamp. From a technical perspective, it helps to achieve the function of a <> by using scripted metric aggregation in a {transform}, which provides a metric output. +IMPORTANT: This example uses a `scripted_metric` aggregation which is not supported on {es} Serverless. + [source,js] -------------------------------------------------- "aggregations": { @@ -453,6 +458,8 @@ example for details. The example below assumes that the HTTP response codes are stored as keywords in the `response` field of the documents. +IMPORTANT: This example uses a `scripted_metric` aggregation which is not supported on {es} Serverless. + [source,js] -------------------------------------------------- "aggregations": { <1> @@ -507,7 +514,9 @@ Finally, returns the `counts` array with the response counts. == Comparing indices by using scripted metric aggregations This example shows how to compare the content of two indices by a {transform} -that uses a scripted metric aggregation. +that uses a scripted metric aggregation. + +IMPORTANT: This example uses a `scripted_metric` aggregation which is not supported on {es} Serverless. [source,console] -------------------------------------------------- @@ -623,6 +632,8 @@ By using the `sessionid` as a group-by field, you are able to enumerate events through the session and get more details of the session by using scripted metric aggregation. +IMPORTANT: This example uses a `scripted_metric` aggregation which is not supported on {es} Serverless. + [source,js] -------------------------------------------------- POST _transform/_preview diff --git a/docs/reference/troubleshooting/common-issues/diagnose-unassigned-shards.asciidoc b/docs/reference/troubleshooting/common-issues/diagnose-unassigned-shards.asciidoc index fe9422d6d4c53..e1ceefb92bbec 100644 --- a/docs/reference/troubleshooting/common-issues/diagnose-unassigned-shards.asciidoc +++ b/docs/reference/troubleshooting/common-issues/diagnose-unassigned-shards.asciidoc @@ -8,5 +8,6 @@ In order to diagnose the unassigned shards in your deployment use the following include::{es-ref-dir}/tab-widgets/troubleshooting/data/diagnose-unassigned-shards-widget.asciidoc[] - +See https://www.youtube.com/watch?v=v2mbeSd1vTQ[this video] +for a walkthrough of monitoring allocation health. diff --git a/docs/reference/troubleshooting/common-issues/disk-usage-exceeded.asciidoc b/docs/reference/troubleshooting/common-issues/disk-usage-exceeded.asciidoc index 7eb27d5428956..2222d09c26db2 100644 --- a/docs/reference/troubleshooting/common-issues/disk-usage-exceeded.asciidoc +++ b/docs/reference/troubleshooting/common-issues/disk-usage-exceeded.asciidoc @@ -106,6 +106,8 @@ As a long-term solution, we recommend you do one of the following best suited to your use case: * add nodes to the affected <> ++ +TIP: You should enable <> for clusters deployed using our {ess}, {ece}, and {eck} platforms. * upgrade existing nodes to increase disk space + diff --git a/docs/reference/troubleshooting/common-issues/high-cpu-usage.asciidoc b/docs/reference/troubleshooting/common-issues/high-cpu-usage.asciidoc index 858683ef97a6d..96a9a8f1e32b7 100644 --- a/docs/reference/troubleshooting/common-issues/high-cpu-usage.asciidoc +++ b/docs/reference/troubleshooting/common-issues/high-cpu-usage.asciidoc @@ -9,12 +9,29 @@ If a thread pool is depleted, {es} will <> related to the thread pool. For example, if the `search` thread pool is depleted, {es} will reject search requests until more threads are available. +You might experience high CPU usage if a <>, and therefore the nodes assigned to that tier, is experiencing more traffic than other tiers. This imbalance in resource utilization is also known as <>. + [discrete] [[diagnose-high-cpu-usage]] ==== Diagnose high CPU usage **Check CPU usage** +You can check the CPU usage per node using the <>: + +// tag::cpu-usage-cat-nodes[] +[source,console] +---- +GET _cat/nodes?v=true&s=cpu:desc +---- + +The response's `cpu` column contains the current CPU usage as a percentage. +The `name` column contains the node's name. Elevated but transient CPU usage is +normal. However, if CPU usage is elevated for an extended duration, it should be +investigated. + +To track CPU usage over time, we recommend enabling monitoring: + include::{es-ref-dir}/tab-widgets/cpu-usage-widget.asciidoc[] **Check hot threads** @@ -24,11 +41,13 @@ threads API>> to check for resource-intensive threads running on the node. [source,console] ---- -GET _nodes/my-node,my-other-node/hot_threads +GET _nodes/hot_threads ---- // TEST[s/\/my-node,my-other-node//] -This API returns a breakdown of any hot threads in plain text. +This API returns a breakdown of any hot threads in plain text. High CPU usage +frequently correlates to <>. [discrete] [[reduce-cpu-usage]] diff --git a/docs/reference/troubleshooting/common-issues/high-jvm-memory-pressure.asciidoc b/docs/reference/troubleshooting/common-issues/high-jvm-memory-pressure.asciidoc index 267d6594b8025..3469a0ca5bf42 100644 --- a/docs/reference/troubleshooting/common-issues/high-jvm-memory-pressure.asciidoc +++ b/docs/reference/troubleshooting/common-issues/high-jvm-memory-pressure.asciidoc @@ -66,6 +66,8 @@ searches, consider the following setting changes: <> cluster setting. +* Set a default search timeout using the <> cluster setting. + [source,console] ---- PUT _settings diff --git a/docs/reference/troubleshooting/common-issues/red-yellow-cluster-status.asciidoc b/docs/reference/troubleshooting/common-issues/red-yellow-cluster-status.asciidoc index cae4eb99dd54a..eaa7cbc9e166c 100644 --- a/docs/reference/troubleshooting/common-issues/red-yellow-cluster-status.asciidoc +++ b/docs/reference/troubleshooting/common-issues/red-yellow-cluster-status.asciidoc @@ -19,6 +19,8 @@ operate to have a green health status. In many cases, your cluster will recover to green health status automatically. If the cluster doesn't automatically recover, then you must <> the remaining problems so management and cleanup activities can proceed. +See https://www.youtube.com/watch?v=v2mbeSd1vTQ[this video] +for a walkthrough of monitoring allocation health. [discrete] [[diagnose-cluster-status]] @@ -90,6 +92,8 @@ PUT _cluster/settings } ---- +See https://www.youtube.com/watch?v=MiKKUdZvwnI[this video] for walkthrough of troubleshooting "no allocations are allowed". + [discrete] [[fix-cluster-status-recover-nodes]] ===== Recover lost nodes @@ -262,3 +266,5 @@ POST _cluster/reroute?metric=none ---- // TEST[s/^/PUT my-index\n/] // TEST[catch:bad_request] + +See https://www.youtube.com/watch?v=6OAg9IyXFO4[this video] for a walkthrough of troubleshooting `no_valid_shard_copy`. \ No newline at end of file diff --git a/docs/reference/troubleshooting/common-issues/rejected-requests.asciidoc b/docs/reference/troubleshooting/common-issues/rejected-requests.asciidoc index 497bddc562c69..4b01c78b3275f 100644 --- a/docs/reference/troubleshooting/common-issues/rejected-requests.asciidoc +++ b/docs/reference/troubleshooting/common-issues/rejected-requests.asciidoc @@ -23,9 +23,52 @@ To check the number of rejected tasks for each thread pool, use the [source,console] ---- -GET /_cat/thread_pool?v=true&h=id,name,active,rejected,completed +GET /_cat/thread_pool?v=true&h=id,name,queue,active,rejected,completed ---- +`write` thread pool rejections frequently appear in the erring API and +correlating log as `EsRejectedExecutionException` with either +`QueueResizingEsThreadPoolExecutor` or `queue capacity`. + +These errors are often related to <>. + +[discrete] +[[check-circuit-breakers]] +==== Check circuit breakers + +To check the number of tripped <>, use the +<>. + +[source,console] +---- +GET /_nodes/stats/breaker +---- + +These statistics are cumulative from node startup. For more information, see +<>. + +[discrete] +[[check-indexing-pressure]] +==== Check indexing pressure + +To check the number of <> +rejections, use the <>. + +[source,console] +---- +GET _nodes/stats?human&filter_path=nodes.*.indexing_pressure +---- + +These stats are cumulative from node startup. + +Indexing pressure rejections appear as an +`EsRejectedExecutionException`, and indicate that they were rejected due +to `combined_coordinating_and_primary`, `coordinating`, `primary`, or `replica`. + +These errors are often related to <>, +<> sizing, or the ingest target's +<>. + [discrete] [[prevent-rejected-requests]] ==== Prevent rejected requests @@ -35,8 +78,3 @@ GET /_cat/thread_pool?v=true&h=id,name,active,rejected,completed If {es} regularly rejects requests and other tasks, your cluster likely has high CPU usage or high JVM memory pressure. For tips, see <> and <>. - -**Prevent circuit breaker errors** - -If you regularly trigger circuit breaker errors, see <> -for tips on diagnosing and preventing them. \ No newline at end of file diff --git a/docs/reference/troubleshooting/common-issues/task-queue-backlog.asciidoc b/docs/reference/troubleshooting/common-issues/task-queue-backlog.asciidoc index 1ff5bf2e5c311..5aa6a0129c2d4 100644 --- a/docs/reference/troubleshooting/common-issues/task-queue-backlog.asciidoc +++ b/docs/reference/troubleshooting/common-issues/task-queue-backlog.asciidoc @@ -1,10 +1,10 @@ [[task-queue-backlog]] === Task queue backlog -A backlogged task queue can prevent tasks from completing and -put the cluster into an unhealthy state. -Resource constraints, a large number of tasks being triggered at once, -and long running tasks can all contribute to a backlogged task queue. +A backlogged task queue can prevent tasks from completing and put the cluster +into an unhealthy state. Resource constraints, a large number of tasks being +triggered at once, and long running tasks can all contribute to a backlogged +task queue. [discrete] [[diagnose-task-queue-backlog]] @@ -12,39 +12,77 @@ and long running tasks can all contribute to a backlogged task queue. **Check the thread pool status** -A <> can result in <>. +A <> can result in +<>. -You can use the <> to -see the number of active threads in each thread pool and -how many tasks are queued, how many have been rejected, and how many have completed. +Thread pool depletion might be restricted to a specific <>. If <> is occuring, one node might experience depletion faster than other nodes, leading to performance issues and a growing task backlog. + +You can use the <> to see the number of +active threads in each thread pool and how many tasks are queued, how many +have been rejected, and how many have completed. [source,console] ---- GET /_cat/thread_pool?v&s=t,n&h=type,name,node_name,active,queue,rejected,completed ---- +The `active` and `queue` statistics are instantaneous while the `rejected` and +`completed` statistics are cumulative from node startup. + **Inspect the hot threads on each node** -If a particular thread pool queue is backed up, -you can periodically poll the <> API -to determine if the thread has sufficient -resources to progress and gauge how quickly it is progressing. +If a particular thread pool queue is backed up, you can periodically poll the +<> API to determine if the thread +has sufficient resources to progress and gauge how quickly it is progressing. [source,console] ---- GET /_nodes/hot_threads ---- -**Look for long running tasks** +**Look for long running node tasks** + +Long-running tasks can also cause a backlog. You can use the <> API to get information about the node tasks that are running. +Check the `running_time_in_nanos` to identify tasks that are taking an +excessive amount of time to complete. + +[source,console] +---- +GET /_tasks?pretty=true&human=true&detailed=true +---- -Long-running tasks can also cause a backlog. -You can use the <> API to get information about the tasks that are running. -Check the `running_time_in_nanos` to identify tasks that are taking an excessive amount of time to complete. +If a particular `action` is suspected, you can filter the tasks further. The most common long-running tasks are <>- or search-related. +* Filter for <> actions: ++ [source,console] ---- -GET /_tasks?filter_path=nodes.*.tasks +GET /_tasks?human&detailed&actions=indices:data/write/bulk +---- + +* Filter for search actions: ++ +[source,console] ---- +GET /_tasks?human&detailed&actions=indices:data/write/search +---- + +The API response may contain additional tasks columns, including `description` and `header`, which provides the task parameters, target, and requestor. You can use this information to perform further diagnosis. + +**Look for long running cluster tasks** + +A task backlog might also appear as a delay in synchronizing the cluster state. You +can use the <> to get information +about the pending cluster state sync tasks that are running. + +[source,console] +---- +GET /_cluster/pending_tasks +---- + +Check the `timeInQueue` to identify tasks that are taking an excessive amount +of time to complete. [discrete] [[resolve-task-queue-backlog]] diff --git a/docs/reference/troubleshooting/diagnostic.asciidoc b/docs/reference/troubleshooting/diagnostic.asciidoc index a944ca88d285d..c6d46b9e94fc8 100644 --- a/docs/reference/troubleshooting/diagnostic.asciidoc +++ b/docs/reference/troubleshooting/diagnostic.asciidoc @@ -13,6 +13,8 @@ This information can be used to troubleshoot problems with your cluster. For exa You can generate diagnostic information using this tool before you contact https://support.elastic.co[Elastic Support] or https://discuss.elastic.co[Elastic Discuss] to minimize turnaround time. +See this https://www.youtube.com/watch?v=Bb6SaqhqYHw[this video] for a walkthrough of capturing an {es} diagnostic. + [discrete] [[diagnostic-tool-requirements]] === Requirements diff --git a/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc b/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc index cbb35f7731034..e47b85aa99547 100644 --- a/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc +++ b/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc @@ -126,7 +126,7 @@ repeatedly-dropped connections will severely affect its operation. The connections from the elected master node to every other node in the cluster are particularly important. The elected master never spontaneously closes its outbound connections to other nodes. Similarly, once an inbound connection is -fully established, a node never spontaneously it unless the node is shutting +fully established, a node never spontaneously closes it unless the node is shutting down. If you see a node unexpectedly leave the cluster with the `disconnected` diff --git a/docs/reference/vectors/vector-functions.asciidoc b/docs/reference/vectors/vector-functions.asciidoc index 2a80290cf9d3b..23419e8eb12b1 100644 --- a/docs/reference/vectors/vector-functions.asciidoc +++ b/docs/reference/vectors/vector-functions.asciidoc @@ -16,7 +16,7 @@ This is the list of available vector functions and vector access methods: 6. <].vectorValue`>> – returns a vector's value as an array of floats 7. <].magnitude`>> – returns a vector's magnitude -NOTE: The `cosineSimilarity` and `dotProduct` functions are not supported for `bit` vectors. +NOTE: The `cosineSimilarity` function is not supported for `bit` vectors. NOTE: The recommended way to access dense vectors is through the `cosineSimilarity`, `dotProduct`, `l1norm` or `l2norm` functions. Please note @@ -332,6 +332,96 @@ When using `bit` vectors, not all the vector functions are available. The suppor * <> – calculates Hamming distance, the sum of the bitwise XOR of the two vectors * <> – calculates L^1^ distance, this is simply the `hamming` distance * <> - calculates L^2^ distance, this is the square root of the `hamming` distance +* <> – calculates dot product. When comparing two `bit` vectors, +this is the sum of the bitwise AND of the two vectors. If providing `float[]` or `byte[]`, who has `dims` number of elements, as a query vector, the `dotProduct` is +the sum of the floating point values using the stored `bit` vector as a mask. -Currently, the `cosineSimilarity` and `dotProduct` functions are not supported for `bit` vectors. +NOTE: When comparing `floats` and `bytes` with `bit` vectors, the `bit` vector is treated as a mask in big-endian order. +For example, if the `bit` vector is `10100001` (e.g. the single byte value `161`) and its compared +with array of values `[1, 2, 3, 4, 5, 6, 7, 8]` the `dotProduct` will be `1 + 3 + 8 = 16`. + +Here is an example of using dot-product with bit vectors. + +[source,console] +-------------------------------------------------- +PUT my-index-bit-vectors +{ + "mappings": { + "properties": { + "my_dense_vector": { + "type": "dense_vector", + "index": false, + "element_type": "bit", + "dims": 40 <1> + } + } + } +} + +PUT my-index-bit-vectors/_doc/1 +{ + "my_dense_vector": [8, 5, -15, 1, -7] <2> +} + +PUT my-index-bit-vectors/_doc/2 +{ + "my_dense_vector": [-1, 115, -3, 4, -128] +} + +PUT my-index-bit-vectors/_doc/3 +{ + "my_dense_vector": [2, 18, -5, 0, -124] +} + +POST my-index-bit-vectors/_refresh +-------------------------------------------------- +// TEST[continued] +<1> The number of dimensions or bits for the `bit` vector. +<2> This vector represents 5 bytes, or `5 * 8 = 40` bits, which equals the configured dimensions + +[source,console] +-------------------------------------------------- +GET my-index-bit-vectors/_search +{ + "query": { + "script_score": { + "query" : { + "match_all": {} + }, + "script": { + "source": "dotProduct(params.query_vector, 'my_dense_vector')", + "params": { + "query_vector": [8, 5, -15, 1, -7] <1> + } + } + } + } +} +-------------------------------------------------- +// TEST[continued] +<1> This vector is 40 bits, and thus will compute a bitwise `&` operation with the stored vectors. + +[source,console] +-------------------------------------------------- +GET my-index-bit-vectors/_search +{ + "query": { + "script_score": { + "query" : { + "match_all": {} + }, + "script": { + "source": "dotProduct(params.query_vector, 'my_dense_vector')", + "params": { + "query_vector": [0.23, 1.45, 3.67, 4.89, -0.56, 2.34, 3.21, 1.78, -2.45, 0.98, -0.12, 3.45, 4.56, 2.78, 1.23, 0.67, 3.89, 4.12, -2.34, 1.56, 0.78, 3.21, 4.12, 2.45, -1.67, 0.34, -3.45, 4.56, -2.78, 1.23, -0.67, 3.89, -4.34, 2.12, -1.56, 0.78, -3.21, 4.45, 2.12, 1.67] <1> + } + } + } + } +} +-------------------------------------------------- +// TEST[continued] +<1> This vector is 40 individual dimensions, and thus will sum the floating point values using the stored `bit` vector as a mask. + +Currently, the `cosineSimilarity` function is not supported for `bit` vectors. diff --git a/docs/reference/watcher/example-watches/watching-time-series-data.asciidoc b/docs/reference/watcher/example-watches/watching-time-series-data.asciidoc index 421c69619cfea..b1c776baae1de 100644 --- a/docs/reference/watcher/example-watches/watching-time-series-data.asciidoc +++ b/docs/reference/watcher/example-watches/watching-time-series-data.asciidoc @@ -62,20 +62,20 @@ contain the words "error" or "problem". To set up the watch: -. Define the watch trigger--a daily schedule that runs at 12:00 UTC: +. Define the watch trigger--a daily schedule that runs at 12:00 Australian Eastern Standard Time (UTC+10:00): + [source,js] -------------------------------------------------- "trigger" : { "schedule" : { + "timezone": "Australia/Brisbane", "daily" : { "at" : "12:00" } } } -------------------------------------------------- + -NOTE: In {watcher}, you specify times in UTC time. Don't forget to do the - conversion from your local time so the schedule triggers at the time - you intend. +NOTE: In {watcher}, if the timezone is omitted then schedules default to UTC. `timezone` can be specified either +as a +/-HH:mm offset from UTC or as a timezone name from the machines local IANA Time Zone Database. . Define the watch input--a search that uses a filter to constrain the results to the past day. diff --git a/docs/reference/watcher/getting-started.asciidoc b/docs/reference/watcher/getting-started.asciidoc index c5e7c108742ed..630445f995e07 100644 --- a/docs/reference/watcher/getting-started.asciidoc +++ b/docs/reference/watcher/getting-started.asciidoc @@ -135,7 +135,7 @@ GET .watcher-history*/_search?pretty "bool" : { "must" : [ { "match" : { "result.condition.met" : true }}, - { "range" : { "result.execution_time" : { "from" : "now-10s" }}} + { "range" : { "result.execution_time" : { "gte" : "now-10s" }}} ] } } diff --git a/docs/reference/watcher/how-watcher-works.asciidoc b/docs/reference/watcher/how-watcher-works.asciidoc index ed6e49b72e9ce..e34d4f799d99b 100644 --- a/docs/reference/watcher/how-watcher-works.asciidoc +++ b/docs/reference/watcher/how-watcher-works.asciidoc @@ -146,15 +146,18 @@ add, the more distributed the watches can be executed. If you add or remove replicas, all watches need to be reloaded. If a shard is relocated, the primary and all replicas of this particular shard will reload. -Because the watches are executed on the node, where the watch shards are, you can create -dedicated watcher nodes by using shard allocation filtering. +Because the watches are executed on the node, where the watch shards are, you +can create dedicated watcher nodes by using shard allocation filtering. To do this +, configure nodes with a dedicated `node.attr.role: watcher` property. -You could configure nodes with a dedicated `node.attr.role: watcher` property and -then configure the `.watches` index like this: +As the `.watches` index is a system index, you can't use the normal `.watcher/_settings` +endpoint to modify its routing allocation. Instead, you can use the following dedicated +endpoint to adjust the allocation of the `.watches` shards to the nodes with the +`watcher` role attribute: [source,console] ------------------------ -PUT .watches/_settings +PUT _watcher/settings { "index.routing.allocation.include.role": "watcher" } diff --git a/docs/reference/watcher/transform/search.asciidoc b/docs/reference/watcher/transform/search.asciidoc index deef57783ea91..21db8293f3fa2 100644 --- a/docs/reference/watcher/transform/search.asciidoc +++ b/docs/reference/watcher/transform/search.asciidoc @@ -119,8 +119,8 @@ time of the watch: { "range" : { "@timestamp" : { - "from" : "{{ctx.trigger.scheduled_time}}||-30s", - "to" : "{{ctx.trigger.triggered_time}}" + "gte" : "{{ctx.trigger.scheduled_time}}||-30s", + "lte" : "{{ctx.trigger.triggered_time}}" } } } @@ -159,8 +159,8 @@ The following is an example of using templates that refer to provided parameters { "range" : { "@timestamp" : { - "from" : "{{ctx.trigger.scheduled_time}}||-30s", - "to" : "{{ctx.trigger.triggered_time}}" + "gte" : "{{ctx.trigger.scheduled_time}}||-30s", + "lte" : "{{ctx.trigger.triggered_time}}" } } } diff --git a/docs/reference/watcher/trigger/schedule.asciidoc b/docs/reference/watcher/trigger/schedule.asciidoc index fa389409d15c4..d2bf466644e10 100644 --- a/docs/reference/watcher/trigger/schedule.asciidoc +++ b/docs/reference/watcher/trigger/schedule.asciidoc @@ -6,12 +6,42 @@ ++++ Schedule <> define when the watch execution should start based -on date and time. All times are specified in UTC time. +on date and time. All times are in UTC time unless a timezone is explicitly specified +in the schedule. {watcher} uses the system clock to determine the current time. To ensure schedules are triggered when expected, you should synchronize the clocks of all nodes in the cluster using a time service such as http://www.ntp.org/[NTP]. +NOTE: {watcher} can't correct for manual adjustments to the system clock. Be aware when making +such changes that watch execution may be affected with watches being skipped or repeated if the +adjustment covers their target execution time. This applies to changes made via NTP as well. + +When specifying a timezone for a watch, keep in mind the effect daylight savings time +transitions may have on the schedule, especially if the watch is scheduled to run +during the transition. Here's how {watcher} handles watches scheduled during discontinuities: + +==== Gap Transitions +These occur when the clock moves forward, such as when daylight savings time starts +and cause certain hours or minutes to be skipped. If your watch is scheduled to run +during a gap transition, the watch is executed at the same time as before the transition. + +Example: If a watch is scheduled to run daily at 1:30AM in the `Europe/London` time zone and +the clock moves forward one hour from 1:00AM (GMT+0) to 2:00AM (GMT+1), the watch is executed +at 2:30AM (GMT+1) which would have been 1:30AM before the transition. Subsequent executions +happen at 1:30AM (GMT+1). + +==== Overlap Transitions +These occur when the clock moves backward, such as when daylight savings time ends +and cause certain hours or minutes to be repeated. If your watch is scheduled to run +during an overlap transition, only the first occurrence of the time causes to the watch +to execute with the second being skipped. + +Example: If a watch is scheduled to run at 1:30 AM and the clock moves backward one hour +from 2:00AM to 1:00AM, the watch is executed at 1:30AM and the second occurrence after the +change is skipped. + +=== Throttling Keep in mind that the throttle period can affect when a watch is actually executed. The default throttle period is five seconds (5000 ms). If you configure a schedule that's more frequent than the throttle period, the throttle period overrides the @@ -20,6 +50,7 @@ and set the schedule to every 10 seconds, the watch is executed no more than once per minute. For more information about throttling, see <>. +=== Schedule Types {watcher} provides several types of schedule triggers: * <> diff --git a/docs/reference/watcher/trigger/schedule/cron.asciidoc b/docs/reference/watcher/trigger/schedule/cron.asciidoc index 673f350435c5f..c33bf524a8737 100644 --- a/docs/reference/watcher/trigger/schedule/cron.asciidoc +++ b/docs/reference/watcher/trigger/schedule/cron.asciidoc @@ -5,14 +5,14 @@ ++++ -Defines a <> using a <> +Defines a <> using a <> that specifiues when to execute a watch. -TIP: While cron expressions are powerful, a regularly occurring schedule -is easier to configure with the other schedule types. -If you must use a cron schedule, make sure you verify it with -<> . +TIP: While cron expressions are powerful, a regularly occurring schedule +is easier to configure with the other schedule types. +If you must use a cron schedule, make sure you verify it with +<> . ===== Configure a cron schedule with one time @@ -60,16 +60,40 @@ minute during the weekend: -------------------------------------------------- // NOTCONSOLE +[[configue_cron_time-zone]] +==== Use a different time zone for a cron schedule +By default, cron expressions are evaluated in the UTC time zone. To use a different time zone, +you can specify the `timezone` parameter in the schedule. For example, the following +`cron` schedule triggers at 6:00 AM and 6:00 PM during weekends in the `America/Los_Angeles` time zone: + + +[source,js] +-------------------------------------------------- +{ + ... + "trigger" : { + "schedule" : { + "timezone" : "America/Los_Angeles", + "cron" : [ + "0 6,18 * * * SAT-SUN", + ] + } + } + ... +} +-------------------------------------------------- +// NOTCONSOLE + [[croneval]] ===== Use croneval to validate cron expressions -{es} provides a <> command line tool -in the `$ES_HOME/bin` directory that you can use to check that your cron expressions +{es} provides a <> command line tool +in the `$ES_HOME/bin` directory that you can use to check that your cron expressions are valid and produce the expected results. -To validate a cron expression, pass it in as a parameter to `elasticsearch-croneval`: +To validate a cron expression, pass it in as a parameter to `elasticsearch-croneval`: [source,bash] -------------------------------------------------- bin/elasticsearch-croneval "0 0/1 * * * ?" --------------------------------------------------- +-------------------------------------------------- diff --git a/docs/reference/watcher/trigger/schedule/daily.asciidoc b/docs/reference/watcher/trigger/schedule/daily.asciidoc index cea2b8316e02f..d258d9c612350 100644 --- a/docs/reference/watcher/trigger/schedule/daily.asciidoc +++ b/docs/reference/watcher/trigger/schedule/daily.asciidoc @@ -97,3 +97,27 @@ or minutes as an array. For example, following `daily` schedule triggers at } -------------------------------------------------- // NOTCONSOLE + +[[specifying-time-zone-for-daily-schedule]] +===== Specifying a time zone for a daily schedule +By default, daily schedules are evaluated in the UTC time zone. To use a different time zone, +you can specify the `timezone` parameter in the schedule. For example, the following +`daily` schedule triggers at 6:00 AM and 6:00 PM in the `Pacific/Galapagos` time zone: + +[source,js] +-------------------------------------------------- +{ + "trigger" : { + "schedule" : { + "timezone" : "Pacific/Galapagos", + "daily" : { + "at" : { + "hour" : [ 6, 18 ], + "minute" : 0 + } + } + } + } +} +-------------------------------------------------- +// NOTCONSOLE diff --git a/docs/reference/watcher/trigger/schedule/monthly.asciidoc b/docs/reference/watcher/trigger/schedule/monthly.asciidoc index 7d13262ed2fa8..694c76aaee23a 100644 --- a/docs/reference/watcher/trigger/schedule/monthly.asciidoc +++ b/docs/reference/watcher/trigger/schedule/monthly.asciidoc @@ -74,4 +74,26 @@ schedule triggers at 12:00 AM and 12:00 PM on the 10th and 20th of each month. } } -------------------------------------------------- -// NOTCONSOLE \ No newline at end of file +// NOTCONSOLE + +==== Configuring time zones for monthly schedules +By default, monthly schedules are evaluated in the UTC time zone. To use a different +time zone, you can specify the `timezone` parameter in the schedule. For example, +the following `monthly` schedule triggers at 6:00 AM and 6:00 PM on the 15th of each month in +the `Asia/Tokyo` time zone: + +[source,js] +-------------------------------------------------- +{ + "trigger" : { + "schedule" : { + "timezone" : "Asia/Tokyo", + "monthly" : { + "on" : [ 15 ], + "at" : [ 6:00, 18:00 ] + } + } + } +} +-------------------------------------------------- +// NOTCONSOLE diff --git a/docs/reference/watcher/trigger/schedule/weekly.asciidoc b/docs/reference/watcher/trigger/schedule/weekly.asciidoc index 5b43de019ad25..53bd2f3167b21 100644 --- a/docs/reference/watcher/trigger/schedule/weekly.asciidoc +++ b/docs/reference/watcher/trigger/schedule/weekly.asciidoc @@ -79,4 +79,26 @@ Alternatively, you can specify days and times in an object that has `on` and } } -------------------------------------------------- -// NOTCONSOLE \ No newline at end of file +// NOTCONSOLE + +==== Use a different time zone for a weekly schedule +By default, weekly schedules are evaluated in the UTC time zone. To use a different time zone, +you can specify the `timezone` parameter in the schedule. For example, the following +`weekly` schedule triggers at 6:00 AM and 6:00 PM on Tuesdays and Fridays in the +`America/Buenos_Aires` time zone: + +[source,js] +-------------------------------------------------- +{ + "trigger" : { + "schedule" : { + "timezone" : "America/Buenos_Aires", + "weekly" : { + "on" : [ "tuesday", "friday" ], + "at" : [ "6:00", "18:00" ] + } + } + } +} +-------------------------------------------------- +// NOTCONSOLE diff --git a/docs/reference/watcher/trigger/schedule/yearly.asciidoc b/docs/reference/watcher/trigger/schedule/yearly.asciidoc index 8fce024bf9f4a..c33321ef5a7dc 100644 --- a/docs/reference/watcher/trigger/schedule/yearly.asciidoc +++ b/docs/reference/watcher/trigger/schedule/yearly.asciidoc @@ -88,3 +88,26 @@ January 20th, December 10th, and December 20th. } -------------------------------------------------- // NOTCONSOLE + +==== Configuring a yearly schedule with a different time zone +By default, the `yearly` schedule is evaluated in the UTC time zone. To use a +different time zone, you can specify the `timezone` parameter in the schedule. +For example, the following `yearly` schedule triggers at 3:30 PM and 8:30 PM +on June 4th in the `Antarctica/Troll` time zone: + +[source,js] +-------------------------------------------------- +{ + "trigger" : { + "schedule" : { + "timezone" : "Antarctica/Troll", + "yearly" : { + "in" : "june", + "on" : 4, + "at" : [ 15:30, 20:30 ] + } + } + } +} +-------------------------------------------------- +// NOTCONSOLE diff --git a/docs/src/yamlRestTest/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java b/docs/src/yamlRestTest/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java index 6191f33f1c5dd..7ddb2ca354cfc 100644 --- a/docs/src/yamlRestTest/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java +++ b/docs/src/yamlRestTest/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.smoketest; diff --git a/gradle.properties b/gradle.properties index 745fb4f9e51ae..aa38a61ab0057 100644 --- a/gradle.properties +++ b/gradle.properties @@ -2,7 +2,7 @@ org.gradle.welcome=never org.gradle.warning.mode=none org.gradle.parallel=true # We need to declare --add-exports to make spotless working seamlessly with jdk16 -org.gradle.jvmargs=-XX:+HeapDumpOnOutOfMemoryError -Xss2m --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED +org.gradle.jvmargs=-XX:+HeapDumpOnOutOfMemoryError -Xss2m --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-opens java.base/java.time=ALL-UNNAMED # Enforce the build to fail on deprecated gradle api usage systemProp.org.gradle.warning.mode=fail diff --git a/gradle/build.versions.toml b/gradle/build.versions.toml index 35c26ef10f9ec..05fda8e0244de 100644 --- a/gradle/build.versions.toml +++ b/gradle/build.versions.toml @@ -1,5 +1,5 @@ [versions] -asm = "9.6" +asm = "9.7.1" jackson = "2.15.0" junit5 = "5.8.1" spock = "2.1-groovy-3.0" @@ -16,8 +16,8 @@ checkstyle = "com.puppycrawl.tools:checkstyle:10.3" commons-codec = "commons-codec:commons-codec:1.11" commmons-io = "commons-io:commons-io:2.2" docker-compose = "com.avast.gradle:gradle-docker-compose-plugin:0.17.5" -forbiddenApis = "de.thetaphi:forbiddenapis:3.6" -gradle-enterprise = "com.gradle:develocity-gradle-plugin:3.17.4" +forbiddenApis = "de.thetaphi:forbiddenapis:3.8" +gradle-enterprise = "com.gradle:develocity-gradle-plugin:3.18.1" hamcrest = "org.hamcrest:hamcrest:2.1" httpcore = "org.apache.httpcomponents:httpcore:4.4.12" httpclient = "org.apache.httpcomponents:httpclient:4.5.14" @@ -39,7 +39,7 @@ maven-model = "org.apache.maven:maven-model:3.6.2" mockito-core = "org.mockito:mockito-core:1.9.5" nebula-info = "com.netflix.nebula:gradle-info-plugin:11.3.3" reflections = "org.reflections:reflections:0.9.12" -shadow-plugin = "com.github.breskeby:shadow:3b035f2" +shadow-plugin = "com.gradleup.shadow:shadow-gradle-plugin:8.3.5" snakeyaml = { group = "org.yaml", name = "snakeyaml", version = { strictly = "2.0" } } spock-core = { group = "org.spockframework", name="spock-core", version.ref="spock" } spock-junit4 = { group = "org.spockframework", name="spock-junit4", version.ref="spock" } diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 472a65f9c6f24..7270f12515b67 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -69,9 +69,9 @@ - - - + + + @@ -179,6 +179,11 @@ + + + + + @@ -214,6 +219,11 @@ + + + + + @@ -229,16 +239,31 @@ + + + + + + + + + + + + + + + @@ -409,21 +434,16 @@ + + + + + - - - - - - - - - - @@ -574,11 +594,6 @@ - - - - - @@ -614,6 +629,11 @@ + + + + + @@ -749,19 +769,14 @@ - - - + + + - - - - - - - - + + + @@ -799,6 +814,16 @@ + + + + + + + + + + @@ -919,9 +944,9 @@ - - - + + + @@ -1176,6 +1201,11 @@ + + + + + @@ -1196,6 +1226,11 @@ + + + + + @@ -1226,16 +1261,21 @@ - - - - - + + + + + + + + + + @@ -1301,6 +1341,11 @@ + + + + + @@ -1311,6 +1356,11 @@ + + + + + @@ -1366,9 +1416,9 @@ - - - + + + @@ -1376,9 +1426,9 @@ - - - + + + @@ -1386,29 +1436,29 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + @@ -1416,9 +1466,9 @@ - - - + + + @@ -1426,14 +1476,14 @@ - - - + + + - - - + + + @@ -1441,14 +1491,14 @@ - - - + + + - - - + + + @@ -1456,14 +1506,19 @@ + + + + + - - - + + + @@ -1836,6 +1891,27 @@ + + + + + + + + + + + + + + + + + + + + + @@ -1896,6 +1972,11 @@ + + + + + @@ -1911,6 +1992,11 @@ + + + + + @@ -2256,6 +2342,11 @@ + + + + + @@ -2286,14 +2377,14 @@ - - - + + + - - - + + + @@ -2328,9 +2419,9 @@ - - - + + + @@ -2784,6 +2875,11 @@ + + + + + @@ -2804,6 +2900,11 @@ + + + + + @@ -2814,124 +2915,144 @@ - - - + + + + + + + + + + + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + + + + + + + + + + + @@ -2949,6 +3070,11 @@ + + + + + @@ -2989,6 +3115,11 @@ + + + + + @@ -3014,6 +3145,11 @@ + + + + + @@ -3172,14 +3308,14 @@ - - - + + + - - - + + + @@ -3217,9 +3353,9 @@ - - - + + + @@ -3322,6 +3458,11 @@ + + + + + @@ -3337,6 +3478,16 @@ + + + + + + + + + + @@ -3362,6 +3513,16 @@ + + + + + + + + + + @@ -3392,11 +3553,21 @@ + + + + + + + + + + @@ -3502,6 +3673,11 @@ + + + + + @@ -3517,66 +3693,131 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -3587,11 +3828,21 @@ + + + + + + + + + + @@ -3622,6 +3873,11 @@ + + + + + @@ -3647,6 +3903,11 @@ + + + + + @@ -4117,9 +4378,9 @@ - - - + + + @@ -4137,9 +4398,9 @@ - - - + + + @@ -4162,9 +4423,9 @@ - - - + + + @@ -4202,9 +4463,9 @@ - - - + + + @@ -4212,11 +4473,21 @@ + + + + + + + + + + @@ -4377,11 +4648,21 @@ + + + + + + + + + + @@ -4430,6 +4711,136 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 9036682bf0f0c..e712035eabc7b 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=682b4df7fe5accdca84a4d1ef6a3a6ab096b3efd5edf7de2bd8c758d95a93703 -distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-all.zip +distributionSha256Sum=7ebdac923867a3cec0098302416d1e3c6c0c729fc4e2e05c10637a8af33a76c5 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.12-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index f5feea6d6b116..f3b75f3b0d4fa 100755 --- a/gradlew +++ b/gradlew @@ -86,8 +86,7 @@ done # shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) -APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s -' "$PWD" ) || exit +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum diff --git a/libs/build.gradle b/libs/build.gradle index afb82b5e63864..efd2329ca2b5e 100644 --- a/libs/build.gradle +++ b/libs/build.gradle @@ -1,15 +1,48 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ -configure(childProjects.values() - project('elasticsearch-log4j')) { +configure(childProjects.values()) { + + apply plugin: 'base' + /* - * All subprojects are java projects using Elasticsearch's standard build - * tools. + * Although these libs are local to Elasticsearch, they can conflict with other similarly + * named libraries when downloaded into a single directory via maven. Here we set the + * name of all libs to begin with the "elasticsearch-" prefix. Additionally, subprojects + * of libs begin with their parents artifactId. */ - apply plugin: 'elasticsearch.build' + def baseProject = project + def baseArtifactId = "elasticsearch-${it.name}" + base { + archivesName = baseArtifactId + } + subprojects { + apply plugin: 'base' + + def subArtifactId = baseArtifactId + def currentProject = project + while (currentProject != baseProject) { + subArtifactId += "-${currentProject.name}" + currentProject = currentProject.parent + } + base { + archivesName = subArtifactId + } + } + + // log4j is a hack, and not really a full elasticsearch built jar + if (project.name != 'log4j') { + + /* + * All subprojects are java projects using Elasticsearch's standard build + * tools. + */ + apply plugin: 'elasticsearch.build' + } } diff --git a/libs/cli/build.gradle b/libs/cli/build.gradle index dc045ba09e531..d5842d4a2c59c 100644 --- a/libs/cli/build.gradle +++ b/libs/cli/build.gradle @@ -1,19 +1,20 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.build' apply plugin: 'elasticsearch.publish' dependencies { api 'net.sf.jopt-simple:jopt-simple:5.0.2' - api project(':libs:elasticsearch-core') + api project(':libs:core') testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-cli' + exclude group: 'org.elasticsearch', module: 'cli' } } diff --git a/libs/cli/src/main/java/module-info.java b/libs/cli/src/main/java/module-info.java index 06bdd30371e46..e3969a1c74375 100644 --- a/libs/cli/src/main/java/module-info.java +++ b/libs/cli/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** Elasticsearch Command Line Interface. */ diff --git a/libs/cli/src/main/java/org/elasticsearch/cli/CliToolProvider.java b/libs/cli/src/main/java/org/elasticsearch/cli/CliToolProvider.java index d0523fadb0e23..99138f2321475 100644 --- a/libs/cli/src/main/java/org/elasticsearch/cli/CliToolProvider.java +++ b/libs/cli/src/main/java/org/elasticsearch/cli/CliToolProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli; diff --git a/libs/cli/src/main/java/org/elasticsearch/cli/Command.java b/libs/cli/src/main/java/org/elasticsearch/cli/Command.java index 32c4446e71dd2..6d38408ed165a 100644 --- a/libs/cli/src/main/java/org/elasticsearch/cli/Command.java +++ b/libs/cli/src/main/java/org/elasticsearch/cli/Command.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli; diff --git a/libs/cli/src/main/java/org/elasticsearch/cli/ExitCodes.java b/libs/cli/src/main/java/org/elasticsearch/cli/ExitCodes.java index f4d712f70f5d4..6c686e2bf9da8 100644 --- a/libs/cli/src/main/java/org/elasticsearch/cli/ExitCodes.java +++ b/libs/cli/src/main/java/org/elasticsearch/cli/ExitCodes.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli; diff --git a/libs/cli/src/main/java/org/elasticsearch/cli/MultiCommand.java b/libs/cli/src/main/java/org/elasticsearch/cli/MultiCommand.java index 3cc196d8f6316..47e12d8f2ac94 100644 --- a/libs/cli/src/main/java/org/elasticsearch/cli/MultiCommand.java +++ b/libs/cli/src/main/java/org/elasticsearch/cli/MultiCommand.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli; diff --git a/libs/cli/src/main/java/org/elasticsearch/cli/ProcessInfo.java b/libs/cli/src/main/java/org/elasticsearch/cli/ProcessInfo.java index 468c4c22a29e8..a109024bff593 100644 --- a/libs/cli/src/main/java/org/elasticsearch/cli/ProcessInfo.java +++ b/libs/cli/src/main/java/org/elasticsearch/cli/ProcessInfo.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli; diff --git a/libs/cli/src/main/java/org/elasticsearch/cli/Terminal.java b/libs/cli/src/main/java/org/elasticsearch/cli/Terminal.java index aaf233438f263..dcf16514f4a41 100644 --- a/libs/cli/src/main/java/org/elasticsearch/cli/Terminal.java +++ b/libs/cli/src/main/java/org/elasticsearch/cli/Terminal.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli; diff --git a/libs/cli/src/main/java/org/elasticsearch/cli/UserException.java b/libs/cli/src/main/java/org/elasticsearch/cli/UserException.java index b4a75267e1138..9123dfa79661a 100644 --- a/libs/cli/src/main/java/org/elasticsearch/cli/UserException.java +++ b/libs/cli/src/main/java/org/elasticsearch/cli/UserException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli; diff --git a/libs/cli/src/test/java/org/elasticsearch/cli/TerminalTests.java b/libs/cli/src/test/java/org/elasticsearch/cli/TerminalTests.java index dffb93ebbf230..d6fba5116536d 100644 --- a/libs/cli/src/test/java/org/elasticsearch/cli/TerminalTests.java +++ b/libs/cli/src/test/java/org/elasticsearch/cli/TerminalTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli; diff --git a/libs/core/build.gradle b/libs/core/build.gradle index a1d77cd02277e..99c22620e7354 100644 --- a/libs/core/build.gradle +++ b/libs/core/build.gradle @@ -1,30 +1,30 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.publish' -apply plugin: 'elasticsearch.mrjar' dependencies { // This dependency is used only by :libs:core for null-checking interop with other tools compileOnly "com.google.code.findbugs:jsr305:3.0.2" - compileOnly project(':libs:elasticsearch-logging') + compileOnly project(':libs:logging') testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" testImplementation "junit:junit:${versions.junit}" testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}" testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-core' + exclude group: 'org.elasticsearch', module: 'core' } } tasks.named('forbiddenApisMain').configure { - // :libs:elasticsearch-core does not depend on server + // :libs:core does not depend on server // TODO: Need to decide how we want to handle for forbidden signatures with the changes to server replaceSignatureFiles 'jdk-signatures' } diff --git a/libs/core/src/main/java/module-info.java b/libs/core/src/main/java/module-info.java index 21a0a9b2c646e..487ce69fad36b 100644 --- a/libs/core/src/main/java/module-info.java +++ b/libs/core/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.jdk.ModuleQualifiedExportsService; @@ -14,7 +15,11 @@ exports org.elasticsearch.core; exports org.elasticsearch.jdk; - exports org.elasticsearch.core.internal.provider to org.elasticsearch.xcontent, org.elasticsearch.nativeaccess; + exports org.elasticsearch.core.internal.provider + to + org.elasticsearch.xcontent, + org.elasticsearch.nativeaccess, + org.elasticsearch.entitlement; uses ModuleQualifiedExportsService; } diff --git a/libs/core/src/main/java/org/elasticsearch/core/AbstractRefCounted.java b/libs/core/src/main/java/org/elasticsearch/core/AbstractRefCounted.java index ca5704fa9866d..ef4184c28225d 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/AbstractRefCounted.java +++ b/libs/core/src/main/java/org/elasticsearch/core/AbstractRefCounted.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; @@ -45,6 +46,12 @@ public final void incRef() { } } + @Override + public final void mustIncRef() { + // making this implementation `final` (to be consistent with every other `RefCounted` method implementation) + RefCounted.super.mustIncRef(); + } + @Override public final boolean tryIncRef() { do { diff --git a/libs/core/src/main/java/org/elasticsearch/core/Assertions.java b/libs/core/src/main/java/org/elasticsearch/core/Assertions.java index ea5a4860af741..31a020afb8366 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/Assertions.java +++ b/libs/core/src/main/java/org/elasticsearch/core/Assertions.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/main/java/org/elasticsearch/core/Booleans.java b/libs/core/src/main/java/org/elasticsearch/core/Booleans.java index 062c613de5dc4..42579a39df0a2 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/Booleans.java +++ b/libs/core/src/main/java/org/elasticsearch/core/Booleans.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/main/java/org/elasticsearch/core/CharArrays.java b/libs/core/src/main/java/org/elasticsearch/core/CharArrays.java index eaa3615275346..750ea73863e6d 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/CharArrays.java +++ b/libs/core/src/main/java/org/elasticsearch/core/CharArrays.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/main/java/org/elasticsearch/core/CheckedConsumer.java b/libs/core/src/main/java/org/elasticsearch/core/CheckedConsumer.java index 56325dc21bb4a..ef7e0c785c2a1 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/CheckedConsumer.java +++ b/libs/core/src/main/java/org/elasticsearch/core/CheckedConsumer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/main/java/org/elasticsearch/core/CheckedFunction.java b/libs/core/src/main/java/org/elasticsearch/core/CheckedFunction.java index 66a142d7d8e85..2ad4b95b419a9 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/CheckedFunction.java +++ b/libs/core/src/main/java/org/elasticsearch/core/CheckedFunction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/main/java/org/elasticsearch/core/CheckedRunnable.java b/libs/core/src/main/java/org/elasticsearch/core/CheckedRunnable.java index 4b9ac273f2943..70b56446c108e 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/CheckedRunnable.java +++ b/libs/core/src/main/java/org/elasticsearch/core/CheckedRunnable.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/main/java/org/elasticsearch/core/ESSloppyMath.java b/libs/core/src/main/java/org/elasticsearch/core/ESSloppyMath.java index 03acc6e1bc2d0..0092cb4cd8092 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/ESSloppyMath.java +++ b/libs/core/src/main/java/org/elasticsearch/core/ESSloppyMath.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/main/java/org/elasticsearch/core/Glob.java b/libs/core/src/main/java/org/elasticsearch/core/Glob.java index 179862716cdc3..8bc1f23c44bb5 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/Glob.java +++ b/libs/core/src/main/java/org/elasticsearch/core/Glob.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/main/java/org/elasticsearch/core/Nullable.java b/libs/core/src/main/java/org/elasticsearch/core/Nullable.java index 3194f1a2e668c..e7c181348e7fe 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/Nullable.java +++ b/libs/core/src/main/java/org/elasticsearch/core/Nullable.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/main/java/org/elasticsearch/core/PathUtils.java b/libs/core/src/main/java/org/elasticsearch/core/PathUtils.java index 71d89c5bf063b..67abe03f5a6ff 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/PathUtils.java +++ b/libs/core/src/main/java/org/elasticsearch/core/PathUtils.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/main/java/org/elasticsearch/core/Predicates.java b/libs/core/src/main/java/org/elasticsearch/core/Predicates.java index 47ac9ef258d68..bd8c15173231e 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/Predicates.java +++ b/libs/core/src/main/java/org/elasticsearch/core/Predicates.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/main/java/org/elasticsearch/core/RefCounted.java b/libs/core/src/main/java/org/elasticsearch/core/RefCounted.java index 865b1afb5d4fe..916760d3438fe 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/RefCounted.java +++ b/libs/core/src/main/java/org/elasticsearch/core/RefCounted.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/main/java/org/elasticsearch/core/Releasable.java b/libs/core/src/main/java/org/elasticsearch/core/Releasable.java index 13da8cf720d77..a99bea8655839 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/Releasable.java +++ b/libs/core/src/main/java/org/elasticsearch/core/Releasable.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/main/java/org/elasticsearch/core/ReleasableIterator.java b/libs/core/src/main/java/org/elasticsearch/core/ReleasableIterator.java index 83a68c984a684..2eb8e6a0e1247 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/ReleasableIterator.java +++ b/libs/core/src/main/java/org/elasticsearch/core/ReleasableIterator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/main/java/org/elasticsearch/core/Releasables.java b/libs/core/src/main/java/org/elasticsearch/core/Releasables.java index 45d98b7761110..96c16139773a1 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/Releasables.java +++ b/libs/core/src/main/java/org/elasticsearch/core/Releasables.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/main/java/org/elasticsearch/core/RestApiVersion.java b/libs/core/src/main/java/org/elasticsearch/core/RestApiVersion.java index 74acb00925e5a..b08d32a9e8ee1 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/RestApiVersion.java +++ b/libs/core/src/main/java/org/elasticsearch/core/RestApiVersion.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/main/java/org/elasticsearch/core/SimpleRefCounted.java b/libs/core/src/main/java/org/elasticsearch/core/SimpleRefCounted.java index 367133641d464..bed6d76551473 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/SimpleRefCounted.java +++ b/libs/core/src/main/java/org/elasticsearch/core/SimpleRefCounted.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/main/java/org/elasticsearch/core/Streams.java b/libs/core/src/main/java/org/elasticsearch/core/Streams.java index f28a8615c82f6..f964df150e988 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/Streams.java +++ b/libs/core/src/main/java/org/elasticsearch/core/Streams.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/main/java/org/elasticsearch/core/Strings.java b/libs/core/src/main/java/org/elasticsearch/core/Strings.java index 4ea0c64ce9d52..ed8dbc9cdbdb6 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/Strings.java +++ b/libs/core/src/main/java/org/elasticsearch/core/Strings.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/main/java/org/elasticsearch/core/SuppressForbidden.java b/libs/core/src/main/java/org/elasticsearch/core/SuppressForbidden.java index 9a7344d08ac2d..8dbcbe6ab3d5b 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/SuppressForbidden.java +++ b/libs/core/src/main/java/org/elasticsearch/core/SuppressForbidden.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/main/java/org/elasticsearch/core/TimeValue.java b/libs/core/src/main/java/org/elasticsearch/core/TimeValue.java index 26d93bca6b09a..89c2494cd128c 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/TimeValue.java +++ b/libs/core/src/main/java/org/elasticsearch/core/TimeValue.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/main/java/org/elasticsearch/core/Tuple.java b/libs/core/src/main/java/org/elasticsearch/core/Tuple.java index cef4f07955274..5a741d26ce2c5 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/Tuple.java +++ b/libs/core/src/main/java/org/elasticsearch/core/Tuple.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/main/java/org/elasticsearch/core/Types.java b/libs/core/src/main/java/org/elasticsearch/core/Types.java index 2956425ede962..1ee4cae027456 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/Types.java +++ b/libs/core/src/main/java/org/elasticsearch/core/Types.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/main/java/org/elasticsearch/core/UpdateForV10.java b/libs/core/src/main/java/org/elasticsearch/core/UpdateForV10.java index 0fe816bd3721d..e4196f6b7c99c 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/UpdateForV10.java +++ b/libs/core/src/main/java/org/elasticsearch/core/UpdateForV10.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/main/java/org/elasticsearch/core/UpdateForV9.java b/libs/core/src/main/java/org/elasticsearch/core/UpdateForV9.java index 2a31e2ccde222..2d0ed12467bb9 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/UpdateForV9.java +++ b/libs/core/src/main/java/org/elasticsearch/core/UpdateForV9.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/main/java/org/elasticsearch/core/internal/provider/EmbeddedImplClassLoader.java b/libs/core/src/main/java/org/elasticsearch/core/internal/provider/EmbeddedImplClassLoader.java index 049b619fb57ac..751c5146b484a 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/internal/provider/EmbeddedImplClassLoader.java +++ b/libs/core/src/main/java/org/elasticsearch/core/internal/provider/EmbeddedImplClassLoader.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core.internal.provider; diff --git a/libs/core/src/main/java/org/elasticsearch/core/internal/provider/EmbeddedModulePath.java b/libs/core/src/main/java/org/elasticsearch/core/internal/provider/EmbeddedModulePath.java index abf29c3c43656..a9dc6881fdf33 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/internal/provider/EmbeddedModulePath.java +++ b/libs/core/src/main/java/org/elasticsearch/core/internal/provider/EmbeddedModulePath.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core.internal.provider; diff --git a/libs/core/src/main/java/org/elasticsearch/core/internal/provider/InMemoryModuleFinder.java b/libs/core/src/main/java/org/elasticsearch/core/internal/provider/InMemoryModuleFinder.java index f9c399e9522b3..c727a5db1e886 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/internal/provider/InMemoryModuleFinder.java +++ b/libs/core/src/main/java/org/elasticsearch/core/internal/provider/InMemoryModuleFinder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core.internal.provider; diff --git a/libs/core/src/main/java/org/elasticsearch/core/internal/provider/ProviderLocator.java b/libs/core/src/main/java/org/elasticsearch/core/internal/provider/ProviderLocator.java index b0e3df81ed28c..902c61402c058 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/internal/provider/ProviderLocator.java +++ b/libs/core/src/main/java/org/elasticsearch/core/internal/provider/ProviderLocator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core.internal.provider; diff --git a/libs/core/src/main/java/org/elasticsearch/jdk/JarHell.java b/libs/core/src/main/java/org/elasticsearch/jdk/JarHell.java index 6abbfec2297ea..95197c6dae0dc 100644 --- a/libs/core/src/main/java/org/elasticsearch/jdk/JarHell.java +++ b/libs/core/src/main/java/org/elasticsearch/jdk/JarHell.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.jdk; diff --git a/libs/core/src/main/java/org/elasticsearch/jdk/JdkJarHellCheck.java b/libs/core/src/main/java/org/elasticsearch/jdk/JdkJarHellCheck.java index 444d390a5b861..7e331ea25ac4d 100644 --- a/libs/core/src/main/java/org/elasticsearch/jdk/JdkJarHellCheck.java +++ b/libs/core/src/main/java/org/elasticsearch/jdk/JdkJarHellCheck.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.jdk; diff --git a/libs/core/src/main/java/org/elasticsearch/jdk/ModuleQualifiedExportsService.java b/libs/core/src/main/java/org/elasticsearch/jdk/ModuleQualifiedExportsService.java index 9e255b109685f..f0bd9568e194f 100644 --- a/libs/core/src/main/java/org/elasticsearch/jdk/ModuleQualifiedExportsService.java +++ b/libs/core/src/main/java/org/elasticsearch/jdk/ModuleQualifiedExportsService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.jdk; diff --git a/libs/core/src/test/java/org/elasticsearch/common/CharArraysTests.java b/libs/core/src/test/java/org/elasticsearch/common/CharArraysTests.java index d7e64bb324cce..e53bbc098e89c 100644 --- a/libs/core/src/test/java/org/elasticsearch/common/CharArraysTests.java +++ b/libs/core/src/test/java/org/elasticsearch/common/CharArraysTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common; diff --git a/libs/core/src/test/java/org/elasticsearch/common/collect/TupleTests.java b/libs/core/src/test/java/org/elasticsearch/common/collect/TupleTests.java index 962f519f93a93..af52b0b440edb 100644 --- a/libs/core/src/test/java/org/elasticsearch/common/collect/TupleTests.java +++ b/libs/core/src/test/java/org/elasticsearch/common/collect/TupleTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.collect; diff --git a/libs/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java b/libs/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java index dd2755ac1f9f7..435853ee9e789 100644 --- a/libs/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java +++ b/libs/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.unit; diff --git a/libs/core/src/test/java/org/elasticsearch/common/util/ESSloppyMathTests.java b/libs/core/src/test/java/org/elasticsearch/common/util/ESSloppyMathTests.java index d8ad88a536dbd..7d6c60833cb82 100644 --- a/libs/core/src/test/java/org/elasticsearch/common/util/ESSloppyMathTests.java +++ b/libs/core/src/test/java/org/elasticsearch/common/util/ESSloppyMathTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.util; diff --git a/libs/core/src/test/java/org/elasticsearch/core/AbstractRefCountedTests.java b/libs/core/src/test/java/org/elasticsearch/core/AbstractRefCountedTests.java index 74dcd19248834..a8b5d2fdaa650 100644 --- a/libs/core/src/test/java/org/elasticsearch/core/AbstractRefCountedTests.java +++ b/libs/core/src/test/java/org/elasticsearch/core/AbstractRefCountedTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/test/java/org/elasticsearch/core/ReleasablesTests.java b/libs/core/src/test/java/org/elasticsearch/core/ReleasablesTests.java index 602437c27bd49..e1bb3b0e3c4fc 100644 --- a/libs/core/src/test/java/org/elasticsearch/core/ReleasablesTests.java +++ b/libs/core/src/test/java/org/elasticsearch/core/ReleasablesTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/test/java/org/elasticsearch/core/StreamsTests.java b/libs/core/src/test/java/org/elasticsearch/core/StreamsTests.java index 668355b2739cf..79baa65b6bdb1 100644 --- a/libs/core/src/test/java/org/elasticsearch/core/StreamsTests.java +++ b/libs/core/src/test/java/org/elasticsearch/core/StreamsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/test/java/org/elasticsearch/core/StringsTests.java b/libs/core/src/test/java/org/elasticsearch/core/StringsTests.java index 74838643ad032..99e23adcc5288 100644 --- a/libs/core/src/test/java/org/elasticsearch/core/StringsTests.java +++ b/libs/core/src/test/java/org/elasticsearch/core/StringsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core; diff --git a/libs/core/src/test/java/org/elasticsearch/core/internal/provider/EmbeddedImplClassLoaderTests.java b/libs/core/src/test/java/org/elasticsearch/core/internal/provider/EmbeddedImplClassLoaderTests.java index db517b9cfabdc..34c8ed1c6d851 100644 --- a/libs/core/src/test/java/org/elasticsearch/core/internal/provider/EmbeddedImplClassLoaderTests.java +++ b/libs/core/src/test/java/org/elasticsearch/core/internal/provider/EmbeddedImplClassLoaderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core.internal.provider; diff --git a/libs/core/src/test/java/org/elasticsearch/core/internal/provider/EmbeddedModulePathTests.java b/libs/core/src/test/java/org/elasticsearch/core/internal/provider/EmbeddedModulePathTests.java index 4571591bd6649..7c84a59797458 100644 --- a/libs/core/src/test/java/org/elasticsearch/core/internal/provider/EmbeddedModulePathTests.java +++ b/libs/core/src/test/java/org/elasticsearch/core/internal/provider/EmbeddedModulePathTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core.internal.provider; diff --git a/libs/core/src/test/java/org/elasticsearch/core/internal/provider/InMemoryModuleFinderTests.java b/libs/core/src/test/java/org/elasticsearch/core/internal/provider/InMemoryModuleFinderTests.java index 361b9ea7fae0c..debf52a04ec51 100644 --- a/libs/core/src/test/java/org/elasticsearch/core/internal/provider/InMemoryModuleFinderTests.java +++ b/libs/core/src/test/java/org/elasticsearch/core/internal/provider/InMemoryModuleFinderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core.internal.provider; diff --git a/libs/core/src/test/java/org/elasticsearch/core/internal/provider/ProviderLocatorTests.java b/libs/core/src/test/java/org/elasticsearch/core/internal/provider/ProviderLocatorTests.java index f75f7ddad50ed..61d4a7c6bd700 100644 --- a/libs/core/src/test/java/org/elasticsearch/core/internal/provider/ProviderLocatorTests.java +++ b/libs/core/src/test/java/org/elasticsearch/core/internal/provider/ProviderLocatorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.core.internal.provider; diff --git a/libs/core/src/test/java/org/elasticsearch/jdk/JarHellTests.java b/libs/core/src/test/java/org/elasticsearch/jdk/JarHellTests.java index 6bcc1d4b8fb58..62d37106f6e47 100644 --- a/libs/core/src/test/java/org/elasticsearch/jdk/JarHellTests.java +++ b/libs/core/src/test/java/org/elasticsearch/jdk/JarHellTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.jdk; diff --git a/libs/dissect/build.gradle b/libs/dissect/build.gradle index e10e665d903dd..f1a09cc0ba0e6 100644 --- a/libs/dissect/build.gradle +++ b/libs/dissect/build.gradle @@ -1,14 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ dependencies { testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-dissect' + exclude group: 'org.elasticsearch', module: 'dissect' } testImplementation "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" testImplementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" diff --git a/libs/dissect/src/main/java/module-info.java b/libs/dissect/src/main/java/module-info.java index 6f4768592dad2..8b282e940927e 100644 --- a/libs/dissect/src/main/java/module-info.java +++ b/libs/dissect/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.dissect { diff --git a/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectException.java b/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectException.java index a47268028e952..d7a9707f12d1e 100644 --- a/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectException.java +++ b/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.dissect; diff --git a/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectKey.java b/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectKey.java index f6cf21e478bc1..bc0a51614e5b8 100644 --- a/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectKey.java +++ b/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectKey.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.dissect; diff --git a/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectMatch.java b/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectMatch.java index 5714870ab8520..2b5c21d4b3bb3 100644 --- a/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectMatch.java +++ b/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectMatch.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.dissect; diff --git a/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectParser.java b/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectParser.java index 3c01e490369de..f841cc0464fdf 100644 --- a/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectParser.java +++ b/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.dissect; diff --git a/libs/dissect/src/test/java/org/elasticsearch/dissect/DissectKeyTests.java b/libs/dissect/src/test/java/org/elasticsearch/dissect/DissectKeyTests.java index 631f525fdfe30..e2ec66bb59c23 100644 --- a/libs/dissect/src/test/java/org/elasticsearch/dissect/DissectKeyTests.java +++ b/libs/dissect/src/test/java/org/elasticsearch/dissect/DissectKeyTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.dissect; diff --git a/libs/dissect/src/test/java/org/elasticsearch/dissect/DissectMatchTests.java b/libs/dissect/src/test/java/org/elasticsearch/dissect/DissectMatchTests.java index 5e80ff5676169..a01d23d79186b 100644 --- a/libs/dissect/src/test/java/org/elasticsearch/dissect/DissectMatchTests.java +++ b/libs/dissect/src/test/java/org/elasticsearch/dissect/DissectMatchTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.dissect; diff --git a/libs/dissect/src/test/java/org/elasticsearch/dissect/DissectParserTests.java b/libs/dissect/src/test/java/org/elasticsearch/dissect/DissectParserTests.java index 2893e419a84a3..a9cc309a6a33a 100644 --- a/libs/dissect/src/test/java/org/elasticsearch/dissect/DissectParserTests.java +++ b/libs/dissect/src/test/java/org/elasticsearch/dissect/DissectParserTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.dissect; diff --git a/libs/entitlement/README.md b/libs/entitlement/README.md new file mode 100644 index 0000000000000..2ab76cf1c2221 --- /dev/null +++ b/libs/entitlement/README.md @@ -0,0 +1,11 @@ +### Entitlement library + +This module implements mechanisms to grant and check permissions under the _entitlements_ system. + +The entitlements system provides an alternative to the legacy `SecurityManager` system, which is deprecated for removal. +The `entitlement-agent` instruments sensitive class library methods with calls to this module, in order to enforce the controls. + +This feature is currently under development, and it is completely disabled by default (the agent is not loaded). To enable it, run Elasticsearch with +```shell +./gradlew run --entitlements +``` diff --git a/libs/entitlement/agent/README.md b/libs/entitlement/agent/README.md new file mode 100644 index 0000000000000..0dc331266cc94 --- /dev/null +++ b/libs/entitlement/agent/README.md @@ -0,0 +1,10 @@ +### Entitlement Agent + +This is a java agent that instruments sensitive class library methods with calls into the `entitlement-bridge` module to check for permissions granted under the _entitlements_ system. + +The entitlements system provides an alternative to the legacy `SecurityManager` system, which is deprecated for removal. +With this agent, the Elasticsearch server can retain some control over which class library methods can be invoked by which callers. + +This module is responsible for inserting the appropriate bytecode to achieve enforcement of the rules governed by the main `entitlement` module. + +It is not responsible for permission granting or checking logic. That responsibility lies with the main `entitlement` module. diff --git a/libs/entitlement/agent/build.gradle b/libs/entitlement/agent/build.gradle new file mode 100644 index 0000000000000..5976c2f64af5f --- /dev/null +++ b/libs/entitlement/agent/build.gradle @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +apply plugin: 'elasticsearch.build' + +dependencies { + compileOnly project(":libs:core") + compileOnly project(":libs:entitlement") + compileOnly project(":libs:entitlement:bridge") +} + +tasks.named('jar').configure { + manifest { + attributes( + 'Agent-Class': 'org.elasticsearch.entitlement.agent.EntitlementAgent' + , 'Can-Retransform-Classes': 'true' + ) + } +} + +tasks.named('forbiddenApisMain').configure { + replaceSignatureFiles 'jdk-signatures' +} + diff --git a/libs/entitlement/agent/src/main/java/org/elasticsearch/entitlement/agent/EntitlementAgent.java b/libs/entitlement/agent/src/main/java/org/elasticsearch/entitlement/agent/EntitlementAgent.java new file mode 100644 index 0000000000000..d062ba1ed2465 --- /dev/null +++ b/libs/entitlement/agent/src/main/java/org/elasticsearch/entitlement/agent/EntitlementAgent.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.agent; + +import java.lang.instrument.Instrumentation; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + +/** + * A Java Agent that sets up the bytecode instrumentation for the entitlement system. + *

    + * Agents are loaded into the unnamed module, which makes module exports awkward. + * To work around this, we keep minimal code in the agent itself, and + * instead use reflection to call into the main entitlement library, + * which bootstraps by using {@link Module#addExports} to make a single {@code initialize} + * method available for us to call from here. + * That method does the rest. + */ +public class EntitlementAgent { + + public static void agentmain(String agentArgs, Instrumentation inst) { + final Class initClazz; + try { + initClazz = Class.forName("org.elasticsearch.entitlement.initialization.EntitlementInitialization"); + } catch (ClassNotFoundException e) { + throw new AssertionError("entitlement agent does could not find EntitlementInitialization", e); + } + + final Method initMethod; + try { + initMethod = initClazz.getMethod("initialize", Instrumentation.class); + } catch (NoSuchMethodException e) { + throw new AssertionError("EntitlementInitialization missing initialize method", e); + } + + try { + initMethod.invoke(null, inst); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new AssertionError("entitlement initialization failed", e); + } + } +} diff --git a/libs/entitlement/asm-provider/README.md b/libs/entitlement/asm-provider/README.md new file mode 100644 index 0000000000000..5343e56c2aa02 --- /dev/null +++ b/libs/entitlement/asm-provider/README.md @@ -0,0 +1,2 @@ +This module uses the ASM library to implement various things, including bytecode instrumentation. +It is loaded using the Embedded Provider Gradle plugin. diff --git a/libs/entitlement/asm-provider/build.gradle b/libs/entitlement/asm-provider/build.gradle new file mode 100644 index 0000000000000..dcec0579a5bae --- /dev/null +++ b/libs/entitlement/asm-provider/build.gradle @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +apply plugin: 'elasticsearch.build' + +dependencies { + compileOnly project(':libs:entitlement') + implementation 'org.ow2.asm:asm:9.7.1' + testImplementation project(":test:framework") + testImplementation project(":libs:entitlement:bridge") + testImplementation 'org.ow2.asm:asm-util:9.7.1' +} + +tasks.named('test').configure { + systemProperty "tests.security.manager", "false" +} + +tasks.named('forbiddenApisMain').configure { + replaceSignatureFiles 'jdk-signatures' +} + diff --git a/libs/entitlement/asm-provider/licenses/asm-LICENSE.txt b/libs/entitlement/asm-provider/licenses/asm-LICENSE.txt new file mode 100644 index 0000000000000..afb064f2f2666 --- /dev/null +++ b/libs/entitlement/asm-provider/licenses/asm-LICENSE.txt @@ -0,0 +1,26 @@ +Copyright (c) 2012 France Télécom +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. diff --git a/x-pack/plugin/inference/licenses/jaxb-NOTICE.txt b/libs/entitlement/asm-provider/licenses/asm-NOTICE.txt similarity index 100% rename from x-pack/plugin/inference/licenses/jaxb-NOTICE.txt rename to libs/entitlement/asm-provider/licenses/asm-NOTICE.txt diff --git a/libs/entitlement/asm-provider/src/main/java/module-info.java b/libs/entitlement/asm-provider/src/main/java/module-info.java new file mode 100644 index 0000000000000..8cbeafc9013aa --- /dev/null +++ b/libs/entitlement/asm-provider/src/main/java/module-info.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import org.elasticsearch.entitlement.instrumentation.InstrumentationService; +import org.elasticsearch.entitlement.instrumentation.impl.InstrumentationServiceImpl; + +module org.elasticsearch.entitlement.instrumentation { + requires org.objectweb.asm; + requires org.elasticsearch.entitlement; + + provides InstrumentationService with InstrumentationServiceImpl; +} diff --git a/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImpl.java b/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImpl.java new file mode 100644 index 0000000000000..0bcbc19047c80 --- /dev/null +++ b/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImpl.java @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.instrumentation.impl; + +import org.elasticsearch.entitlement.instrumentation.CheckMethod; +import org.elasticsearch.entitlement.instrumentation.InstrumentationService; +import org.elasticsearch.entitlement.instrumentation.Instrumenter; +import org.elasticsearch.entitlement.instrumentation.MethodKey; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.Type; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +public class InstrumentationServiceImpl implements InstrumentationService { + + @Override + public Instrumenter newInstrumenter(Class clazz, Map methods) { + return InstrumenterImpl.create(clazz, methods); + } + + @Override + public Map lookupMethods(Class checkerClass) throws IOException { + var methodsToInstrument = new HashMap(); + var classFileInfo = InstrumenterImpl.getClassFileInfo(checkerClass); + ClassReader reader = new ClassReader(classFileInfo.bytecodes()); + ClassVisitor visitor = new ClassVisitor(Opcodes.ASM9) { + @Override + public MethodVisitor visitMethod( + int access, + String checkerMethodName, + String checkerMethodDescriptor, + String signature, + String[] exceptions + ) { + var mv = super.visitMethod(access, checkerMethodName, checkerMethodDescriptor, signature, exceptions); + + var checkerMethodArgumentTypes = Type.getArgumentTypes(checkerMethodDescriptor); + var methodToInstrument = parseCheckerMethodSignature(checkerMethodName, checkerMethodArgumentTypes); + + var checkerParameterDescriptors = Arrays.stream(checkerMethodArgumentTypes).map(Type::getDescriptor).toList(); + var checkMethod = new CheckMethod(Type.getInternalName(checkerClass), checkerMethodName, checkerParameterDescriptors); + + methodsToInstrument.put(methodToInstrument, checkMethod); + + return mv; + } + }; + reader.accept(visitor, 0); + return methodsToInstrument; + } + + private static final Type CLASS_TYPE = Type.getType(Class.class); + + static MethodKey parseCheckerMethodSignature(String checkerMethodName, Type[] checkerMethodArgumentTypes) { + boolean targetMethodIsStatic; + int classNameEndIndex = checkerMethodName.lastIndexOf("$$"); + int methodNameStartIndex; + if (classNameEndIndex == -1) { + targetMethodIsStatic = false; + classNameEndIndex = checkerMethodName.lastIndexOf('$'); + methodNameStartIndex = classNameEndIndex + 1; + } else { + targetMethodIsStatic = true; + methodNameStartIndex = classNameEndIndex + 2; + } + + var classNameStartIndex = checkerMethodName.indexOf('$'); + if (classNameStartIndex == -1 || classNameStartIndex >= classNameEndIndex) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Checker method %s has incorrect name format. " + + "It should be either check$$methodName (instance), check$package_ClassName$methodName (static) or " + + "check$package_ClassName$ (ctor)", + checkerMethodName + ) + ); + } + + // No "methodName" (check$package_ClassName$) -> method is ctor + final boolean targetMethodIsCtor = classNameEndIndex + 1 == checkerMethodName.length(); + final String targetMethodName = targetMethodIsCtor ? "" : checkerMethodName.substring(methodNameStartIndex); + + final String targetClassName = checkerMethodName.substring(classNameStartIndex + 1, classNameEndIndex).replace('_', '/'); + if (targetClassName.isBlank()) { + throw new IllegalArgumentException(String.format(Locale.ROOT, "Checker method %s has no class name", checkerMethodName)); + } + + final List targetParameterTypes; + if (targetMethodIsStatic || targetMethodIsCtor) { + if (checkerMethodArgumentTypes.length < 1 || CLASS_TYPE.equals(checkerMethodArgumentTypes[0]) == false) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Checker method %s has incorrect argument types. " + "It must have a first argument of Class type.", + checkerMethodName + ) + ); + } + + targetParameterTypes = Arrays.stream(checkerMethodArgumentTypes).skip(1).map(Type::getInternalName).toList(); + } else { + if (checkerMethodArgumentTypes.length < 2 + || CLASS_TYPE.equals(checkerMethodArgumentTypes[0]) == false + || checkerMethodArgumentTypes[1].getSort() != Type.OBJECT) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Checker method %s has incorrect argument types. " + + "It must have a first argument of Class type, and a second argument of the class containing the method to " + + "instrument", + checkerMethodName + ) + ); + } + targetParameterTypes = Arrays.stream(checkerMethodArgumentTypes).skip(2).map(Type::getInternalName).toList(); + } + boolean hasReceiver = (targetMethodIsStatic || targetMethodIsCtor) == false; + return new MethodKey(targetClassName, targetMethodName, targetParameterTypes); + } +} diff --git a/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterImpl.java b/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterImpl.java new file mode 100644 index 0000000000000..06408941ac96e --- /dev/null +++ b/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterImpl.java @@ -0,0 +1,295 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.instrumentation.impl; + +import org.elasticsearch.entitlement.instrumentation.CheckMethod; +import org.elasticsearch.entitlement.instrumentation.Instrumenter; +import org.elasticsearch.entitlement.instrumentation.MethodKey; +import org.objectweb.asm.AnnotationVisitor; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.FieldVisitor; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.RecordComponentVisitor; +import org.objectweb.asm.Type; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Map; +import java.util.stream.Stream; + +import static org.objectweb.asm.ClassWriter.COMPUTE_FRAMES; +import static org.objectweb.asm.ClassWriter.COMPUTE_MAXS; +import static org.objectweb.asm.Opcodes.ACC_STATIC; +import static org.objectweb.asm.Opcodes.GETSTATIC; +import static org.objectweb.asm.Opcodes.INVOKEINTERFACE; +import static org.objectweb.asm.Opcodes.INVOKESTATIC; +import static org.objectweb.asm.Opcodes.INVOKEVIRTUAL; + +public class InstrumenterImpl implements Instrumenter { + + private final String getCheckerClassMethodDescriptor; + private final String handleClass; + + /** + * To avoid class name collisions during testing without an agent to replace classes in-place. + */ + private final String classNameSuffix; + private final Map checkMethods; + + InstrumenterImpl( + String handleClass, + String getCheckerClassMethodDescriptor, + String classNameSuffix, + Map checkMethods + ) { + this.handleClass = handleClass; + this.getCheckerClassMethodDescriptor = getCheckerClassMethodDescriptor; + this.classNameSuffix = classNameSuffix; + this.checkMethods = checkMethods; + } + + public static InstrumenterImpl create(Class checkerClass, Map checkMethods) { + Type checkerClassType = Type.getType(checkerClass); + String handleClass = checkerClassType.getInternalName() + "Handle"; + String getCheckerClassMethodDescriptor = Type.getMethodDescriptor(checkerClassType); + return new InstrumenterImpl(handleClass, getCheckerClassMethodDescriptor, "", checkMethods); + } + + static ClassFileInfo getClassFileInfo(Class clazz) throws IOException { + String internalName = Type.getInternalName(clazz); + String fileName = "/" + internalName + ".class"; + byte[] originalBytecodes; + try (InputStream classStream = clazz.getResourceAsStream(fileName)) { + if (classStream == null) { + throw new IllegalStateException("Classfile not found in jar: " + fileName); + } + originalBytecodes = classStream.readAllBytes(); + } + return new ClassFileInfo(fileName, originalBytecodes); + } + + @Override + public byte[] instrumentClass(String className, byte[] classfileBuffer) { + ClassReader reader = new ClassReader(classfileBuffer); + ClassWriter writer = new ClassWriter(reader, COMPUTE_FRAMES | COMPUTE_MAXS); + ClassVisitor visitor = new EntitlementClassVisitor(Opcodes.ASM9, writer, className); + reader.accept(visitor, 0); + return writer.toByteArray(); + } + + class EntitlementClassVisitor extends ClassVisitor { + + private static final String ENTITLEMENT_ANNOTATION = "EntitlementInstrumented"; + + private final String className; + + private boolean isAnnotationPresent; + private boolean annotationNeeded = true; + + EntitlementClassVisitor(int api, ClassVisitor classVisitor, String className) { + super(api, classVisitor); + this.className = className; + } + + @Override + public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) { + super.visit(version, access, name + classNameSuffix, signature, superName, interfaces); + } + + @Override + public AnnotationVisitor visitAnnotation(String descriptor, boolean visible) { + if (visible && descriptor.equals(ENTITLEMENT_ANNOTATION)) { + isAnnotationPresent = true; + annotationNeeded = false; + } + return cv.visitAnnotation(descriptor, visible); + } + + @Override + public void visitNestMember(String nestMember) { + addClassAnnotationIfNeeded(); + super.visitNestMember(nestMember); + } + + @Override + public void visitPermittedSubclass(String permittedSubclass) { + addClassAnnotationIfNeeded(); + super.visitPermittedSubclass(permittedSubclass); + } + + @Override + public void visitInnerClass(String name, String outerName, String innerName, int access) { + addClassAnnotationIfNeeded(); + super.visitInnerClass(name, outerName, innerName, access); + } + + @Override + public FieldVisitor visitField(int access, String name, String descriptor, String signature, Object value) { + addClassAnnotationIfNeeded(); + return super.visitField(access, name, descriptor, signature, value); + } + + @Override + public RecordComponentVisitor visitRecordComponent(String name, String descriptor, String signature) { + addClassAnnotationIfNeeded(); + return super.visitRecordComponent(name, descriptor, signature); + } + + @Override + public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { + addClassAnnotationIfNeeded(); + var mv = super.visitMethod(access, name, descriptor, signature, exceptions); + if (isAnnotationPresent == false) { + boolean isStatic = (access & ACC_STATIC) != 0; + boolean isCtor = "".equals(name); + boolean hasReceiver = (isStatic || isCtor) == false; + var key = new MethodKey(className, name, Stream.of(Type.getArgumentTypes(descriptor)).map(Type::getInternalName).toList()); + var instrumentationMethod = checkMethods.get(key); + if (instrumentationMethod != null) { + // LOGGER.debug("Will instrument method {}", key); + return new EntitlementMethodVisitor(Opcodes.ASM9, mv, isStatic, isCtor, descriptor, instrumentationMethod); + } else { + // LOGGER.trace("Will not instrument method {}", key); + } + } + return mv; + } + + /** + * A class annotation can be added via visitAnnotation; we need to call visitAnnotation after all other visitAnnotation + * calls (in case one of them detects our annotation is already present), but before any other subsequent visit* method is called + * (up to visitMethod -- if no visitMethod is called, there is nothing to instrument). + * This includes visitNestMember, visitPermittedSubclass, visitInnerClass, visitField, visitRecordComponent and, of course, + * visitMethod (see {@link ClassVisitor} javadoc). + */ + private void addClassAnnotationIfNeeded() { + if (annotationNeeded) { + // logger.debug("Adding {} annotation", ENTITLEMENT_ANNOTATION); + AnnotationVisitor av = cv.visitAnnotation(ENTITLEMENT_ANNOTATION, true); + if (av != null) { + av.visitEnd(); + } + annotationNeeded = false; + } + } + } + + class EntitlementMethodVisitor extends MethodVisitor { + private final boolean instrumentedMethodIsStatic; + private final boolean instrumentedMethodIsCtor; + private final String instrumentedMethodDescriptor; + private final CheckMethod checkMethod; + private boolean hasCallerSensitiveAnnotation = false; + + EntitlementMethodVisitor( + int api, + MethodVisitor methodVisitor, + boolean instrumentedMethodIsStatic, + boolean instrumentedMethodIsCtor, + String instrumentedMethodDescriptor, + CheckMethod checkMethod + ) { + super(api, methodVisitor); + this.instrumentedMethodIsStatic = instrumentedMethodIsStatic; + this.instrumentedMethodIsCtor = instrumentedMethodIsCtor; + this.instrumentedMethodDescriptor = instrumentedMethodDescriptor; + this.checkMethod = checkMethod; + } + + @Override + public AnnotationVisitor visitAnnotation(String descriptor, boolean visible) { + if (visible && descriptor.endsWith("CallerSensitive;")) { + hasCallerSensitiveAnnotation = true; + } + return super.visitAnnotation(descriptor, visible); + } + + @Override + public void visitCode() { + pushEntitlementChecker(); + pushCallerClass(); + forwardIncomingArguments(); + invokeInstrumentationMethod(); + super.visitCode(); + } + + private void pushEntitlementChecker() { + InstrumenterImpl.this.pushEntitlementChecker(mv); + } + + private void pushCallerClass() { + if (hasCallerSensitiveAnnotation) { + mv.visitMethodInsn( + INVOKESTATIC, + "jdk/internal/reflect/Reflection", + "getCallerClass", + Type.getMethodDescriptor(Type.getType(Class.class)), + false + ); + } else { + mv.visitFieldInsn( + GETSTATIC, + Type.getInternalName(StackWalker.Option.class), + "RETAIN_CLASS_REFERENCE", + Type.getDescriptor(StackWalker.Option.class) + ); + mv.visitMethodInsn( + INVOKESTATIC, + Type.getInternalName(StackWalker.class), + "getInstance", + Type.getMethodDescriptor(Type.getType(StackWalker.class), Type.getType(StackWalker.Option.class)), + false + ); + mv.visitMethodInsn( + INVOKEVIRTUAL, + Type.getInternalName(StackWalker.class), + "getCallerClass", + Type.getMethodDescriptor(Type.getType(Class.class)), + false + ); + } + } + + private void forwardIncomingArguments() { + int localVarIndex = 0; + if (instrumentedMethodIsCtor) { + localVarIndex++; + } else if (instrumentedMethodIsStatic == false) { + mv.visitVarInsn(Opcodes.ALOAD, localVarIndex++); + } + for (Type type : Type.getArgumentTypes(instrumentedMethodDescriptor)) { + mv.visitVarInsn(type.getOpcode(Opcodes.ILOAD), localVarIndex); + localVarIndex += type.getSize(); + } + } + + private void invokeInstrumentationMethod() { + mv.visitMethodInsn( + INVOKEINTERFACE, + checkMethod.className(), + checkMethod.methodName(), + Type.getMethodDescriptor( + Type.VOID_TYPE, + checkMethod.parameterDescriptors().stream().map(Type::getType).toArray(Type[]::new) + ), + true + ); + } + } + + protected void pushEntitlementChecker(MethodVisitor mv) { + mv.visitMethodInsn(INVOKESTATIC, handleClass, "instance", getCheckerClassMethodDescriptor, false); + } + + record ClassFileInfo(String fileName, byte[] bytecodes) {} +} diff --git a/libs/entitlement/asm-provider/src/main/resources/META-INF/services/org.elasticsearch.entitlement.instrumentation.InstrumentationService b/libs/entitlement/asm-provider/src/main/resources/META-INF/services/org.elasticsearch.entitlement.instrumentation.InstrumentationService new file mode 100644 index 0000000000000..da2e1bc1e67ba --- /dev/null +++ b/libs/entitlement/asm-provider/src/main/resources/META-INF/services/org.elasticsearch.entitlement.instrumentation.InstrumentationService @@ -0,0 +1,10 @@ +# + # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + # or more contributor license agreements. Licensed under the "Elastic License + # 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + # Public License v 1"; you may not use this file except in compliance with, at + # your election, the "Elastic License 2.0", the "GNU Affero General Public + # License v3.0 only", or the "Server Side Public License, v 1". +# + +org.elasticsearch.entitlement.instrumentation.impl.InstrumentationServiceImpl diff --git a/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/ASMUtils.java b/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/ASMUtils.java new file mode 100644 index 0000000000000..d7aaa6d854e9c --- /dev/null +++ b/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/ASMUtils.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.instrumentation.impl; + +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.util.Printer; +import org.objectweb.asm.util.Textifier; +import org.objectweb.asm.util.TraceClassVisitor; + +import java.io.PrintWriter; +import java.io.StringWriter; + +public class ASMUtils { + public static String bytecode2text(byte[] classBytes) { + ClassReader classReader = new ClassReader(classBytes); + StringWriter stringWriter = new StringWriter(); + try (PrintWriter printWriter = new PrintWriter(stringWriter)) { + Printer printer = new Textifier(); // For a textual representation + TraceClassVisitor traceClassVisitor = new TraceClassVisitor(null, printer, printWriter); + classReader.accept(traceClassVisitor, 0); + return stringWriter.toString(); + } + } +} diff --git a/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests.java b/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests.java new file mode 100644 index 0000000000000..ab0d96a8df96d --- /dev/null +++ b/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests.java @@ -0,0 +1,292 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.instrumentation.impl; + +import org.elasticsearch.entitlement.instrumentation.CheckMethod; +import org.elasticsearch.entitlement.instrumentation.InstrumentationService; +import org.elasticsearch.entitlement.instrumentation.MethodKey; +import org.elasticsearch.test.ESTestCase; +import org.objectweb.asm.Type; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; + +@ESTestCase.WithoutSecurityManager +public class InstrumentationServiceImplTests extends ESTestCase { + + final InstrumentationService instrumentationService = new InstrumentationServiceImpl(); + + static class TestTargetClass {} + + interface TestChecker { + void check$org_example_TestTargetClass$$staticMethod(Class clazz, int arg0, String arg1, Object arg2); + + void check$org_example_TestTargetClass$instanceMethodNoArgs(Class clazz, TestTargetClass that); + + void check$org_example_TestTargetClass$instanceMethodWithArgs(Class clazz, TestTargetClass that, int x, int y); + } + + interface TestCheckerOverloads { + void check$org_example_TestTargetClass$$staticMethodWithOverload(Class clazz, int x, int y); + + void check$org_example_TestTargetClass$$staticMethodWithOverload(Class clazz, int x, String y); + } + + interface TestCheckerCtors { + void check$org_example_TestTargetClass$(Class clazz); + + void check$org_example_TestTargetClass$(Class clazz, int x, String y); + } + + public void testInstrumentationTargetLookup() throws IOException { + Map checkMethods = instrumentationService.lookupMethods(TestChecker.class); + + assertThat(checkMethods, aMapWithSize(3)); + assertThat( + checkMethods, + hasEntry( + equalTo(new MethodKey("org/example/TestTargetClass", "staticMethod", List.of("I", "java/lang/String", "java/lang/Object"))), + equalTo( + new CheckMethod( + "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestChecker", + "check$org_example_TestTargetClass$$staticMethod", + List.of("Ljava/lang/Class;", "I", "Ljava/lang/String;", "Ljava/lang/Object;") + ) + ) + ) + ); + assertThat( + checkMethods, + hasEntry( + equalTo(new MethodKey("org/example/TestTargetClass", "instanceMethodNoArgs", List.of())), + equalTo( + new CheckMethod( + "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestChecker", + "check$org_example_TestTargetClass$instanceMethodNoArgs", + List.of( + "Ljava/lang/Class;", + "Lorg/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestTargetClass;" + ) + ) + ) + ) + ); + assertThat( + checkMethods, + hasEntry( + equalTo(new MethodKey("org/example/TestTargetClass", "instanceMethodWithArgs", List.of("I", "I"))), + equalTo( + new CheckMethod( + "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestChecker", + "check$org_example_TestTargetClass$instanceMethodWithArgs", + List.of( + "Ljava/lang/Class;", + "Lorg/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestTargetClass;", + "I", + "I" + ) + ) + ) + ) + ); + } + + public void testInstrumentationTargetLookupWithOverloads() throws IOException { + Map checkMethods = instrumentationService.lookupMethods(TestCheckerOverloads.class); + + assertThat(checkMethods, aMapWithSize(2)); + assertThat( + checkMethods, + hasEntry( + equalTo(new MethodKey("org/example/TestTargetClass", "staticMethodWithOverload", List.of("I", "java/lang/String"))), + equalTo( + new CheckMethod( + "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestCheckerOverloads", + "check$org_example_TestTargetClass$$staticMethodWithOverload", + List.of("Ljava/lang/Class;", "I", "Ljava/lang/String;") + ) + ) + ) + ); + assertThat( + checkMethods, + hasEntry( + equalTo(new MethodKey("org/example/TestTargetClass", "staticMethodWithOverload", List.of("I", "I"))), + equalTo( + new CheckMethod( + "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestCheckerOverloads", + "check$org_example_TestTargetClass$$staticMethodWithOverload", + List.of("Ljava/lang/Class;", "I", "I") + ) + ) + ) + ); + } + + public void testInstrumentationTargetLookupWithCtors() throws IOException { + Map checkMethods = instrumentationService.lookupMethods(TestCheckerCtors.class); + + assertThat(checkMethods, aMapWithSize(2)); + assertThat( + checkMethods, + hasEntry( + equalTo(new MethodKey("org/example/TestTargetClass", "", List.of("I", "java/lang/String"))), + equalTo( + new CheckMethod( + "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestCheckerCtors", + "check$org_example_TestTargetClass$", + List.of("Ljava/lang/Class;", "I", "Ljava/lang/String;") + ) + ) + ) + ); + assertThat( + checkMethods, + hasEntry( + equalTo(new MethodKey("org/example/TestTargetClass", "", List.of())), + equalTo( + new CheckMethod( + "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestCheckerCtors", + "check$org_example_TestTargetClass$", + List.of("Ljava/lang/Class;") + ) + ) + ) + ); + } + + public void testParseCheckerMethodSignatureStaticMethod() { + var methodKey = InstrumentationServiceImpl.parseCheckerMethodSignature( + "check$org_example_TestClass$$staticMethod", + new Type[] { Type.getType(Class.class) } + ); + + assertThat(methodKey, equalTo(new MethodKey("org/example/TestClass", "staticMethod", List.of()))); + } + + public void testParseCheckerMethodSignatureStaticMethodWithArgs() { + var methodKey = InstrumentationServiceImpl.parseCheckerMethodSignature( + "check$org_example_TestClass$$staticMethod", + new Type[] { Type.getType(Class.class), Type.getType("I"), Type.getType(String.class) } + ); + + assertThat(methodKey, equalTo(new MethodKey("org/example/TestClass", "staticMethod", List.of("I", "java/lang/String")))); + } + + public void testParseCheckerMethodSignatureStaticMethodInnerClass() { + var methodKey = InstrumentationServiceImpl.parseCheckerMethodSignature( + "check$org_example_TestClass$InnerClass$$staticMethod", + new Type[] { Type.getType(Class.class) } + ); + + assertThat(methodKey, equalTo(new MethodKey("org/example/TestClass$InnerClass", "staticMethod", List.of()))); + } + + public void testParseCheckerMethodSignatureCtor() { + var methodKey = InstrumentationServiceImpl.parseCheckerMethodSignature( + "check$org_example_TestClass$", + new Type[] { Type.getType(Class.class) } + ); + + assertThat(methodKey, equalTo(new MethodKey("org/example/TestClass", "", List.of()))); + } + + public void testParseCheckerMethodSignatureCtorWithArgs() { + var methodKey = InstrumentationServiceImpl.parseCheckerMethodSignature( + "check$org_example_TestClass$", + new Type[] { Type.getType(Class.class), Type.getType("I"), Type.getType(String.class) } + ); + + assertThat(methodKey, equalTo(new MethodKey("org/example/TestClass", "", List.of("I", "java/lang/String")))); + } + + public void testParseCheckerMethodSignatureOneDollarSign() { + assertParseCheckerMethodSignatureThrows("has incorrect name format", "check$method", Type.getType(Class.class)); + } + + public void testParseCheckerMethodSignatureMissingClass() { + assertParseCheckerMethodSignatureThrows("has incorrect name format", "check$$staticMethod", Type.getType(Class.class)); + } + + public void testParseCheckerMethodSignatureBlankClass() { + assertParseCheckerMethodSignatureThrows("no class name", "check$$$staticMethod", Type.getType(Class.class)); + } + + public void testParseCheckerMethodSignatureStaticMethodIncorrectArgumentCount() { + assertParseCheckerMethodSignatureThrows("It must have a first argument of Class type", "check$ClassName$staticMethod"); + } + + public void testParseCheckerMethodSignatureStaticMethodIncorrectArgumentType() { + assertParseCheckerMethodSignatureThrows( + "It must have a first argument of Class type", + "check$ClassName$$staticMethod", + Type.getType(String.class) + ); + } + + public void testParseCheckerMethodSignatureInstanceMethod() { + var methodKey = InstrumentationServiceImpl.parseCheckerMethodSignature( + "check$org_example_TestClass$instanceMethod", + new Type[] { Type.getType(Class.class), Type.getType(TestTargetClass.class) } + ); + + assertThat(methodKey, equalTo(new MethodKey("org/example/TestClass", "instanceMethod", List.of()))); + } + + public void testParseCheckerMethodSignatureInstanceMethodWithArgs() { + var methodKey = InstrumentationServiceImpl.parseCheckerMethodSignature( + "check$org_example_TestClass$instanceMethod", + new Type[] { Type.getType(Class.class), Type.getType(TestTargetClass.class), Type.getType("I"), Type.getType(String.class) } + ); + + assertThat(methodKey, equalTo(new MethodKey("org/example/TestClass", "instanceMethod", List.of("I", "java/lang/String")))); + } + + public void testParseCheckerMethodSignatureInstanceMethodIncorrectArgumentTypes() { + assertParseCheckerMethodSignatureThrows( + "It must have a first argument of Class type", + "check$org_example_TestClass$instanceMethod", + Type.getType(String.class) + ); + } + + public void testParseCheckerMethodSignatureInstanceMethodIncorrectArgumentCount() { + assertParseCheckerMethodSignatureThrows( + "a second argument of the class containing the method to instrument", + "check$org_example_TestClass$instanceMethod", + Type.getType(Class.class) + ); + } + + public void testParseCheckerMethodSignatureInstanceMethodIncorrectArgumentTypes2() { + assertParseCheckerMethodSignatureThrows( + "a second argument of the class containing the method to instrument", + "check$org_example_TestClass$instanceMethod", + Type.getType(Class.class), + Type.getType("I") + ); + } + + private static void assertParseCheckerMethodSignatureThrows(String messageText, String methodName, Type... methodArgs) { + var exception = assertThrows( + IllegalArgumentException.class, + () -> InstrumentationServiceImpl.parseCheckerMethodSignature(methodName, methodArgs) + ); + + assertThat(exception.getMessage(), containsString(messageText)); + } + +} diff --git a/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterTests.java b/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterTests.java new file mode 100644 index 0000000000000..35932969a3541 --- /dev/null +++ b/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterTests.java @@ -0,0 +1,397 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.instrumentation.impl; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.entitlement.instrumentation.CheckMethod; +import org.elasticsearch.entitlement.instrumentation.MethodKey; +import org.elasticsearch.entitlement.instrumentation.impl.InstrumenterImpl.ClassFileInfo; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; +import org.objectweb.asm.Type; + +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.Executable; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Stream; + +import static org.elasticsearch.entitlement.instrumentation.impl.ASMUtils.bytecode2text; +import static org.elasticsearch.entitlement.instrumentation.impl.InstrumenterImpl.getClassFileInfo; +import static org.hamcrest.Matchers.equalTo; + +/** + * This tests {@link InstrumenterImpl} can instrument various method signatures + * (e.g. overloaded methods, overloaded targets, multiple instrumentation, etc.) + */ +@ESTestCase.WithoutSecurityManager +public class InstrumenterTests extends ESTestCase { + private static final Logger logger = LogManager.getLogger(InstrumenterTests.class); + + static class TestLoader extends ClassLoader { + final byte[] testClassBytes; + final Class testClass; + + TestLoader(String testClassName, byte[] testClassBytes) { + super(InstrumenterTests.class.getClassLoader()); + this.testClassBytes = testClassBytes; + this.testClass = defineClass(testClassName, testClassBytes, 0, testClassBytes.length); + } + + Method getSameMethod(Method method) { + try { + return testClass.getMethod(method.getName(), method.getParameterTypes()); + } catch (NoSuchMethodException e) { + throw new AssertionError(e); + } + } + + Constructor getSameConstructor(Constructor ctor) { + try { + return testClass.getConstructor(ctor.getParameterTypes()); + } catch (NoSuchMethodException e) { + throw new AssertionError(e); + } + } + } + + /** + * Contains all the virtual methods from {@link TestClassToInstrument}, + * allowing this test to call them on the dynamically loaded instrumented class. + */ + public interface Testable { + // This method is here to demonstrate Instrumenter does not get confused by overloads + void someMethod(int arg); + + void someMethod(int arg, String anotherArg); + } + + /** + * This is a placeholder for real class library methods. + * Without the java agent, we can't instrument the real methods, so we instrument this instead. + *

    + * The instrumented copy of this class will not extend this class, but it will implement {@link Testable}. + */ + public static class TestClassToInstrument implements Testable { + + public TestClassToInstrument() {} + + public TestClassToInstrument(int arg) {} + + public void someMethod(int arg) {} + + public void someMethod(int arg, String anotherArg) {} + + public static void someStaticMethod(int arg) {} + + public static void someStaticMethod(int arg, String anotherArg) {} + + public static void anotherStaticMethod(int arg) {} + } + + /** + * Interface to test specific, "synthetic" cases (e.g. overloaded methods, overloaded constructors, etc.) that + * may be not present/may be difficult to find or not clear in the production EntitlementChecker interface. + *

    + * This interface isn't subject to the {@code check$} method naming conventions because it doesn't + * participate in the automated scan that configures the instrumenter based on the method names; + * instead, we configure the instrumenter minimally as needed for each test. + */ + public interface MockEntitlementChecker { + void checkSomeStaticMethod(Class callerClass, int arg); + + void checkSomeStaticMethodOverload(Class callerClass, int arg, String anotherArg); + + void checkAnotherStaticMethod(Class callerClass, int arg); + + void checkSomeInstanceMethod(Class callerClass, Testable that, int arg, String anotherArg); + + void checkCtor(Class callerClass); + + void checkCtorOverload(Class callerClass, int arg); + + } + + public static class TestEntitlementCheckerHolder { + static MockEntitlementCheckerImpl checkerInstance = new MockEntitlementCheckerImpl(); + + public static MockEntitlementChecker instance() { + return checkerInstance; + } + } + + public static class MockEntitlementCheckerImpl implements MockEntitlementChecker { + /** + * This allows us to test that the instrumentation is correct in both cases: + * if the check throws, and if it doesn't. + */ + volatile boolean isActive; + + int checkSomeStaticMethodIntCallCount = 0; + int checkAnotherStaticMethodIntCallCount = 0; + int checkSomeStaticMethodIntStringCallCount = 0; + int checkSomeInstanceMethodCallCount = 0; + + int checkCtorCallCount = 0; + int checkCtorIntCallCount = 0; + + private void throwIfActive() { + if (isActive) { + throw new TestException(); + } + } + + @Override + public void checkSomeStaticMethod(Class callerClass, int arg) { + checkSomeStaticMethodIntCallCount++; + assertSame(InstrumenterTests.class, callerClass); + assertEquals(123, arg); + throwIfActive(); + } + + @Override + public void checkSomeStaticMethodOverload(Class callerClass, int arg, String anotherArg) { + checkSomeStaticMethodIntStringCallCount++; + assertSame(InstrumenterTests.class, callerClass); + assertEquals(123, arg); + assertEquals("abc", anotherArg); + throwIfActive(); + } + + @Override + public void checkAnotherStaticMethod(Class callerClass, int arg) { + checkAnotherStaticMethodIntCallCount++; + assertSame(InstrumenterTests.class, callerClass); + assertEquals(123, arg); + throwIfActive(); + } + + @Override + public void checkSomeInstanceMethod(Class callerClass, Testable that, int arg, String anotherArg) { + checkSomeInstanceMethodCallCount++; + assertSame(InstrumenterTests.class, callerClass); + assertThat(that.getClass().getName(), equalTo(TestClassToInstrument.class.getName())); + assertEquals(123, arg); + assertEquals("def", anotherArg); + throwIfActive(); + } + + @Override + public void checkCtor(Class callerClass) { + checkCtorCallCount++; + assertSame(InstrumenterTests.class, callerClass); + throwIfActive(); + } + + @Override + public void checkCtorOverload(Class callerClass, int arg) { + checkCtorIntCallCount++; + assertSame(InstrumenterTests.class, callerClass); + assertEquals(123, arg); + throwIfActive(); + } + } + + @Before + public void resetInstance() { + TestEntitlementCheckerHolder.checkerInstance = new MockEntitlementCheckerImpl(); + } + + public void testStaticMethod() throws Exception { + Method targetMethod = TestClassToInstrument.class.getMethod("someStaticMethod", int.class); + TestLoader loader = instrumentTestClass(createInstrumenter(Map.of("checkSomeStaticMethod", targetMethod))); + + // Before checking is active, nothing should throw + assertStaticMethod(loader, targetMethod, 123); + // After checking is activated, everything should throw + assertStaticMethodThrows(loader, targetMethod, 123); + } + + public void testNotInstrumentedTwice() throws Exception { + Method targetMethod = TestClassToInstrument.class.getMethod("someStaticMethod", int.class); + var instrumenter = createInstrumenter(Map.of("checkSomeStaticMethod", targetMethod)); + + var loader1 = instrumentTestClass(instrumenter); + byte[] instrumentedTwiceBytecode = instrumenter.instrumentClass(TestClassToInstrument.class.getName(), loader1.testClassBytes); + logger.trace(() -> Strings.format("Bytecode after 2nd instrumentation:\n%s", bytecode2text(instrumentedTwiceBytecode))); + var loader2 = new TestLoader(TestClassToInstrument.class.getName(), instrumentedTwiceBytecode); + + assertStaticMethodThrows(loader2, targetMethod, 123); + assertEquals(1, TestEntitlementCheckerHolder.checkerInstance.checkSomeStaticMethodIntCallCount); + } + + public void testMultipleMethods() throws Exception { + Method targetMethod1 = TestClassToInstrument.class.getMethod("someStaticMethod", int.class); + Method targetMethod2 = TestClassToInstrument.class.getMethod("anotherStaticMethod", int.class); + + var instrumenter = createInstrumenter(Map.of("checkSomeStaticMethod", targetMethod1, "checkAnotherStaticMethod", targetMethod2)); + var loader = instrumentTestClass(instrumenter); + + assertStaticMethodThrows(loader, targetMethod1, 123); + assertEquals(1, TestEntitlementCheckerHolder.checkerInstance.checkSomeStaticMethodIntCallCount); + assertStaticMethodThrows(loader, targetMethod2, 123); + assertEquals(1, TestEntitlementCheckerHolder.checkerInstance.checkAnotherStaticMethodIntCallCount); + } + + public void testStaticMethodOverload() throws Exception { + Method targetMethod1 = TestClassToInstrument.class.getMethod("someStaticMethod", int.class); + Method targetMethod2 = TestClassToInstrument.class.getMethod("someStaticMethod", int.class, String.class); + var instrumenter = createInstrumenter( + Map.of("checkSomeStaticMethod", targetMethod1, "checkSomeStaticMethodOverload", targetMethod2) + ); + var loader = instrumentTestClass(instrumenter); + + assertStaticMethodThrows(loader, targetMethod1, 123); + assertStaticMethodThrows(loader, targetMethod2, 123, "abc"); + assertEquals(1, TestEntitlementCheckerHolder.checkerInstance.checkSomeStaticMethodIntCallCount); + assertEquals(1, TestEntitlementCheckerHolder.checkerInstance.checkSomeStaticMethodIntStringCallCount); + } + + public void testInstanceMethodOverload() throws Exception { + Method targetMethod = TestClassToInstrument.class.getMethod("someMethod", int.class, String.class); + var instrumenter = createInstrumenter(Map.of("checkSomeInstanceMethod", targetMethod)); + var loader = instrumentTestClass(instrumenter); + + TestEntitlementCheckerHolder.checkerInstance.isActive = true; + Testable testTargetClass = (Testable) (loader.testClass.getConstructor().newInstance()); + + // This overload is not instrumented, so it will not throw + testTargetClass.someMethod(123); + expectThrows(TestException.class, () -> testTargetClass.someMethod(123, "def")); + + assertEquals(1, TestEntitlementCheckerHolder.checkerInstance.checkSomeInstanceMethodCallCount); + } + + public void testConstructors() throws Exception { + Constructor ctor1 = TestClassToInstrument.class.getConstructor(); + Constructor ctor2 = TestClassToInstrument.class.getConstructor(int.class); + var loader = instrumentTestClass(createInstrumenter(Map.of("checkCtor", ctor1, "checkCtorOverload", ctor2))); + + assertCtorThrows(loader, ctor1); + assertCtorThrows(loader, ctor2, 123); + assertEquals(1, TestEntitlementCheckerHolder.checkerInstance.checkCtorCallCount); + assertEquals(1, TestEntitlementCheckerHolder.checkerInstance.checkCtorIntCallCount); + } + + /** + * These tests don't replace classToInstrument in-place but instead load a separate class with the same class name. + * This requires a configuration slightly different from what we'd use in production. + */ + private static InstrumenterImpl createInstrumenter(Map methods) throws NoSuchMethodException { + Map checkMethods = new HashMap<>(); + for (var entry : methods.entrySet()) { + checkMethods.put(getMethodKey(entry.getValue()), getCheckMethod(entry.getKey(), entry.getValue())); + } + String checkerClass = Type.getInternalName(MockEntitlementChecker.class); + String handleClass = Type.getInternalName(InstrumenterTests.TestEntitlementCheckerHolder.class); + String getCheckerClassMethodDescriptor = Type.getMethodDescriptor(Type.getObjectType(checkerClass)); + + return new InstrumenterImpl(handleClass, getCheckerClassMethodDescriptor, "", checkMethods); + } + + private static TestLoader instrumentTestClass(InstrumenterImpl instrumenter) throws IOException { + var clazz = TestClassToInstrument.class; + ClassFileInfo initial = getClassFileInfo(clazz); + byte[] newBytecode = instrumenter.instrumentClass(Type.getInternalName(clazz), initial.bytecodes()); + if (logger.isTraceEnabled()) { + logger.trace("Bytecode after instrumentation:\n{}", bytecode2text(newBytecode)); + } + return new TestLoader(clazz.getName(), newBytecode); + } + + private static MethodKey getMethodKey(Executable method) { + logger.info("method key: {}", method.getName()); + String methodName = method instanceof Constructor ? "" : method.getName(); + return new MethodKey( + Type.getInternalName(method.getDeclaringClass()), + methodName, + Stream.of(method.getParameterTypes()).map(Type::getType).map(Type::getInternalName).toList() + ); + } + + private static CheckMethod getCheckMethod(String methodName, Executable targetMethod) throws NoSuchMethodException { + boolean isStatic = Modifier.isStatic(targetMethod.getModifiers()); + boolean isInstance = isStatic == false && targetMethod instanceof Method; + int extraArgs = 1; // caller class + if (isInstance) { + ++extraArgs; + } + Class[] targetParameterTypes = targetMethod.getParameterTypes(); + Class[] checkParameterTypes = new Class[targetParameterTypes.length + extraArgs]; + checkParameterTypes[0] = Class.class; + if (isInstance) { + checkParameterTypes[1] = Testable.class; + } + System.arraycopy(targetParameterTypes, 0, checkParameterTypes, extraArgs, targetParameterTypes.length); + var checkMethod = MockEntitlementChecker.class.getMethod(methodName, checkParameterTypes); + return new CheckMethod( + Type.getInternalName(MockEntitlementChecker.class), + checkMethod.getName(), + Arrays.stream(Type.getArgumentTypes(checkMethod)).map(Type::getDescriptor).toList() + ); + } + + private static void unwrapInvocationException(InvocationTargetException e) { + Throwable cause = e.getCause(); + if (cause instanceof TestException n) { + // Sometimes we're expecting this one! + throw n; + } else { + throw new AssertionError(cause); + } + } + + /** + * Calling a static method of a dynamically loaded class is significantly more cumbersome + * than calling a virtual method. + */ + static void callStaticMethod(Method method, Object... args) { + try { + method.invoke(null, args); + } catch (InvocationTargetException e) { + unwrapInvocationException(e); + } catch (IllegalAccessException e) { + throw new AssertionError(e); + } + } + + private void assertStaticMethodThrows(TestLoader loader, Method method, Object... args) { + Method testMethod = loader.getSameMethod(method); + TestEntitlementCheckerHolder.checkerInstance.isActive = true; + expectThrows(TestException.class, () -> callStaticMethod(testMethod, args)); + } + + private void assertStaticMethod(TestLoader loader, Method method, Object... args) { + Method testMethod = loader.getSameMethod(method); + TestEntitlementCheckerHolder.checkerInstance.isActive = false; + callStaticMethod(testMethod, args); + } + + private void assertCtorThrows(TestLoader loader, Constructor ctor, Object... args) { + Constructor testCtor = loader.getSameConstructor(ctor); + TestEntitlementCheckerHolder.checkerInstance.isActive = true; + expectThrows(TestException.class, () -> { + try { + testCtor.newInstance(args); + } catch (InvocationTargetException e) { + unwrapInvocationException(e); + } catch (IllegalAccessException | InstantiationException e) { + throw new AssertionError(e); + } + }); + } +} diff --git a/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/TestException.java b/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/TestException.java new file mode 100644 index 0000000000000..5e308e5bd4a98 --- /dev/null +++ b/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/TestException.java @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.instrumentation.impl; + +final class TestException extends RuntimeException {} diff --git a/libs/entitlement/bridge/README.md b/libs/entitlement/bridge/README.md new file mode 100644 index 0000000000000..8beea6b3fe894 --- /dev/null +++ b/libs/entitlement/bridge/README.md @@ -0,0 +1,11 @@ +### Entitlement Bridge + +This is the code called directly from instrumented methods. +It's a minimal shim that is patched into the `java.base` module +so that it is callable from the class library methods instrumented by the agent. +Its job is to forward the entitlement checks to the main library, +which is loaded normally. + +It is not responsible for injecting the bytecode instrumentation (that's the agent) +nor for implementing the permission checks (that's the main library). + diff --git a/libs/entitlement/bridge/build.gradle b/libs/entitlement/bridge/build.gradle new file mode 100644 index 0000000000000..a9f8f6e3a3b0a --- /dev/null +++ b/libs/entitlement/bridge/build.gradle @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import org.elasticsearch.gradle.internal.precommit.CheckForbiddenApisTask + +apply plugin: 'elasticsearch.build' +apply plugin: 'elasticsearch.mrjar' + +tasks.named('jar').configure { + // guarding for intellij + if (sourceSets.findByName("main23")) { + from sourceSets.main23.output + } +} + +tasks.withType(CheckForbiddenApisTask).configureEach { + replaceSignatureFiles 'jdk-signatures' +} diff --git a/libs/entitlement/bridge/src/main/java/module-info.java b/libs/entitlement/bridge/src/main/java/module-info.java new file mode 100644 index 0000000000000..93292109a726e --- /dev/null +++ b/libs/entitlement/bridge/src/main/java/module-info.java @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +// This module-info is used just to satisfy your IDE. +// At build and run time, the bridge is patched into the java.base module. +module org.elasticsearch.entitlement.bridge { + exports org.elasticsearch.entitlement.bridge; +} diff --git a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java new file mode 100644 index 0000000000000..67d006868b48d --- /dev/null +++ b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.bridge; + +import java.net.URL; +import java.net.URLStreamHandlerFactory; +import java.util.List; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocketFactory; + +@SuppressWarnings("unused") // Called from instrumentation code inserted by the Entitlements agent +public interface EntitlementChecker { + + // Exit the JVM process + void check$java_lang_Runtime$exit(Class callerClass, Runtime runtime, int status); + + void check$java_lang_Runtime$halt(Class callerClass, Runtime runtime, int status); + + // ClassLoader ctor + void check$java_lang_ClassLoader$(Class callerClass); + + void check$java_lang_ClassLoader$(Class callerClass, ClassLoader parent); + + void check$java_lang_ClassLoader$(Class callerClass, String name, ClassLoader parent); + + // SecureClassLoader ctor + void check$java_security_SecureClassLoader$(Class callerClass); + + void check$java_security_SecureClassLoader$(Class callerClass, ClassLoader parent); + + void check$java_security_SecureClassLoader$(Class callerClass, String name, ClassLoader parent); + + // URLClassLoader constructors + void check$java_net_URLClassLoader$(Class callerClass, URL[] urls); + + void check$java_net_URLClassLoader$(Class callerClass, URL[] urls, ClassLoader parent); + + void check$java_net_URLClassLoader$(Class callerClass, URL[] urls, ClassLoader parent, URLStreamHandlerFactory factory); + + void check$java_net_URLClassLoader$(Class callerClass, String name, URL[] urls, ClassLoader parent); + + void check$java_net_URLClassLoader$(Class callerClass, String name, URL[] urls, ClassLoader parent, URLStreamHandlerFactory factory); + + // "setFactory" methods + void check$javax_net_ssl_HttpsURLConnection$setSSLSocketFactory(Class callerClass, HttpsURLConnection conn, SSLSocketFactory sf); + + void check$javax_net_ssl_HttpsURLConnection$$setDefaultSSLSocketFactory(Class callerClass, SSLSocketFactory sf); + + void check$javax_net_ssl_HttpsURLConnection$$setDefaultHostnameVerifier(Class callerClass, HostnameVerifier hv); + + void check$javax_net_ssl_SSLContext$$setDefault(Class callerClass, SSLContext context); + + // Process creation + void check$java_lang_ProcessBuilder$start(Class callerClass, ProcessBuilder that); + + void check$java_lang_ProcessBuilder$$startPipeline(Class callerClass, List builders); + +} diff --git a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementCheckerHandle.java b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementCheckerHandle.java new file mode 100644 index 0000000000000..26c9c83b8eb51 --- /dev/null +++ b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementCheckerHandle.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.bridge; + +/** + * Makes the {@link EntitlementChecker} available to injected bytecode. + */ +public class EntitlementCheckerHandle { + + /** + * This is how the bytecodes injected by our instrumentation access the {@link EntitlementChecker} + * so they can call the appropriate check method. + */ + public static EntitlementChecker instance() { + return Holder.instance; + } + + /** + * Having a separate inner {@code Holder} class ensures that the field is initialized + * the first time {@link #instance()} is called, rather than the first time anyone anywhere + * references the {@link EntitlementCheckerHandle} class. + */ + private static class Holder { + /** + * The {@code EntitlementInitialization} class is what actually instantiates it and makes it available; + * here, we copy it into a static final variable for maximum performance. + */ + private static final EntitlementChecker instance = HandleLoader.load(EntitlementChecker.class); + } + + // no construction + private EntitlementCheckerHandle() {} +} diff --git a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/HandleLoader.java b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/HandleLoader.java new file mode 100644 index 0000000000000..bbfec47884f79 --- /dev/null +++ b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/HandleLoader.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.bridge; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + +class HandleLoader { + + static T load(Class checkerClass) { + String initClassName = "org.elasticsearch.entitlement.initialization.EntitlementInitialization"; + final Class initClazz; + try { + initClazz = ClassLoader.getSystemClassLoader().loadClass(initClassName); + } catch (ClassNotFoundException e) { + throw new AssertionError("java.base cannot find entitlement initialization", e); + } + final Method checkerMethod; + try { + checkerMethod = initClazz.getMethod("checker"); + } catch (NoSuchMethodException e) { + throw new AssertionError("EntitlementInitialization is missing checker() method", e); + } + try { + return checkerClass.cast(checkerMethod.invoke(null)); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new AssertionError(e); + } + } + + // no instance + private HandleLoader() {} +} diff --git a/libs/entitlement/bridge/src/main23/java/org/elasticsearch/entitlement/bridge/Java23EntitlementChecker.java b/libs/entitlement/bridge/src/main23/java/org/elasticsearch/entitlement/bridge/Java23EntitlementChecker.java new file mode 100644 index 0000000000000..244632e80ffa0 --- /dev/null +++ b/libs/entitlement/bridge/src/main23/java/org/elasticsearch/entitlement/bridge/Java23EntitlementChecker.java @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.bridge; + +public interface Java23EntitlementChecker extends EntitlementChecker {} diff --git a/libs/entitlement/bridge/src/main23/java/org/elasticsearch/entitlement/bridge/Java23EntitlementCheckerHandle.java b/libs/entitlement/bridge/src/main23/java/org/elasticsearch/entitlement/bridge/Java23EntitlementCheckerHandle.java new file mode 100644 index 0000000000000..f41c5dcdf14fd --- /dev/null +++ b/libs/entitlement/bridge/src/main23/java/org/elasticsearch/entitlement/bridge/Java23EntitlementCheckerHandle.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.bridge; + +/** + * Java23 variant of {@link EntitlementChecker} handle holder. + */ +public class Java23EntitlementCheckerHandle { + + public static Java23EntitlementChecker instance() { + return Holder.instance; + } + + private static class Holder { + private static final Java23EntitlementChecker instance = HandleLoader.load(Java23EntitlementChecker.class); + } + + // no construction + private Java23EntitlementCheckerHandle() {} +} diff --git a/libs/entitlement/build.gradle b/libs/entitlement/build.gradle new file mode 100644 index 0000000000000..841591873153c --- /dev/null +++ b/libs/entitlement/build.gradle @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import org.elasticsearch.gradle.internal.precommit.CheckForbiddenApisTask + +apply plugin: 'elasticsearch.build' +apply plugin: 'elasticsearch.publish' +apply plugin: 'elasticsearch.embedded-providers' +apply plugin: 'elasticsearch.mrjar' + +embeddedProviders { + impl 'entitlement', project(':libs:entitlement:asm-provider') +} + +dependencies { + compileOnly project(':libs:core') // For @SuppressForbidden + compileOnly project(':libs:logging') + compileOnly project(":libs:x-content") // for parsing policy files + compileOnly project(':libs:entitlement:bridge') + testImplementation(project(":test:framework")) { + exclude group: 'org.elasticsearch', module: 'entitlement' + } + + // guarding for intellij + if (sourceSets.findByName("main23")) { + main23CompileOnly project(path: ':libs:entitlement:bridge', configuration: 'java23') + } +} + +tasks.withType(CheckForbiddenApisTask).configureEach { + replaceSignatureFiles 'jdk-signatures' +} diff --git a/libs/entitlement/entitlements-loading.svg b/libs/entitlement/entitlements-loading.svg new file mode 100644 index 0000000000000..4f0213b853bee --- /dev/null +++ b/libs/entitlement/entitlements-loading.svg @@ -0,0 +1,4 @@ + + + +
    ES main
    ES main
    Boot Loader
    Boot Loader
    Platform Loader
    Platform Loader
    System Loader
    System Loader
    reflection
    reflection
    Agent Jar
    Agent Jar
    Server
    Server
    (Instrumented)
    JDK classes
    (Instrumented)...
    agent main
    (in unnamed module)
    agent main...
    entitlements ready
    entitlements ready
    reflection
    reflection
    Bridge
    (patched into java.base)
    Bridge...
    Entitlements
    Entitlements
    Entitlements bootstrap
    Entitlements bootstrap
    • Grant access to unnamed module
    • Set (static, protected) init arguments
    • Load agent
    Grant access to unnamed modu...
    (reflectively) call 
    entitlements init
    with Instrumentation
    (reflectively) call...
    Entitlements init
    Entitlements init
    • Load plugin policies
    • Load server policy
    • Create entitlements manager
      • Policies
      • Method to lookup plugin by Module
    • Set entitlements manager in static (accessible by bridge)
    • Instrument jdk classes
    • run self test (force bridge to capture entitlements manager)
    Load plugin policiesLoad server policyCreate e...
    Text is not SVG - cannot display
    \ No newline at end of file diff --git a/libs/entitlement/qa/build.gradle b/libs/entitlement/qa/build.gradle new file mode 100644 index 0000000000000..7f46b2fe20a8a --- /dev/null +++ b/libs/entitlement/qa/build.gradle @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +apply plugin: 'elasticsearch.internal-java-rest-test' +// Necessary to use tests in Serverless +apply plugin: 'elasticsearch.internal-test-artifact' + +dependencies { + javaRestTestImplementation project(':libs:entitlement:qa:common') + clusterModules project(':libs:entitlement:qa:entitlement-allowed') + clusterModules project(':libs:entitlement:qa:entitlement-allowed-nonmodular') + clusterPlugins project(':libs:entitlement:qa:entitlement-denied') + clusterPlugins project(':libs:entitlement:qa:entitlement-denied-nonmodular') +} diff --git a/libs/entitlement/qa/common/build.gradle b/libs/entitlement/qa/common/build.gradle new file mode 100644 index 0000000000000..df3bc66cba21b --- /dev/null +++ b/libs/entitlement/qa/common/build.gradle @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +apply plugin: 'elasticsearch.build' + +dependencies { + implementation project(':server') + implementation project(':libs:logging') +} diff --git a/libs/entitlement/qa/common/src/main/java/module-info.java b/libs/entitlement/qa/common/src/main/java/module-info.java new file mode 100644 index 0000000000000..2dd37e3174e08 --- /dev/null +++ b/libs/entitlement/qa/common/src/main/java/module-info.java @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +module org.elasticsearch.entitlement.qa.common { + requires org.elasticsearch.server; + requires org.elasticsearch.base; + requires org.elasticsearch.logging; + + exports org.elasticsearch.entitlement.qa.common; +} diff --git a/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java b/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java new file mode 100644 index 0000000000000..4afceedbe3f01 --- /dev/null +++ b/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java @@ -0,0 +1,178 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.qa.common; + +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.RestStatus; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URL; +import java.net.URLClassLoader; +import java.security.NoSuchAlgorithmException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; + +import static java.util.Map.entry; +import static org.elasticsearch.entitlement.qa.common.RestEntitlementsCheckAction.CheckAction.alwaysDenied; +import static org.elasticsearch.entitlement.qa.common.RestEntitlementsCheckAction.CheckAction.deniedToPlugins; +import static org.elasticsearch.entitlement.qa.common.RestEntitlementsCheckAction.CheckAction.forPlugins; +import static org.elasticsearch.rest.RestRequest.Method.GET; + +public class RestEntitlementsCheckAction extends BaseRestHandler { + private static final Logger logger = LogManager.getLogger(RestEntitlementsCheckAction.class); + private final String prefix; + + record CheckAction(Runnable action, boolean isAlwaysDeniedToPlugins) { + /** + * These cannot be granted to plugins, so our test plugins cannot test the "allowed" case. + * Used both for always-denied entitlements as well as those granted only to the server itself. + */ + static CheckAction deniedToPlugins(Runnable action) { + return new CheckAction(action, true); + } + + static CheckAction forPlugins(Runnable action) { + return new CheckAction(action, false); + } + + static CheckAction alwaysDenied(Runnable action) { + return new CheckAction(action, true); + } + } + + private static final Map checkActions = Map.ofEntries( + entry("runtime_exit", deniedToPlugins(RestEntitlementsCheckAction::runtimeExit)), + entry("runtime_halt", deniedToPlugins(RestEntitlementsCheckAction::runtimeHalt)), + entry("create_classloader", forPlugins(RestEntitlementsCheckAction::createClassLoader)), + entry("processBuilder_start", deniedToPlugins(RestEntitlementsCheckAction::processBuilder_start)), + entry("processBuilder_startPipeline", deniedToPlugins(RestEntitlementsCheckAction::processBuilder_startPipeline)), + entry("set_https_connection_properties", forPlugins(RestEntitlementsCheckAction::setHttpsConnectionProperties)), + entry("set_default_ssl_socket_factory", alwaysDenied(RestEntitlementsCheckAction::setDefaultSSLSocketFactory)), + entry("set_default_hostname_verifier", alwaysDenied(RestEntitlementsCheckAction::setDefaultHostnameVerifier)), + entry("set_default_ssl_context", alwaysDenied(RestEntitlementsCheckAction::setDefaultSSLContext)) + ); + + private static void setDefaultSSLContext() { + logger.info("Calling SSLContext.setDefault"); + try { + SSLContext.setDefault(SSLContext.getDefault()); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + + private static void setDefaultHostnameVerifier() { + logger.info("Calling HttpsURLConnection.setDefaultHostnameVerifier"); + HttpsURLConnection.setDefaultHostnameVerifier((hostname, session) -> false); + } + + private static void setDefaultSSLSocketFactory() { + logger.info("Calling HttpsURLConnection.setDefaultSSLSocketFactory"); + HttpsURLConnection.setDefaultSSLSocketFactory(new TestSSLSocketFactory()); + } + + @SuppressForbidden(reason = "Specifically testing Runtime.exit") + private static void runtimeExit() { + Runtime.getRuntime().exit(123); + } + + @SuppressForbidden(reason = "Specifically testing Runtime.halt") + private static void runtimeHalt() { + Runtime.getRuntime().halt(123); + } + + private static void createClassLoader() { + try (var classLoader = new URLClassLoader("test", new URL[0], RestEntitlementsCheckAction.class.getClassLoader())) { + logger.info("Created URLClassLoader [{}]", classLoader.getName()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static void processBuilder_start() { + try { + new ProcessBuilder("").start(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + private static void processBuilder_startPipeline() { + try { + ProcessBuilder.startPipeline(List.of()); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + private static void setHttpsConnectionProperties() { + logger.info("Calling setSSLSocketFactory"); + var connection = new TestHttpsURLConnection(); + connection.setSSLSocketFactory(new TestSSLSocketFactory()); + } + + public RestEntitlementsCheckAction(String prefix) { + this.prefix = prefix; + } + + public static Set getCheckActionsAllowedInPlugins() { + return checkActions.entrySet() + .stream() + .filter(kv -> kv.getValue().isAlwaysDeniedToPlugins() == false) + .map(Map.Entry::getKey) + .collect(Collectors.toSet()); + } + + public static Set getAllCheckActions() { + return checkActions.keySet(); + } + + @Override + public List routes() { + return List.of(new Route(GET, "/_entitlement/" + prefix + "/_check")); + } + + @Override + public String getName() { + return "check_" + prefix + "_action"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { + logger.info("RestEntitlementsCheckAction rest handler [{}]", request.path()); + var actionName = request.param("action"); + if (Strings.isNullOrEmpty(actionName)) { + throw new IllegalArgumentException("Missing action parameter"); + } + var checkAction = checkActions.get(actionName); + if (checkAction == null) { + throw new IllegalArgumentException(Strings.format("Unknown action [%s]", actionName)); + } + + return channel -> { + logger.info("Calling check action [{}]", actionName); + checkAction.action().run(); + channel.sendResponse(new RestResponse(RestStatus.OK, Strings.format("Succesfully executed action [%s]", actionName))); + }; + } +} diff --git a/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/TestHttpsURLConnection.java b/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/TestHttpsURLConnection.java new file mode 100644 index 0000000000000..5a96e582db02b --- /dev/null +++ b/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/TestHttpsURLConnection.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.qa.common; + +import java.io.IOException; +import java.security.cert.Certificate; + +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLPeerUnverifiedException; + +class TestHttpsURLConnection extends HttpsURLConnection { + TestHttpsURLConnection() { + super(null); + } + + @Override + public void connect() throws IOException {} + + @Override + public void disconnect() {} + + @Override + public boolean usingProxy() { + return false; + } + + @Override + public String getCipherSuite() { + return ""; + } + + @Override + public Certificate[] getLocalCertificates() { + return new Certificate[0]; + } + + @Override + public Certificate[] getServerCertificates() throws SSLPeerUnverifiedException { + return new Certificate[0]; + } +} diff --git a/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/TestSSLSocketFactory.java b/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/TestSSLSocketFactory.java new file mode 100644 index 0000000000000..feb19df780175 --- /dev/null +++ b/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/TestSSLSocketFactory.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.qa.common; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.Socket; +import java.net.UnknownHostException; + +import javax.net.ssl.SSLSocketFactory; + +class TestSSLSocketFactory extends SSLSocketFactory { + @Override + public Socket createSocket(String host, int port) throws IOException, UnknownHostException { + return null; + } + + @Override + public Socket createSocket(String host, int port, InetAddress localHost, int localPort) { + return null; + } + + @Override + public Socket createSocket(InetAddress host, int port) throws IOException { + return null; + } + + @Override + public Socket createSocket(InetAddress address, int port, InetAddress localAddress, int localPort) throws IOException { + return null; + } + + @Override + public String[] getDefaultCipherSuites() { + return new String[0]; + } + + @Override + public String[] getSupportedCipherSuites() { + return new String[0]; + } + + @Override + public Socket createSocket(Socket s, String host, int port, boolean autoClose) throws IOException { + return null; + } +} diff --git a/libs/entitlement/qa/entitlement-allowed-nonmodular/build.gradle b/libs/entitlement/qa/entitlement-allowed-nonmodular/build.gradle new file mode 100644 index 0000000000000..316b61e15707e --- /dev/null +++ b/libs/entitlement/qa/entitlement-allowed-nonmodular/build.gradle @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +apply plugin: 'elasticsearch.base-internal-es-plugin' + +esplugin { + name = 'entitlement-allowed-nonmodular' + description = 'A non-modular test module that invokes entitlement checks that are supposed to be granted' + classname = 'org.elasticsearch.entitlement.qa.nonmodular.EntitlementAllowedNonModularPlugin' +} + +dependencies { + implementation project(':libs:entitlement:qa:common') +} + +tasks.named("javadoc").configure { + enabled = false +} diff --git a/libs/entitlement/qa/entitlement-allowed-nonmodular/src/main/java/org/elasticsearch/entitlement/qa/nonmodular/EntitlementAllowedNonModularPlugin.java b/libs/entitlement/qa/entitlement-allowed-nonmodular/src/main/java/org/elasticsearch/entitlement/qa/nonmodular/EntitlementAllowedNonModularPlugin.java new file mode 100644 index 0000000000000..82146e6a87759 --- /dev/null +++ b/libs/entitlement/qa/entitlement-allowed-nonmodular/src/main/java/org/elasticsearch/entitlement/qa/nonmodular/EntitlementAllowedNonModularPlugin.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.entitlement.qa.nonmodular; + +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.entitlement.qa.common.RestEntitlementsCheckAction; +import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestHandler; + +import java.util.List; +import java.util.function.Predicate; +import java.util.function.Supplier; + +public class EntitlementAllowedNonModularPlugin extends Plugin implements ActionPlugin { + @Override + public List getRestHandlers( + final Settings settings, + NamedWriteableRegistry namedWriteableRegistry, + final RestController restController, + final ClusterSettings clusterSettings, + final IndexScopedSettings indexScopedSettings, + final SettingsFilter settingsFilter, + final IndexNameExpressionResolver indexNameExpressionResolver, + final Supplier nodesInCluster, + Predicate clusterSupportsFeature + ) { + return List.of(new RestEntitlementsCheckAction("allowed_nonmodular")); + } +} diff --git a/libs/entitlement/qa/entitlement-allowed-nonmodular/src/main/plugin-metadata/entitlement-policy.yaml b/libs/entitlement/qa/entitlement-allowed-nonmodular/src/main/plugin-metadata/entitlement-policy.yaml new file mode 100644 index 0000000000000..30fc9f0abeec0 --- /dev/null +++ b/libs/entitlement/qa/entitlement-allowed-nonmodular/src/main/plugin-metadata/entitlement-policy.yaml @@ -0,0 +1,3 @@ +ALL-UNNAMED: + - create_class_loader + - set_https_connection_properties diff --git a/libs/entitlement/qa/entitlement-allowed/build.gradle b/libs/entitlement/qa/entitlement-allowed/build.gradle new file mode 100644 index 0000000000000..b9518f8d65fb5 --- /dev/null +++ b/libs/entitlement/qa/entitlement-allowed/build.gradle @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +apply plugin: 'elasticsearch.base-internal-es-plugin' + +esplugin { + name = 'entitlement-allowed' + description = 'A test module that invokes entitlement checks that are supposed to be granted' + classname = 'org.elasticsearch.entitlement.qa.EntitlementAllowedPlugin' +} + +dependencies { + implementation project(':libs:entitlement:qa:common') +} + +tasks.named("javadoc").configure { + enabled = false +} + diff --git a/libs/entitlement/qa/entitlement-allowed/src/main/java/module-info.java b/libs/entitlement/qa/entitlement-allowed/src/main/java/module-info.java new file mode 100644 index 0000000000000..a88611e6ac9a5 --- /dev/null +++ b/libs/entitlement/qa/entitlement-allowed/src/main/java/module-info.java @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +module org.elasticsearch.entitlement.qa.allowed { + requires org.elasticsearch.server; + requires org.elasticsearch.base; + requires org.elasticsearch.logging; + requires org.elasticsearch.entitlement.qa.common; +} diff --git a/libs/entitlement/qa/entitlement-allowed/src/main/java/org/elasticsearch/entitlement/qa/EntitlementAllowedPlugin.java b/libs/entitlement/qa/entitlement-allowed/src/main/java/org/elasticsearch/entitlement/qa/EntitlementAllowedPlugin.java new file mode 100644 index 0000000000000..8649daf272e70 --- /dev/null +++ b/libs/entitlement/qa/entitlement-allowed/src/main/java/org/elasticsearch/entitlement/qa/EntitlementAllowedPlugin.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.entitlement.qa; + +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.entitlement.qa.common.RestEntitlementsCheckAction; +import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestHandler; + +import java.util.List; +import java.util.function.Predicate; +import java.util.function.Supplier; + +public class EntitlementAllowedPlugin extends Plugin implements ActionPlugin { + @Override + public List getRestHandlers( + final Settings settings, + NamedWriteableRegistry namedWriteableRegistry, + final RestController restController, + final ClusterSettings clusterSettings, + final IndexScopedSettings indexScopedSettings, + final SettingsFilter settingsFilter, + final IndexNameExpressionResolver indexNameExpressionResolver, + final Supplier nodesInCluster, + Predicate clusterSupportsFeature + ) { + return List.of(new RestEntitlementsCheckAction("allowed")); + } +} diff --git a/libs/entitlement/qa/entitlement-allowed/src/main/plugin-metadata/entitlement-policy.yaml b/libs/entitlement/qa/entitlement-allowed/src/main/plugin-metadata/entitlement-policy.yaml new file mode 100644 index 0000000000000..0a25570a9f624 --- /dev/null +++ b/libs/entitlement/qa/entitlement-allowed/src/main/plugin-metadata/entitlement-policy.yaml @@ -0,0 +1,3 @@ +org.elasticsearch.entitlement.qa.common: + - create_class_loader + - set_https_connection_properties diff --git a/libs/entitlement/qa/entitlement-denied-nonmodular/build.gradle b/libs/entitlement/qa/entitlement-denied-nonmodular/build.gradle new file mode 100644 index 0000000000000..6a88dd66eaf75 --- /dev/null +++ b/libs/entitlement/qa/entitlement-denied-nonmodular/build.gradle @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +apply plugin: 'elasticsearch.base-internal-es-plugin' + +esplugin { + name = 'entitlement-denied-nonmodular' + description = 'A non-modular test module that invokes non-granted entitlement and triggers exceptions' + classname = 'org.elasticsearch.entitlement.qa.nonmodular.EntitlementDeniedNonModularPlugin' +} + +dependencies { + implementation project(':libs:entitlement:qa:common') +} + +tasks.named("javadoc").configure { + enabled = false +} diff --git a/libs/entitlement/qa/entitlement-denied-nonmodular/src/main/java/org/elasticsearch/entitlement/qa/nonmodular/EntitlementDeniedNonModularPlugin.java b/libs/entitlement/qa/entitlement-denied-nonmodular/src/main/java/org/elasticsearch/entitlement/qa/nonmodular/EntitlementDeniedNonModularPlugin.java new file mode 100644 index 0000000000000..7ca89c735a602 --- /dev/null +++ b/libs/entitlement/qa/entitlement-denied-nonmodular/src/main/java/org/elasticsearch/entitlement/qa/nonmodular/EntitlementDeniedNonModularPlugin.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.entitlement.qa.nonmodular; + +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.entitlement.qa.common.RestEntitlementsCheckAction; +import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestHandler; + +import java.util.List; +import java.util.function.Predicate; +import java.util.function.Supplier; + +public class EntitlementDeniedNonModularPlugin extends Plugin implements ActionPlugin { + @Override + public List getRestHandlers( + final Settings settings, + NamedWriteableRegistry namedWriteableRegistry, + final RestController restController, + final ClusterSettings clusterSettings, + final IndexScopedSettings indexScopedSettings, + final SettingsFilter settingsFilter, + final IndexNameExpressionResolver indexNameExpressionResolver, + final Supplier nodesInCluster, + Predicate clusterSupportsFeature + ) { + return List.of(new RestEntitlementsCheckAction("denied_nonmodular")); + } +} diff --git a/libs/entitlement/qa/entitlement-denied/build.gradle b/libs/entitlement/qa/entitlement-denied/build.gradle new file mode 100644 index 0000000000000..9d1872563b8c5 --- /dev/null +++ b/libs/entitlement/qa/entitlement-denied/build.gradle @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +apply plugin: 'elasticsearch.base-internal-es-plugin' + +esplugin { + name = 'entitlement-denied' + description = 'A test module that invokes non-granted entitlement and triggers exceptions' + classname = 'org.elasticsearch.entitlement.qa.EntitlementDeniedPlugin' +} + +dependencies { + implementation project(':libs:entitlement:qa:common') +} + +tasks.named("javadoc").configure { + enabled = false +} + diff --git a/libs/entitlement/qa/entitlement-denied/src/main/java/module-info.java b/libs/entitlement/qa/entitlement-denied/src/main/java/module-info.java new file mode 100644 index 0000000000000..3def472be7a45 --- /dev/null +++ b/libs/entitlement/qa/entitlement-denied/src/main/java/module-info.java @@ -0,0 +1,6 @@ +module org.elasticsearch.entitlement.qa.denied { + requires org.elasticsearch.server; + requires org.elasticsearch.base; + requires org.apache.logging.log4j; + requires org.elasticsearch.entitlement.qa.common; +} diff --git a/libs/entitlement/qa/entitlement-denied/src/main/java/org/elasticsearch/entitlement/qa/EntitlementDeniedPlugin.java b/libs/entitlement/qa/entitlement-denied/src/main/java/org/elasticsearch/entitlement/qa/EntitlementDeniedPlugin.java new file mode 100644 index 0000000000000..2a2fd35d47cf3 --- /dev/null +++ b/libs/entitlement/qa/entitlement-denied/src/main/java/org/elasticsearch/entitlement/qa/EntitlementDeniedPlugin.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.entitlement.qa; + +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.entitlement.qa.common.RestEntitlementsCheckAction; +import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestHandler; + +import java.util.List; +import java.util.function.Predicate; +import java.util.function.Supplier; + +public class EntitlementDeniedPlugin extends Plugin implements ActionPlugin { + @Override + public List getRestHandlers( + final Settings settings, + NamedWriteableRegistry namedWriteableRegistry, + final RestController restController, + final ClusterSettings clusterSettings, + final IndexScopedSettings indexScopedSettings, + final SettingsFilter settingsFilter, + final IndexNameExpressionResolver indexNameExpressionResolver, + final Supplier nodesInCluster, + Predicate clusterSupportsFeature + ) { + return List.of(new RestEntitlementsCheckAction("denied")); + } +} diff --git a/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsAllowedIT.java b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsAllowedIT.java new file mode 100644 index 0000000000000..c38e8b3f35efb --- /dev/null +++ b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsAllowedIT.java @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.qa; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.entitlement.qa.common.RestEntitlementsCheckAction; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.ClassRule; + +import java.io.IOException; +import java.util.stream.Stream; + +import static org.hamcrest.Matchers.equalTo; + +public class EntitlementsAllowedIT extends ESRestTestCase { + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .module("entitlement-allowed") + .module("entitlement-allowed-nonmodular") + .systemProperty("es.entitlements.enabled", "true") + .setting("xpack.security.enabled", "false") + .build(); + + private final String pathPrefix; + private final String actionName; + + public EntitlementsAllowedIT(@Name("pathPrefix") String pathPrefix, @Name("actionName") String actionName) { + this.pathPrefix = pathPrefix; + this.actionName = actionName; + } + + @ParametersFactory + public static Iterable data() { + return Stream.of("allowed", "allowed_nonmodular") + .flatMap( + path -> RestEntitlementsCheckAction.getCheckActionsAllowedInPlugins().stream().map(action -> new Object[] { path, action }) + ) + .toList(); + } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + public void testCheckActionWithPolicyPass() throws IOException { + logger.info("Executing Entitlement test [{}] for [{}]", pathPrefix, actionName); + var request = new Request("GET", "/_entitlement/" + pathPrefix + "/_check"); + request.addParameter("action", actionName); + Response result = client().performRequest(request); + assertThat(result.getStatusLine().getStatusCode(), equalTo(200)); + } +} diff --git a/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsDeniedIT.java b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsDeniedIT.java new file mode 100644 index 0000000000000..b17a57512cde0 --- /dev/null +++ b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsDeniedIT.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.qa; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.client.Request; +import org.elasticsearch.entitlement.qa.common.RestEntitlementsCheckAction; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.ClassRule; + +import java.io.IOException; +import java.util.stream.Stream; + +import static org.hamcrest.Matchers.containsString; + +public class EntitlementsDeniedIT extends ESRestTestCase { + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .plugin("entitlement-denied") + .plugin("entitlement-denied-nonmodular") + .systemProperty("es.entitlements.enabled", "true") + .setting("xpack.security.enabled", "false") + // Logs in libs/entitlement/qa/build/test-results/javaRestTest/TEST-org.elasticsearch.entitlement.qa.EntitlementsDeniedIT.xml + .setting("logger.org.elasticsearch.entitlement", "TRACE") + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + private final String pathPrefix; + private final String actionName; + + public EntitlementsDeniedIT(@Name("pathPrefix") String pathPrefix, @Name("actionName") String actionName) { + this.pathPrefix = pathPrefix; + this.actionName = actionName; + } + + @ParametersFactory + public static Iterable data() { + return Stream.of("denied", "denied_nonmodular") + .flatMap(path -> RestEntitlementsCheckAction.getAllCheckActions().stream().map(action -> new Object[] { path, action })) + .toList(); + } + + public void testCheckThrows() { + logger.info("Executing Entitlement test [{}] for [{}]", pathPrefix, actionName); + var exception = expectThrows(IOException.class, () -> { + var request = new Request("GET", "/_entitlement/" + pathPrefix + "/_check"); + request.addParameter("action", actionName); + client().performRequest(request); + }); + assertThat(exception.getMessage(), containsString("not_entitled_exception")); + } +} diff --git a/libs/entitlement/src/main/java/module-info.java b/libs/entitlement/src/main/java/module-info.java new file mode 100644 index 0000000000000..b8a125b98e641 --- /dev/null +++ b/libs/entitlement/src/main/java/module-info.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +module org.elasticsearch.entitlement { + requires org.elasticsearch.xcontent; + requires org.elasticsearch.logging; + requires java.instrument; + requires org.elasticsearch.base; + requires jdk.attach; + + requires static org.elasticsearch.entitlement.bridge; // At runtime, this will be in java.base + + exports org.elasticsearch.entitlement.runtime.api; + exports org.elasticsearch.entitlement.runtime.policy; + exports org.elasticsearch.entitlement.instrumentation; + exports org.elasticsearch.entitlement.bootstrap to org.elasticsearch.server; + exports org.elasticsearch.entitlement.initialization to java.base; + + uses org.elasticsearch.entitlement.instrumentation.InstrumentationService; +} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java new file mode 100644 index 0000000000000..257d130302580 --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java @@ -0,0 +1,172 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.bootstrap; + +import com.sun.tools.attach.AgentInitializationException; +import com.sun.tools.attach.AgentLoadException; +import com.sun.tools.attach.AttachNotSupportedException; +import com.sun.tools.attach.VirtualMachine; + +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.entitlement.initialization.EntitlementInitialization; +import org.elasticsearch.entitlement.runtime.api.NotEntitledException; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collection; +import java.util.function.Function; + +import static java.util.Objects.requireNonNull; + +public class EntitlementBootstrap { + + public record BootstrapArgs(Collection pluginData, Function, String> pluginResolver) { + public BootstrapArgs { + requireNonNull(pluginData); + requireNonNull(pluginResolver); + } + } + + public record PluginData(Path pluginPath, boolean isModular, boolean isExternalPlugin) { + public PluginData { + requireNonNull(pluginPath); + } + } + + private static BootstrapArgs bootstrapArgs; + + public static BootstrapArgs bootstrapArgs() { + return bootstrapArgs; + } + + /** + * Activates entitlement checking. Once this method returns, calls to methods protected by Entitlements from classes without a valid + * policy will throw {@link org.elasticsearch.entitlement.runtime.api.NotEntitledException}. + * @param pluginData a collection of (plugin path, boolean, boolean), that holds the paths of all the installed Elasticsearch modules + * and plugins, whether they are Java modular or not, and whether they are Elasticsearch modules or external plugins. + * @param pluginResolver a functor to map a Java Class to the plugin it belongs to (the plugin name). + */ + public static void bootstrap(Collection pluginData, Function, String> pluginResolver) { + logger.debug("Loading entitlement agent"); + if (EntitlementBootstrap.bootstrapArgs != null) { + throw new IllegalStateException("plugin data is already set"); + } + EntitlementBootstrap.bootstrapArgs = new BootstrapArgs(pluginData, pluginResolver); + exportInitializationToAgent(); + loadAgent(findAgentJar()); + selfTest(); + } + + @SuppressForbidden(reason = "The VirtualMachine API is the only way to attach a java agent dynamically") + private static void loadAgent(String agentPath) { + try { + VirtualMachine vm = VirtualMachine.attach(Long.toString(ProcessHandle.current().pid())); + try { + vm.loadAgent(agentPath); + } finally { + vm.detach(); + } + } catch (AttachNotSupportedException | IOException | AgentLoadException | AgentInitializationException e) { + throw new IllegalStateException("Unable to attach entitlement agent", e); + } + } + + private static void exportInitializationToAgent() { + String initPkg = EntitlementInitialization.class.getPackageName(); + // agent will live in unnamed module + Module unnamedModule = ClassLoader.getSystemClassLoader().getUnnamedModule(); + EntitlementInitialization.class.getModule().addExports(initPkg, unnamedModule); + } + + private static String findAgentJar() { + String propertyName = "es.entitlement.agentJar"; + String propertyValue = System.getProperty(propertyName); + if (propertyValue != null) { + return propertyValue; + } + + Path dir = Path.of("lib", "entitlement-agent"); + if (Files.exists(dir) == false) { + throw new IllegalStateException("Directory for entitlement jar does not exist: " + dir); + } + try (var s = Files.list(dir)) { + var candidates = s.limit(2).toList(); + if (candidates.size() != 1) { + throw new IllegalStateException("Expected one jar in " + dir + "; found " + candidates.size()); + } + return candidates.get(0).toString(); + } catch (IOException e) { + throw new IllegalStateException("Failed to list entitlement jars in: " + dir, e); + } + } + + /** + * Attempt a few sensitive operations to ensure that some are permitted and some are forbidden. + *

    + * + * This serves two purposes: + * + *

      + *
    1. + * a smoke test to make sure the entitlements system is not completely broken, and + *
    2. + *
    3. + * an early test of certain important operations so they don't fail later on at an awkward time. + *
    4. + *
    + * + * @throws IllegalStateException if the entitlements system can't prevent an unauthorized action of our choosing + */ + private static void selfTest() { + ensureCannotStartProcess(); + ensureCanCreateTempFile(); + } + + private static void ensureCannotStartProcess() { + try { + // The command doesn't matter; it doesn't even need to exist + new ProcessBuilder("").start(); + } catch (NotEntitledException e) { + logger.debug("Success: Entitlement protection correctly prevented process creation"); + return; + } catch (IOException e) { + throw new IllegalStateException("Failed entitlement protection self-test", e); + } + throw new IllegalStateException("Entitlement protection self-test was incorrectly permitted"); + } + + /** + * Originally {@code Security.selfTest}. + */ + @SuppressForbidden(reason = "accesses jvm default tempdir as a self-test") + private static void ensureCanCreateTempFile() { + try { + Path p = Files.createTempFile(null, null); + p.toFile().deleteOnExit(); + + // Make an effort to clean up the file immediately; also, deleteOnExit leaves the file if the JVM exits abnormally. + try { + Files.delete(p); + } catch (IOException ignored) { + // Can be caused by virus scanner + } + } catch (NotEntitledException e) { + throw new IllegalStateException("Entitlement protection self-test was incorrectly forbidden", e); + } catch (Exception e) { + throw new IllegalStateException("Unable to perform entitlement protection self-test", e); + } + logger.debug("Success: Entitlement protection correctly permitted temp file creation"); + } + + private static final Logger logger = LogManager.getLogger(EntitlementBootstrap.class); +} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java new file mode 100644 index 0000000000000..ba5ccbafa70ae --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java @@ -0,0 +1,201 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.initialization; + +import org.elasticsearch.core.Strings; +import org.elasticsearch.core.internal.provider.ProviderLocator; +import org.elasticsearch.entitlement.bootstrap.EntitlementBootstrap; +import org.elasticsearch.entitlement.bridge.EntitlementChecker; +import org.elasticsearch.entitlement.instrumentation.CheckMethod; +import org.elasticsearch.entitlement.instrumentation.InstrumentationService; +import org.elasticsearch.entitlement.instrumentation.Instrumenter; +import org.elasticsearch.entitlement.instrumentation.MethodKey; +import org.elasticsearch.entitlement.instrumentation.Transformer; +import org.elasticsearch.entitlement.runtime.api.ElasticsearchEntitlementChecker; +import org.elasticsearch.entitlement.runtime.policy.CreateClassLoaderEntitlement; +import org.elasticsearch.entitlement.runtime.policy.Entitlement; +import org.elasticsearch.entitlement.runtime.policy.ExitVMEntitlement; +import org.elasticsearch.entitlement.runtime.policy.Policy; +import org.elasticsearch.entitlement.runtime.policy.PolicyManager; +import org.elasticsearch.entitlement.runtime.policy.PolicyParser; +import org.elasticsearch.entitlement.runtime.policy.Scope; + +import java.io.IOException; +import java.lang.instrument.Instrumentation; +import java.lang.module.ModuleFinder; +import java.lang.module.ModuleReference; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.elasticsearch.entitlement.runtime.policy.PolicyManager.ALL_UNNAMED; + +/** + * Called by the agent during {@code agentmain} to configure the entitlement system, + * instantiate and configure an {@link EntitlementChecker}, + * make it available to the bootstrap library via {@link #checker()}, + * and then install the {@link org.elasticsearch.entitlement.instrumentation.Instrumenter} + * to begin injecting our instrumentation. + */ +public class EntitlementInitialization { + + private static final String POLICY_FILE_NAME = "entitlement-policy.yaml"; + private static final Module ENTITLEMENTS_MODULE = PolicyManager.class.getModule(); + + private static ElasticsearchEntitlementChecker manager; + + // Note: referenced by bridge reflectively + public static EntitlementChecker checker() { + return manager; + } + + // Note: referenced by agent reflectively + public static void initialize(Instrumentation inst) throws Exception { + manager = initChecker(); + + Map checkMethods = INSTRUMENTER_FACTORY.lookupMethods(EntitlementChecker.class); + + var classesToTransform = checkMethods.keySet().stream().map(MethodKey::className).collect(Collectors.toSet()); + + Instrumenter instrumenter = INSTRUMENTER_FACTORY.newInstrumenter(EntitlementChecker.class, checkMethods); + inst.addTransformer(new Transformer(instrumenter, classesToTransform), true); + inst.retransformClasses(findClassesToRetransform(inst.getAllLoadedClasses(), classesToTransform)); + } + + private static Class[] findClassesToRetransform(Class[] loadedClasses, Set classesToTransform) { + List> retransform = new ArrayList<>(); + for (Class loadedClass : loadedClasses) { + if (classesToTransform.contains(loadedClass.getName().replace(".", "/"))) { + retransform.add(loadedClass); + } + } + return retransform.toArray(new Class[0]); + } + + private static PolicyManager createPolicyManager() throws IOException { + Map pluginPolicies = createPluginPolicies(EntitlementBootstrap.bootstrapArgs().pluginData()); + + // TODO(ES-10031): Decide what goes in the elasticsearch default policy and extend it + var serverPolicy = new Policy( + "server", + List.of( + new Scope("org.elasticsearch.base", List.of(new CreateClassLoaderEntitlement())), + new Scope("org.elasticsearch.xcontent", List.of(new CreateClassLoaderEntitlement())), + new Scope("org.elasticsearch.server", List.of(new ExitVMEntitlement(), new CreateClassLoaderEntitlement())) + ) + ); + // agents run without a module, so this is a special hack for the apm agent + // this should be removed once https://github.com/elastic/elasticsearch/issues/109335 is completed + List agentEntitlements = List.of(new CreateClassLoaderEntitlement()); + var resolver = EntitlementBootstrap.bootstrapArgs().pluginResolver(); + return new PolicyManager(serverPolicy, agentEntitlements, pluginPolicies, resolver, ENTITLEMENTS_MODULE); + } + + private static Map createPluginPolicies(Collection pluginData) throws IOException { + Map pluginPolicies = new HashMap<>(pluginData.size()); + for (var entry : pluginData) { + Path pluginRoot = entry.pluginPath(); + String pluginName = pluginRoot.getFileName().toString(); + + final Policy policy = loadPluginPolicy(pluginRoot, entry.isModular(), pluginName, entry.isExternalPlugin()); + + pluginPolicies.put(pluginName, policy); + } + return pluginPolicies; + } + + private static Policy loadPluginPolicy(Path pluginRoot, boolean isModular, String pluginName, boolean isExternalPlugin) + throws IOException { + Path policyFile = pluginRoot.resolve(POLICY_FILE_NAME); + + final Set moduleNames = getModuleNames(pluginRoot, isModular); + final Policy policy = parsePolicyIfExists(pluginName, policyFile, isExternalPlugin); + + // TODO: should this check actually be part of the parser? + for (Scope scope : policy.scopes()) { + if (moduleNames.contains(scope.moduleName()) == false) { + throw new IllegalStateException( + Strings.format( + "Invalid module name in policy: plugin [%s] does not have module [%s]; available modules [%s]; policy file [%s]", + pluginName, + scope.moduleName(), + String.join(", ", moduleNames), + policyFile + ) + ); + } + } + return policy; + } + + private static Policy parsePolicyIfExists(String pluginName, Path policyFile, boolean isExternalPlugin) throws IOException { + if (Files.exists(policyFile)) { + return new PolicyParser(Files.newInputStream(policyFile, StandardOpenOption.READ), pluginName, isExternalPlugin).parsePolicy(); + } + return new Policy(pluginName, List.of()); + } + + private static Set getModuleNames(Path pluginRoot, boolean isModular) { + if (isModular) { + ModuleFinder moduleFinder = ModuleFinder.of(pluginRoot); + Set moduleReferences = moduleFinder.findAll(); + + return moduleReferences.stream().map(mr -> mr.descriptor().name()).collect(Collectors.toUnmodifiableSet()); + } + // When isModular == false we use the same "ALL-UNNAMED" constant as the JDK to indicate (any) unnamed module for this plugin + return Set.of(ALL_UNNAMED); + } + + private static ElasticsearchEntitlementChecker initChecker() throws IOException { + final PolicyManager policyManager = createPolicyManager(); + + int javaVersion = Runtime.version().feature(); + final String classNamePrefix; + if (javaVersion >= 23) { + classNamePrefix = "Java23"; + } else { + classNamePrefix = ""; + } + final String className = "org.elasticsearch.entitlement.runtime.api." + classNamePrefix + "ElasticsearchEntitlementChecker"; + Class clazz; + try { + clazz = Class.forName(className); + } catch (ClassNotFoundException e) { + throw new AssertionError("entitlement lib cannot find entitlement impl", e); + } + Constructor constructor; + try { + constructor = clazz.getConstructor(PolicyManager.class); + } catch (NoSuchMethodException e) { + throw new AssertionError("entitlement impl is missing no arg constructor", e); + } + try { + return (ElasticsearchEntitlementChecker) constructor.newInstance(policyManager); + } catch (IllegalAccessException | InvocationTargetException | InstantiationException e) { + throw new AssertionError(e); + } + } + + private static final InstrumentationService INSTRUMENTER_FACTORY = new ProviderLocator<>( + "entitlement", + InstrumentationService.class, + "org.elasticsearch.entitlement.instrumentation", + Set.of() + ).get(); +} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/CheckMethod.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/CheckMethod.java new file mode 100644 index 0000000000000..384d455c7a34b --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/CheckMethod.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.instrumentation; + +import java.util.List; + +/** + * A structure to use as a representation of the checkXxx method the instrumentation will inject. + * + * @param className the "internal name" of the class: includes the package info, but with periods replaced by slashes + * @param methodName the checker method name + * @param parameterDescriptors a list of + * type descriptors) + * for methodName parameters. + */ +public record CheckMethod(String className, String methodName, List parameterDescriptors) {} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/InstrumentationService.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/InstrumentationService.java new file mode 100644 index 0000000000000..66d8ad9488cfa --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/InstrumentationService.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.instrumentation; + +import java.io.IOException; +import java.util.Map; + +/** + * The SPI service entry point for instrumentation. + */ +public interface InstrumentationService { + Instrumenter newInstrumenter(Class clazz, Map methods); + + Map lookupMethods(Class clazz) throws IOException; +} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/Instrumenter.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/Instrumenter.java new file mode 100644 index 0000000000000..9f39cbbbd0df0 --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/Instrumenter.java @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.instrumentation; + +public interface Instrumenter { + byte[] instrumentClass(String className, byte[] classfileBuffer); +} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/MethodKey.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/MethodKey.java new file mode 100644 index 0000000000000..caaa10bec52dd --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/MethodKey.java @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.instrumentation; + +import java.util.List; + +/** + * A structure to use as a key/lookup for a method target of instrumentation + * + * @param className the "internal name" of the class: includes the package info, but with periods replaced by slashes + * @param methodName the method name + * @param parameterTypes a list of "internal names" for the parameter types that appear in the method's descriptor (not the receiver) + */ +public record MethodKey(String className, String methodName, List parameterTypes) {} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/Transformer.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/Transformer.java new file mode 100644 index 0000000000000..c6512ee975dbf --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/Transformer.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.instrumentation; + +import java.lang.instrument.ClassFileTransformer; +import java.security.ProtectionDomain; +import java.util.Set; + +/** + * A {@link ClassFileTransformer} that applies an {@link Instrumenter} to the appropriate classes. + */ +public class Transformer implements ClassFileTransformer { + private final Instrumenter instrumenter; + private final Set classesToTransform; + + public Transformer(Instrumenter instrumenter, Set classesToTransform) { + this.instrumenter = instrumenter; + this.classesToTransform = classesToTransform; + // TODO: Should warn if any MethodKey doesn't match any methods + } + + @Override + public byte[] transform( + ClassLoader loader, + String className, + Class classBeingRedefined, + ProtectionDomain protectionDomain, + byte[] classfileBuffer + ) { + if (classesToTransform.contains(className)) { + // System.out.println("Transforming " + className); + return instrumenter.instrumentClass(className, classfileBuffer); + } else { + // System.out.println("Not transforming " + className); + return classfileBuffer; + } + } + + // private static final Logger LOGGER = LogManager.getLogger(Transformer.class); +} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java new file mode 100644 index 0000000000000..450786ee57d86 --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java @@ -0,0 +1,141 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.api; + +import org.elasticsearch.entitlement.bridge.EntitlementChecker; +import org.elasticsearch.entitlement.runtime.policy.PolicyManager; + +import java.net.URL; +import java.net.URLStreamHandlerFactory; +import java.util.List; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocketFactory; + +/** + * Implementation of the {@link EntitlementChecker} interface, providing additional + * API methods for managing the checks. + * The trampoline module loads this object via SPI. + */ +public class ElasticsearchEntitlementChecker implements EntitlementChecker { + + private final PolicyManager policyManager; + + public ElasticsearchEntitlementChecker(PolicyManager policyManager) { + this.policyManager = policyManager; + } + + @Override + public void check$java_lang_Runtime$exit(Class callerClass, Runtime runtime, int status) { + policyManager.checkExitVM(callerClass); + } + + @Override + public void check$java_lang_Runtime$halt(Class callerClass, Runtime runtime, int status) { + policyManager.checkExitVM(callerClass); + } + + @Override + public void check$java_lang_ClassLoader$(Class callerClass) { + policyManager.checkCreateClassLoader(callerClass); + } + + @Override + public void check$java_lang_ClassLoader$(Class callerClass, ClassLoader parent) { + policyManager.checkCreateClassLoader(callerClass); + } + + @Override + public void check$java_lang_ClassLoader$(Class callerClass, String name, ClassLoader parent) { + policyManager.checkCreateClassLoader(callerClass); + } + + @Override + public void check$java_security_SecureClassLoader$(Class callerClass) { + policyManager.checkCreateClassLoader(callerClass); + } + + @Override + public void check$java_security_SecureClassLoader$(Class callerClass, ClassLoader parent) { + policyManager.checkCreateClassLoader(callerClass); + } + + @Override + public void check$java_security_SecureClassLoader$(Class callerClass, String name, ClassLoader parent) { + policyManager.checkCreateClassLoader(callerClass); + } + + @Override + public void check$java_net_URLClassLoader$(Class callerClass, URL[] urls) { + policyManager.checkCreateClassLoader(callerClass); + } + + @Override + public void check$java_net_URLClassLoader$(Class callerClass, URL[] urls, ClassLoader parent) { + policyManager.checkCreateClassLoader(callerClass); + } + + @Override + public void check$java_net_URLClassLoader$(Class callerClass, URL[] urls, ClassLoader parent, URLStreamHandlerFactory factory) { + policyManager.checkCreateClassLoader(callerClass); + } + + @Override + public void check$java_net_URLClassLoader$(Class callerClass, String name, URL[] urls, ClassLoader parent) { + policyManager.checkCreateClassLoader(callerClass); + } + + @Override + public void check$java_net_URLClassLoader$( + Class callerClass, + String name, + URL[] urls, + ClassLoader parent, + URLStreamHandlerFactory factory + ) { + policyManager.checkCreateClassLoader(callerClass); + } + + @Override + public void check$java_lang_ProcessBuilder$start(Class callerClass, ProcessBuilder processBuilder) { + policyManager.checkStartProcess(callerClass); + } + + @Override + public void check$java_lang_ProcessBuilder$$startPipeline(Class callerClass, List builders) { + policyManager.checkStartProcess(callerClass); + } + + @Override + public void check$javax_net_ssl_HttpsURLConnection$setSSLSocketFactory( + Class callerClass, + HttpsURLConnection connection, + SSLSocketFactory sf + ) { + policyManager.checkSetHttpsConnectionProperties(callerClass); + } + + @Override + public void check$javax_net_ssl_HttpsURLConnection$$setDefaultSSLSocketFactory(Class callerClass, SSLSocketFactory sf) { + policyManager.checkSetGlobalHttpsConnectionProperties(callerClass); + } + + @Override + public void check$javax_net_ssl_HttpsURLConnection$$setDefaultHostnameVerifier(Class callerClass, HostnameVerifier hv) { + policyManager.checkSetGlobalHttpsConnectionProperties(callerClass); + } + + @Override + public void check$javax_net_ssl_SSLContext$$setDefault(Class callerClass, SSLContext context) { + policyManager.checkSetGlobalHttpsConnectionProperties(callerClass); + } +} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/NotEntitledException.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/NotEntitledException.java new file mode 100644 index 0000000000000..5afffc84f77a8 --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/NotEntitledException.java @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.api; + +public class NotEntitledException extends RuntimeException { + public NotEntitledException(String message) { + super(message); + } + + public NotEntitledException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/CreateClassLoaderEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/CreateClassLoaderEntitlement.java new file mode 100644 index 0000000000000..55e4b66595642 --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/CreateClassLoaderEntitlement.java @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +public record CreateClassLoaderEntitlement() implements Entitlement { + @ExternalEntitlement + public CreateClassLoaderEntitlement {} +} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Entitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Entitlement.java new file mode 100644 index 0000000000000..5b53c399cc1b7 --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Entitlement.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +/** + * Marker interface to ensure that only {@link Entitlement} are + * part of a {@link Policy}. All entitlement classes should implement + * this. + */ +public interface Entitlement { + +} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExitVMEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExitVMEntitlement.java new file mode 100644 index 0000000000000..e5c836ea22b20 --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExitVMEntitlement.java @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +/** + * Internal policy type (not-parseable -- not available to plugins). + */ +public record ExitVMEntitlement() implements Entitlement {} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExternalEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExternalEntitlement.java new file mode 100644 index 0000000000000..768babdb840f5 --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExternalEntitlement.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * This annotation indicates an {@link Entitlement} is available + * to "external" classes such as those used in plugins. Any {@link Entitlement} + * using this annotation is considered parseable as part of a policy file + * for entitlements. + */ +@Target(ElementType.CONSTRUCTOR) +@Retention(RetentionPolicy.RUNTIME) +public @interface ExternalEntitlement { + + /** + * This is the list of parameter names that are + * parseable in {@link PolicyParser#parseEntitlement(String, String)}. + * The number and order of parameter names much match the number and order + * of constructor parameters as this is how the parser will pass in the + * parsed values from a policy file. However, the names themselves do NOT + * have to match the parameter names of the constructor. + */ + String[] parameterNames() default {}; + + /** + * This flag indicates if this Entitlement can be used in external plugins, + * or if it can be used only in Elasticsearch modules ("internal" plugins). + * Using an entitlement that is not {@code pluginsAccessible} in an external + * plugin policy will throw in exception while parsing. + */ + boolean esModulesOnly() default true; +} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileEntitlement.java new file mode 100644 index 0000000000000..4fdbcc93ea6e0 --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileEntitlement.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import java.util.List; +import java.util.Objects; + +/** + * Describes a file entitlement with a path and actions. + */ +public class FileEntitlement implements Entitlement { + + public static final int READ_ACTION = 0x1; + public static final int WRITE_ACTION = 0x2; + + public static final String READ = "read"; + public static final String WRITE = "write"; + + private final String path; + private final int actions; + + @ExternalEntitlement(parameterNames = { "path", "actions" }, esModulesOnly = false) + public FileEntitlement(String path, List actionsList) { + this.path = path; + int actionsInt = 0; + + for (String actionString : actionsList) { + if (READ.equals(actionString)) { + if ((actionsInt & READ_ACTION) == READ_ACTION) { + throw new IllegalArgumentException("file action [read] specified multiple times"); + } + actionsInt |= READ_ACTION; + } else if (WRITE.equals(actionString)) { + if ((actionsInt & WRITE_ACTION) == WRITE_ACTION) { + throw new IllegalArgumentException("file action [write] specified multiple times"); + } + actionsInt |= WRITE_ACTION; + } else { + throw new IllegalArgumentException("unknown file action [" + actionString + "]"); + } + } + + this.actions = actionsInt; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + FileEntitlement that = (FileEntitlement) o; + return actions == that.actions && Objects.equals(path, that.path); + } + + @Override + public int hashCode() { + return Objects.hash(path, actions); + } + + @Override + public String toString() { + return "FileEntitlement{" + "path='" + path + '\'' + ", actions=" + actions + '}'; + } +} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Policy.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Policy.java new file mode 100644 index 0000000000000..3546472f485fb --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Policy.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import java.util.List; +import java.util.Objects; + +/** + * A holder for scoped entitlements. + */ +public record Policy(String name, List scopes) { + public Policy(String name, List scopes) { + this.name = Objects.requireNonNull(name); + this.scopes = List.copyOf(scopes); + } +} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java new file mode 100644 index 0000000000000..188ce1d747db6 --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java @@ -0,0 +1,274 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import org.elasticsearch.core.Strings; +import org.elasticsearch.entitlement.runtime.api.NotEntitledException; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; + +import java.lang.StackWalker.StackFrame; +import java.lang.module.ModuleFinder; +import java.lang.module.ModuleReference; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.lang.StackWalker.Option.RETAIN_CLASS_REFERENCE; +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.groupingBy; + +public class PolicyManager { + private static final Logger logger = LogManager.getLogger(PolicyManager.class); + + record ModuleEntitlements(Map, List> entitlementsByType) { + public static final ModuleEntitlements NONE = new ModuleEntitlements(Map.of()); + + ModuleEntitlements { + entitlementsByType = Map.copyOf(entitlementsByType); + } + + public static ModuleEntitlements from(List entitlements) { + return new ModuleEntitlements(entitlements.stream().collect(groupingBy(Entitlement::getClass))); + } + + public boolean hasEntitlement(Class entitlementClass) { + return entitlementsByType.containsKey(entitlementClass); + } + + public Stream getEntitlements(Class entitlementClass) { + return entitlementsByType.get(entitlementClass).stream().map(entitlementClass::cast); + } + } + + final Map moduleEntitlementsMap = new ConcurrentHashMap<>(); + + protected final Map> serverEntitlements; + protected final List agentEntitlements; + protected final Map>> pluginsEntitlements; + private final Function, String> pluginResolver; + + public static final String ALL_UNNAMED = "ALL-UNNAMED"; + + private static final Set systemModules = findSystemModules(); + + private static Set findSystemModules() { + var systemModulesDescriptors = ModuleFinder.ofSystem() + .findAll() + .stream() + .map(ModuleReference::descriptor) + .collect(Collectors.toUnmodifiableSet()); + return ModuleLayer.boot() + .modules() + .stream() + .filter(m -> systemModulesDescriptors.contains(m.getDescriptor())) + .collect(Collectors.toUnmodifiableSet()); + } + + /** + * Frames originating from this module are ignored in the permission logic. + */ + private final Module entitlementsModule; + + public PolicyManager( + Policy serverPolicy, + List agentEntitlements, + Map pluginPolicies, + Function, String> pluginResolver, + Module entitlementsModule + ) { + this.serverEntitlements = buildScopeEntitlementsMap(requireNonNull(serverPolicy)); + this.agentEntitlements = agentEntitlements; + this.pluginsEntitlements = requireNonNull(pluginPolicies).entrySet() + .stream() + .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, e -> buildScopeEntitlementsMap(e.getValue()))); + this.pluginResolver = pluginResolver; + this.entitlementsModule = entitlementsModule; + } + + private static Map> buildScopeEntitlementsMap(Policy policy) { + return policy.scopes().stream().collect(Collectors.toUnmodifiableMap(scope -> scope.moduleName(), scope -> scope.entitlements())); + } + + public void checkStartProcess(Class callerClass) { + neverEntitled(callerClass, "start process"); + } + + private void neverEntitled(Class callerClass, String operationDescription) { + var requestingModule = requestingClass(callerClass); + if (isTriviallyAllowed(requestingModule)) { + return; + } + + throw new NotEntitledException( + Strings.format( + "Not entitled: caller [%s], module [%s], operation [%s]", + callerClass, + requestingModule.getName(), + operationDescription + ) + ); + } + + public void checkExitVM(Class callerClass) { + checkEntitlementPresent(callerClass, ExitVMEntitlement.class); + } + + public void checkCreateClassLoader(Class callerClass) { + checkEntitlementPresent(callerClass, CreateClassLoaderEntitlement.class); + } + + public void checkSetHttpsConnectionProperties(Class callerClass) { + checkEntitlementPresent(callerClass, SetHttpsConnectionPropertiesEntitlement.class); + } + + public void checkSetGlobalHttpsConnectionProperties(Class callerClass) { + neverEntitled(callerClass, "set global https connection properties"); + } + + private void checkEntitlementPresent(Class callerClass, Class entitlementClass) { + var requestingClass = requestingClass(callerClass); + if (isTriviallyAllowed(requestingClass)) { + return; + } + + ModuleEntitlements entitlements = getEntitlements(requestingClass); + if (entitlements.hasEntitlement(entitlementClass)) { + logger.debug( + () -> Strings.format( + "Entitled: class [%s], module [%s], entitlement [%s]", + requestingClass, + requestingClass.getModule().getName(), + entitlementClass.getSimpleName() + ) + ); + return; + } + throw new NotEntitledException( + Strings.format( + "Missing entitlement: class [%s], module [%s], entitlement [%s]", + requestingClass, + requestingClass.getModule().getName(), + entitlementClass.getSimpleName() + ) + ); + } + + ModuleEntitlements getEntitlements(Class requestingClass) { + return moduleEntitlementsMap.computeIfAbsent(requestingClass.getModule(), m -> computeEntitlements(requestingClass)); + } + + private ModuleEntitlements computeEntitlements(Class requestingClass) { + Module requestingModule = requestingClass.getModule(); + if (isServerModule(requestingModule)) { + return getModuleScopeEntitlements(requestingClass, serverEntitlements, requestingModule.getName()); + } + + // plugins + var pluginName = pluginResolver.apply(requestingClass); + if (pluginName != null) { + var pluginEntitlements = pluginsEntitlements.get(pluginName); + if (pluginEntitlements != null) { + final String scopeName; + if (requestingModule.isNamed() == false) { + scopeName = ALL_UNNAMED; + } else { + scopeName = requestingModule.getName(); + } + return getModuleScopeEntitlements(requestingClass, pluginEntitlements, scopeName); + } + } + + if (requestingModule.isNamed() == false) { + // agents are the only thing running non-modular + return ModuleEntitlements.from(agentEntitlements); + } + + logger.warn("No applicable entitlement policy for class [{}]", requestingClass.getName()); + return ModuleEntitlements.NONE; + } + + private ModuleEntitlements getModuleScopeEntitlements( + Class callerClass, + Map> scopeEntitlements, + String moduleName + ) { + var entitlements = scopeEntitlements.get(moduleName); + if (entitlements == null) { + logger.warn("No applicable entitlement policy for module [{}], class [{}]", moduleName, callerClass); + return ModuleEntitlements.NONE; + } + return ModuleEntitlements.from(entitlements); + } + + private static boolean isServerModule(Module requestingModule) { + return requestingModule.isNamed() && requestingModule.getLayer() == ModuleLayer.boot(); + } + + /** + * Walks the stack to determine which class should be checked for entitlements. + * + * @param callerClass when non-null will be returned; + * this is a fast-path check that can avoid the stack walk + * in cases where the caller class is available. + * @return the requesting class, or {@code null} if the entire call stack + * comes from the entitlement library itself. + */ + Class requestingClass(Class callerClass) { + if (callerClass != null) { + // fast path + return callerClass; + } + Optional> result = StackWalker.getInstance(RETAIN_CLASS_REFERENCE) + .walk(frames -> findRequestingClass(frames.map(StackFrame::getDeclaringClass))); + return result.orElse(null); + } + + /** + * Given a stream of classes corresponding to the frames from a {@link StackWalker}, + * returns the module whose entitlements should be checked. + * + * @throws NullPointerException if the requesting module is {@code null} + */ + Optional> findRequestingClass(Stream> classes) { + return classes.filter(c -> c.getModule() != entitlementsModule) // Ignore the entitlements library + .skip(1) // Skip the sensitive caller method + .findFirst(); + } + + /** + * @return true if permission is granted regardless of the entitlement + */ + private static boolean isTriviallyAllowed(Class requestingClass) { + if (logger.isTraceEnabled()) { + logger.trace("Stack trace for upcoming trivially-allowed check", new Exception()); + } + if (requestingClass == null) { + logger.debug("Entitlement trivially allowed: no caller frames outside the entitlement library"); + return true; + } + if (systemModules.contains(requestingClass.getModule())) { + logger.debug("Entitlement trivially allowed from system module [{}]", requestingClass.getModule().getName()); + return true; + } + logger.trace("Entitlement not trivially allowed"); + return false; + } + + @Override + public String toString() { + return "PolicyManager{" + "serverEntitlements=" + serverEntitlements + ", pluginsEntitlements=" + pluginsEntitlements + '}'; + } +} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java new file mode 100644 index 0000000000000..013acf8f22fae --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java @@ -0,0 +1,193 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.yaml.YamlXContent; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * A parser to parse policy files for entitlements. + */ +public class PolicyParser { + + private static final Map> EXTERNAL_ENTITLEMENTS = Stream.of( + FileEntitlement.class, + CreateClassLoaderEntitlement.class, + SetHttpsConnectionPropertiesEntitlement.class + ).collect(Collectors.toUnmodifiableMap(PolicyParser::getEntitlementTypeName, Function.identity())); + + protected final XContentParser policyParser; + protected final String policyName; + private final boolean isExternalPlugin; + + static String getEntitlementTypeName(Class entitlementClass) { + var entitlementClassName = entitlementClass.getSimpleName(); + + if (entitlementClassName.endsWith("Entitlement") == false) { + throw new IllegalArgumentException( + entitlementClassName + " is not a valid Entitlement class name. A valid class name must end with 'Entitlement'" + ); + } + + var strippedClassName = entitlementClassName.substring(0, entitlementClassName.indexOf("Entitlement")); + return Arrays.stream(strippedClassName.split("(?=\\p{Lu})")) + .filter(Predicate.not(String::isEmpty)) + .map(s -> s.toLowerCase(Locale.ROOT)) + .collect(Collectors.joining("_")); + } + + public PolicyParser(InputStream inputStream, String policyName, boolean isExternalPlugin) throws IOException { + this.policyParser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, Objects.requireNonNull(inputStream)); + this.policyName = policyName; + this.isExternalPlugin = isExternalPlugin; + } + + public Policy parsePolicy() { + try { + if (policyParser.nextToken() != XContentParser.Token.START_OBJECT) { + throw newPolicyParserException("expected object "); + } + List scopes = new ArrayList<>(); + while (policyParser.nextToken() != XContentParser.Token.END_OBJECT) { + if (policyParser.currentToken() != XContentParser.Token.FIELD_NAME) { + throw newPolicyParserException("expected object "); + } + String scopeName = policyParser.currentName(); + Scope scope = parseScope(scopeName); + scopes.add(scope); + } + return new Policy(policyName, scopes); + } catch (IOException ioe) { + throw new UncheckedIOException(ioe); + } + } + + protected Scope parseScope(String scopeName) throws IOException { + try { + if (policyParser.nextToken() != XContentParser.Token.START_ARRAY) { + throw newPolicyParserException(scopeName, "expected array of "); + } + List entitlements = new ArrayList<>(); + while (policyParser.nextToken() != XContentParser.Token.END_ARRAY) { + if (policyParser.currentToken() == XContentParser.Token.VALUE_STRING) { + String entitlementType = policyParser.text(); + Entitlement entitlement = parseEntitlement(scopeName, entitlementType); + entitlements.add(entitlement); + } else if (policyParser.currentToken() == XContentParser.Token.START_OBJECT) { + if (policyParser.nextToken() != XContentParser.Token.FIELD_NAME) { + throw newPolicyParserException(scopeName, "expected object "); + } + String entitlementType = policyParser.currentName(); + Entitlement entitlement = parseEntitlement(scopeName, entitlementType); + entitlements.add(entitlement); + if (policyParser.nextToken() != XContentParser.Token.END_OBJECT) { + throw newPolicyParserException(scopeName, "expected closing object"); + } + } else { + throw newPolicyParserException(scopeName, "expected object "); + } + } + return new Scope(scopeName, entitlements); + } catch (IOException ioe) { + throw new UncheckedIOException(ioe); + } + } + + protected Entitlement parseEntitlement(String scopeName, String entitlementType) throws IOException { + Class entitlementClass = EXTERNAL_ENTITLEMENTS.get(entitlementType); + + if (entitlementClass == null) { + throw newPolicyParserException(scopeName, "unknown entitlement type [" + entitlementType + "]"); + } + + Constructor entitlementConstructor = entitlementClass.getConstructors()[0]; + ExternalEntitlement entitlementMetadata = entitlementConstructor.getAnnotation(ExternalEntitlement.class); + if (entitlementMetadata == null) { + throw newPolicyParserException(scopeName, "unknown entitlement type [" + entitlementType + "]"); + } + + if (entitlementMetadata.esModulesOnly() && isExternalPlugin) { + throw newPolicyParserException("entitlement type [" + entitlementType + "] is allowed only on modules"); + } + + Class[] parameterTypes = entitlementConstructor.getParameterTypes(); + String[] parametersNames = entitlementMetadata.parameterNames(); + + if (parameterTypes.length != 0 || parametersNames.length != 0) { + if (policyParser.nextToken() != XContentParser.Token.START_OBJECT) { + throw newPolicyParserException(scopeName, entitlementType, "expected entitlement parameters"); + } + } + + Map parsedValues = policyParser.map(); + + Object[] parameterValues = new Object[parameterTypes.length]; + for (int parameterIndex = 0; parameterIndex < parameterTypes.length; ++parameterIndex) { + String parameterName = parametersNames[parameterIndex]; + Object parameterValue = parsedValues.remove(parameterName); + if (parameterValue == null) { + throw newPolicyParserException(scopeName, entitlementType, "missing entitlement parameter [" + parameterName + "]"); + } + Class parameterType = parameterTypes[parameterIndex]; + if (parameterType.isAssignableFrom(parameterValue.getClass()) == false) { + throw newPolicyParserException( + scopeName, + entitlementType, + "unexpected parameter type [" + parameterType.getSimpleName() + "] for entitlement parameter [" + parameterName + "]" + ); + } + parameterValues[parameterIndex] = parameterValue; + } + if (parsedValues.isEmpty() == false) { + throw newPolicyParserException(scopeName, entitlementType, "extraneous entitlement parameter(s) " + parsedValues); + } + + try { + return (Entitlement) entitlementConstructor.newInstance(parameterValues); + } catch (InvocationTargetException | InstantiationException | IllegalAccessException e) { + throw new IllegalStateException("internal error"); + } + } + + protected PolicyParserException newPolicyParserException(String message) { + return PolicyParserException.newPolicyParserException(policyParser.getTokenLocation(), policyName, message); + } + + protected PolicyParserException newPolicyParserException(String scopeName, String message) { + return PolicyParserException.newPolicyParserException(policyParser.getTokenLocation(), policyName, scopeName, message); + } + + protected PolicyParserException newPolicyParserException(String scopeName, String entitlementType, String message) { + return PolicyParserException.newPolicyParserException( + policyParser.getTokenLocation(), + policyName, + scopeName, + entitlementType, + message + ); + } +} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserException.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserException.java new file mode 100644 index 0000000000000..5dfa12f11d0be --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserException.java @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import org.elasticsearch.xcontent.XContentLocation; + +/** + * An exception specifically for policy parsing errors. + */ +public class PolicyParserException extends RuntimeException { + + public static PolicyParserException newPolicyParserException(XContentLocation location, String policyName, String message) { + return new PolicyParserException( + "[" + location.lineNumber() + ":" + location.columnNumber() + "] policy parsing error for [" + policyName + "]: " + message + ); + } + + public static PolicyParserException newPolicyParserException( + XContentLocation location, + String policyName, + String scopeName, + String message + ) { + if (scopeName == null) { + return new PolicyParserException( + "[" + location.lineNumber() + ":" + location.columnNumber() + "] policy parsing error for [" + policyName + "]: " + message + ); + } else { + return new PolicyParserException( + "[" + + location.lineNumber() + + ":" + + location.columnNumber() + + "] policy parsing error for [" + + policyName + + "] in scope [" + + scopeName + + "]: " + + message + ); + } + } + + public static PolicyParserException newPolicyParserException( + XContentLocation location, + String policyName, + String scopeName, + String entitlementType, + String message + ) { + if (scopeName == null) { + return new PolicyParserException( + "[" + + location.lineNumber() + + ":" + + location.columnNumber() + + "] policy parsing error for [" + + policyName + + "] for entitlement type [" + + entitlementType + + "]: " + + message + ); + } else { + return new PolicyParserException( + "[" + + location.lineNumber() + + ":" + + location.columnNumber() + + "] policy parsing error for [" + + policyName + + "] in scope [" + + scopeName + + "] for entitlement type [" + + entitlementType + + "]: " + + message + ); + } + } + + private PolicyParserException(String message) { + super(message); + } +} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Scope.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Scope.java new file mode 100644 index 0000000000000..55e257797d603 --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/Scope.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import java.util.List; +import java.util.Objects; + +/** + * A holder for entitlements within a single scope. + */ +public record Scope(String moduleName, List entitlements) { + + public Scope(String moduleName, List entitlements) { + this.moduleName = Objects.requireNonNull(moduleName); + this.entitlements = List.copyOf(entitlements); + } + +} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/SetHttpsConnectionPropertiesEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/SetHttpsConnectionPropertiesEntitlement.java new file mode 100644 index 0000000000000..bb2f65def9e18 --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/SetHttpsConnectionPropertiesEntitlement.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +/** + * An Entitlement to allow setting properties to a single Https connection after this has been created + */ +public record SetHttpsConnectionPropertiesEntitlement() implements Entitlement { + @ExternalEntitlement(esModulesOnly = false) + public SetHttpsConnectionPropertiesEntitlement {} +} diff --git a/libs/entitlement/src/main23/java/org/elasticsearch/entitlement/runtime/api/Java23ElasticsearchEntitlementChecker.java b/libs/entitlement/src/main23/java/org/elasticsearch/entitlement/runtime/api/Java23ElasticsearchEntitlementChecker.java new file mode 100644 index 0000000000000..706a6649de329 --- /dev/null +++ b/libs/entitlement/src/main23/java/org/elasticsearch/entitlement/runtime/api/Java23ElasticsearchEntitlementChecker.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.api; + +import org.elasticsearch.entitlement.bridge.Java23EntitlementChecker; +import org.elasticsearch.entitlement.runtime.policy.PolicyManager; + +public class Java23ElasticsearchEntitlementChecker extends ElasticsearchEntitlementChecker implements Java23EntitlementChecker { + + public Java23ElasticsearchEntitlementChecker(PolicyManager policyManager) { + super(policyManager); + } + + @Override + public void check$java_lang_Runtime$exit(Class callerClass, Runtime runtime, int status) { + // TODO: this is just an example, we shouldn't really override a method implemented in the superclass + super.check$java_lang_Runtime$exit(callerClass, runtime, status); + } +} diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java new file mode 100644 index 0000000000000..d22c2f598e344 --- /dev/null +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java @@ -0,0 +1,292 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import org.elasticsearch.entitlement.runtime.policy.PolicyManager.ModuleEntitlements; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.compiler.InMemoryJavaCompiler; +import org.elasticsearch.test.jar.JarUtils; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.lang.module.Configuration; +import java.lang.module.ModuleFinder; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + +import static java.util.Map.entry; +import static org.elasticsearch.entitlement.runtime.policy.PolicyManager.ALL_UNNAMED; +import static org.elasticsearch.test.LambdaMatchers.transformedMatch; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.sameInstance; + +@ESTestCase.WithoutSecurityManager +public class PolicyManagerTests extends ESTestCase { + /** + * A module you can use for test cases that don't actually care about the + * entitlements module. + */ + private static Module NO_ENTITLEMENTS_MODULE; + + @BeforeClass + public static void beforeClass() { + try { + // Any old module will do for tests using NO_ENTITLEMENTS_MODULE + NO_ENTITLEMENTS_MODULE = makeClassInItsOwnModule().getModule(); + } catch (Exception e) { + throw new IllegalStateException(e); + } + + } + + public void testGetEntitlementsThrowsOnMissingPluginUnnamedModule() { + var policyManager = new PolicyManager( + createEmptyTestServerPolicy(), + List.of(), + Map.of("plugin1", createPluginPolicy("plugin.module")), + c -> "plugin1", + NO_ENTITLEMENTS_MODULE + ); + + // Any class from the current module (unnamed) will do + var callerClass = this.getClass(); + var requestingModule = callerClass.getModule(); + + assertEquals("No policy for the unnamed module", ModuleEntitlements.NONE, policyManager.getEntitlements(callerClass)); + + assertEquals(Map.of(requestingModule, ModuleEntitlements.NONE), policyManager.moduleEntitlementsMap); + } + + public void testGetEntitlementsThrowsOnMissingPolicyForPlugin() { + var policyManager = new PolicyManager(createEmptyTestServerPolicy(), List.of(), Map.of(), c -> "plugin1", NO_ENTITLEMENTS_MODULE); + + // Any class from the current module (unnamed) will do + var callerClass = this.getClass(); + var requestingModule = callerClass.getModule(); + + assertEquals("No policy for this plugin", ModuleEntitlements.NONE, policyManager.getEntitlements(callerClass)); + + assertEquals(Map.of(requestingModule, ModuleEntitlements.NONE), policyManager.moduleEntitlementsMap); + } + + public void testGetEntitlementsFailureIsCached() { + var policyManager = new PolicyManager(createEmptyTestServerPolicy(), List.of(), Map.of(), c -> "plugin1", NO_ENTITLEMENTS_MODULE); + + // Any class from the current module (unnamed) will do + var callerClass = this.getClass(); + var requestingModule = callerClass.getModule(); + + assertEquals(ModuleEntitlements.NONE, policyManager.getEntitlements(callerClass)); + assertEquals(Map.of(requestingModule, ModuleEntitlements.NONE), policyManager.moduleEntitlementsMap); + + // A second time + assertEquals(ModuleEntitlements.NONE, policyManager.getEntitlements(callerClass)); + + // Nothing new in the map + assertEquals(Map.of(requestingModule, ModuleEntitlements.NONE), policyManager.moduleEntitlementsMap); + } + + public void testGetEntitlementsReturnsEntitlementsForPluginUnnamedModule() { + var policyManager = new PolicyManager( + createEmptyTestServerPolicy(), + List.of(), + Map.ofEntries(entry("plugin2", createPluginPolicy(ALL_UNNAMED))), + c -> "plugin2", + NO_ENTITLEMENTS_MODULE + ); + + // Any class from the current module (unnamed) will do + var callerClass = this.getClass(); + + var entitlements = policyManager.getEntitlements(callerClass); + assertThat(entitlements.hasEntitlement(CreateClassLoaderEntitlement.class), is(true)); + } + + public void testGetEntitlementsThrowsOnMissingPolicyForServer() throws ClassNotFoundException { + var policyManager = new PolicyManager(createTestServerPolicy("example"), List.of(), Map.of(), c -> null, NO_ENTITLEMENTS_MODULE); + + // Tests do not run modular, so we cannot use a server class. + // But we know that in production code the server module and its classes are in the boot layer. + // So we use a random module in the boot layer, and a random class from that module (not java.base -- it is + // loaded too early) to mimic a class that would be in the server module. + var mockServerClass = ModuleLayer.boot().findLoader("jdk.httpserver").loadClass("com.sun.net.httpserver.HttpServer"); + var requestingModule = mockServerClass.getModule(); + + assertEquals("No policy for this module in server", ModuleEntitlements.NONE, policyManager.getEntitlements(mockServerClass)); + + assertEquals(Map.of(requestingModule, ModuleEntitlements.NONE), policyManager.moduleEntitlementsMap); + } + + public void testGetEntitlementsReturnsEntitlementsForServerModule() throws ClassNotFoundException { + var policyManager = new PolicyManager( + createTestServerPolicy("jdk.httpserver"), + List.of(), + Map.of(), + c -> null, + NO_ENTITLEMENTS_MODULE + ); + + // Tests do not run modular, so we cannot use a server class. + // But we know that in production code the server module and its classes are in the boot layer. + // So we use a random module in the boot layer, and a random class from that module (not java.base -- it is + // loaded too early) to mimic a class that would be in the server module. + var mockServerClass = ModuleLayer.boot().findLoader("jdk.httpserver").loadClass("com.sun.net.httpserver.HttpServer"); + var requestingModule = mockServerClass.getModule(); + + var entitlements = policyManager.getEntitlements(mockServerClass); + assertThat(entitlements.hasEntitlement(CreateClassLoaderEntitlement.class), is(true)); + assertThat(entitlements.hasEntitlement(ExitVMEntitlement.class), is(true)); + } + + public void testGetEntitlementsReturnsEntitlementsForPluginModule() throws IOException, ClassNotFoundException { + final Path home = createTempDir(); + + Path jar = createMockPluginJar(home); + + var policyManager = new PolicyManager( + createEmptyTestServerPolicy(), + List.of(), + Map.of("mock-plugin", createPluginPolicy("org.example.plugin")), + c -> "mock-plugin", + NO_ENTITLEMENTS_MODULE + ); + + var layer = createLayerForJar(jar, "org.example.plugin"); + var mockPluginClass = layer.findLoader("org.example.plugin").loadClass("q.B"); + var requestingModule = mockPluginClass.getModule(); + + var entitlements = policyManager.getEntitlements(mockPluginClass); + assertThat(entitlements.hasEntitlement(CreateClassLoaderEntitlement.class), is(true)); + assertThat( + entitlements.getEntitlements(FileEntitlement.class).toList(), + contains(transformedMatch(FileEntitlement::toString, containsString("/test/path"))) + ); + } + + public void testGetEntitlementsResultIsCached() { + var policyManager = new PolicyManager( + createEmptyTestServerPolicy(), + List.of(), + Map.ofEntries(entry("plugin2", createPluginPolicy(ALL_UNNAMED))), + c -> "plugin2", + NO_ENTITLEMENTS_MODULE + ); + + // Any class from the current module (unnamed) will do + var callerClass = this.getClass(); + + var entitlements = policyManager.getEntitlements(callerClass); + assertThat(entitlements.hasEntitlement(CreateClassLoaderEntitlement.class), is(true)); + assertThat(policyManager.moduleEntitlementsMap, aMapWithSize(1)); + var cachedResult = policyManager.moduleEntitlementsMap.values().stream().findFirst().get(); + var entitlementsAgain = policyManager.getEntitlements(callerClass); + + // Nothing new in the map + assertThat(policyManager.moduleEntitlementsMap, aMapWithSize(1)); + assertThat(entitlementsAgain, sameInstance(cachedResult)); + } + + public void testRequestingClassFastPath() throws IOException, ClassNotFoundException { + var callerClass = makeClassInItsOwnModule(); + assertEquals(callerClass, policyManagerWithEntitlementsModule(NO_ENTITLEMENTS_MODULE).requestingClass(callerClass)); + } + + public void testRequestingModuleWithStackWalk() throws IOException, ClassNotFoundException { + var entitlementsClass = makeClassInItsOwnModule(); // A class in the entitlements library itself + var requestingClass = makeClassInItsOwnModule(); // This guy is always the right answer + var instrumentedClass = makeClassInItsOwnModule(); // The class that called the check method + var ignorableClass = makeClassInItsOwnModule(); + + var policyManager = policyManagerWithEntitlementsModule(entitlementsClass.getModule()); + + assertEquals( + "Skip entitlement library and the instrumented method", + requestingClass, + policyManager.findRequestingClass(Stream.of(entitlementsClass, instrumentedClass, requestingClass, ignorableClass)).orElse(null) + ); + assertEquals( + "Skip multiple library frames", + requestingClass, + policyManager.findRequestingClass(Stream.of(entitlementsClass, entitlementsClass, instrumentedClass, requestingClass)) + .orElse(null) + ); + assertThrows( + "Non-modular caller frames are not supported", + NullPointerException.class, + () -> policyManager.findRequestingClass(Stream.of(entitlementsClass, null)) + ); + } + + private static Class makeClassInItsOwnModule() throws IOException, ClassNotFoundException { + final Path home = createTempDir(); + Path jar = createMockPluginJar(home); + var layer = createLayerForJar(jar, "org.example.plugin"); + return layer.findLoader("org.example.plugin").loadClass("q.B"); + } + + private static PolicyManager policyManagerWithEntitlementsModule(Module entitlementsModule) { + return new PolicyManager(createEmptyTestServerPolicy(), List.of(), Map.of(), c -> "test", entitlementsModule); + } + + private static Policy createEmptyTestServerPolicy() { + return new Policy("server", List.of()); + } + + private static Policy createTestServerPolicy(String scopeName) { + return new Policy("server", List.of(new Scope(scopeName, List.of(new ExitVMEntitlement(), new CreateClassLoaderEntitlement())))); + } + + private static Policy createPluginPolicy(String... pluginModules) { + return new Policy( + "plugin", + Arrays.stream(pluginModules) + .map( + name -> new Scope( + name, + List.of(new FileEntitlement("/test/path", List.of(FileEntitlement.READ)), new CreateClassLoaderEntitlement()) + ) + ) + .toList() + ); + } + + private static Path createMockPluginJar(Path home) throws IOException { + Path jar = home.resolve("mock-plugin.jar"); + + Map sources = Map.ofEntries( + entry("module-info", "module org.example.plugin { exports q; }"), + entry("q.B", "package q; public class B { }") + ); + + var classToBytes = InMemoryJavaCompiler.compile(sources); + JarUtils.createJarWithEntries( + jar, + Map.ofEntries(entry("module-info.class", classToBytes.get("module-info")), entry("q/B.class", classToBytes.get("q.B"))) + ); + return jar; + } + + private static ModuleLayer createLayerForJar(Path jar, String moduleName) { + Configuration cf = ModuleLayer.boot().configuration().resolve(ModuleFinder.of(jar), ModuleFinder.of(), Set.of(moduleName)); + var moduleController = ModuleLayer.defineModulesWithOneLoader( + cf, + List.of(ModuleLayer.boot()), + ClassLoader.getPlatformClassLoader() + ); + return moduleController.layer(); + } +} diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java new file mode 100644 index 0000000000000..dfcc5d8916f2c --- /dev/null +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import org.elasticsearch.test.ESTestCase; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; + +public class PolicyParserFailureTests extends ESTestCase { + + public void testParserSyntaxFailures() { + PolicyParserException ppe = expectThrows( + PolicyParserException.class, + () -> new PolicyParser(new ByteArrayInputStream("[]".getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false) + .parsePolicy() + ); + assertEquals("[1:1] policy parsing error for [test-failure-policy.yaml]: expected object ", ppe.getMessage()); + } + + public void testEntitlementDoesNotExist() { + PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream(""" + entitlement-module-name: + - does_not_exist: {} + """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy()); + assertEquals( + "[2:5] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name]: " + + "unknown entitlement type [does_not_exist]", + ppe.getMessage() + ); + } + + public void testEntitlementMissingParameter() { + PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream(""" + entitlement-module-name: + - file: {} + """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy()); + assertEquals( + "[2:12] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] " + + "for entitlement type [file]: missing entitlement parameter [path]", + ppe.getMessage() + ); + + ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream(""" + entitlement-module-name: + - file: + path: test-path + """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy()); + assertEquals( + "[4:1] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] " + + "for entitlement type [file]: missing entitlement parameter [actions]", + ppe.getMessage() + ); + } + + public void testEntitlementExtraneousParameter() { + PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream(""" + entitlement-module-name: + - file: + path: test-path + actions: + - read + extra: test + """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy()); + assertEquals( + "[7:1] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] " + + "for entitlement type [file]: extraneous entitlement parameter(s) {extra=test}", + ppe.getMessage() + ); + } + + public void testEntitlementIsNotForExternalPlugins() { + PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream(""" + entitlement-module-name: + - create_class_loader + """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", true).parsePolicy()); + assertEquals( + "[2:5] policy parsing error for [test-failure-policy.yaml]: entitlement type [create_class_loader] is allowed only on modules", + ppe.getMessage() + ); + } +} diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java new file mode 100644 index 0000000000000..4d17fc92e1578 --- /dev/null +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import org.elasticsearch.test.ESTestCase; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; + +public class PolicyParserTests extends ESTestCase { + + private static class TestWrongEntitlementName implements Entitlement {} + + public void testGetEntitlementTypeName() { + assertEquals("create_class_loader", PolicyParser.getEntitlementTypeName(CreateClassLoaderEntitlement.class)); + + var ex = expectThrows(IllegalArgumentException.class, () -> PolicyParser.getEntitlementTypeName(TestWrongEntitlementName.class)); + assertThat( + ex.getMessage(), + equalTo("TestWrongEntitlementName is not a valid Entitlement class name. A valid class name must end with 'Entitlement'") + ); + } + + public void testPolicyBuilder() throws IOException { + Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml", false) + .parsePolicy(); + Policy expected = new Policy( + "test-policy.yaml", + List.of(new Scope("entitlement-module-name", List.of(new FileEntitlement("test/path/to/file", List.of("read", "write"))))) + ); + assertEquals(expected, parsedPolicy); + } + + public void testPolicyBuilderOnExternalPlugin() throws IOException { + Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml", true) + .parsePolicy(); + Policy expected = new Policy( + "test-policy.yaml", + List.of(new Scope("entitlement-module-name", List.of(new FileEntitlement("test/path/to/file", List.of("read", "write"))))) + ); + assertEquals(expected, parsedPolicy); + } + + public void testParseCreateClassloader() throws IOException { + Policy parsedPolicy = new PolicyParser(new ByteArrayInputStream(""" + entitlement-module-name: + - create_class_loader + """.getBytes(StandardCharsets.UTF_8)), "test-policy.yaml", false).parsePolicy(); + Policy expected = new Policy( + "test-policy.yaml", + List.of(new Scope("entitlement-module-name", List.of(new CreateClassLoaderEntitlement()))) + ); + assertEquals(expected, parsedPolicy); + } + + public void testParseSetHttpsConnectionProperties() throws IOException { + Policy parsedPolicy = new PolicyParser(new ByteArrayInputStream(""" + entitlement-module-name: + - set_https_connection_properties + """.getBytes(StandardCharsets.UTF_8)), "test-policy.yaml", true).parsePolicy(); + Policy expected = new Policy( + "test-policy.yaml", + List.of(new Scope("entitlement-module-name", List.of(new SetHttpsConnectionPropertiesEntitlement()))) + ); + assertEquals(expected, parsedPolicy); + } +} diff --git a/libs/entitlement/src/test/resources/org/elasticsearch/entitlement/runtime/policy/test-policy.yaml b/libs/entitlement/src/test/resources/org/elasticsearch/entitlement/runtime/policy/test-policy.yaml new file mode 100644 index 0000000000000..f13f574535bec --- /dev/null +++ b/libs/entitlement/src/test/resources/org/elasticsearch/entitlement/runtime/policy/test-policy.yaml @@ -0,0 +1,6 @@ +entitlement-module-name: + - file: + path: "test/path/to/file" + actions: + - "read" + - "write" diff --git a/libs/entitlement/tools/build.gradle b/libs/entitlement/tools/build.gradle new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/libs/entitlement/tools/common/build.gradle b/libs/entitlement/tools/common/build.gradle new file mode 100644 index 0000000000000..89772b4132c5f --- /dev/null +++ b/libs/entitlement/tools/common/build.gradle @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +apply plugin: 'elasticsearch.build' + +tasks.named('forbiddenApisMain').configure { + replaceSignatureFiles 'jdk-signatures' +} diff --git a/libs/entitlement/tools/common/src/main/java/org/elasticsearch/entitlement/tools/Utils.java b/libs/entitlement/tools/common/src/main/java/org/elasticsearch/entitlement/tools/Utils.java new file mode 100644 index 0000000000000..c72e550a529cd --- /dev/null +++ b/libs/entitlement/tools/common/src/main/java/org/elasticsearch/entitlement/tools/Utils.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.tools; + +import java.io.IOException; +import java.lang.module.ModuleDescriptor; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class Utils { + + public static Map> findModuleExports(FileSystem fs) throws IOException { + var modulesExports = new HashMap>(); + try (var stream = Files.walk(fs.getPath("modules"))) { + stream.filter(p -> p.getFileName().toString().equals("module-info.class")).forEach(x -> { + try (var is = Files.newInputStream(x)) { + var md = ModuleDescriptor.read(is); + modulesExports.put( + md.name(), + md.exports() + .stream() + .filter(e -> e.isQualified() == false) + .map(ModuleDescriptor.Exports::source) + .collect(Collectors.toSet()) + ); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + return modulesExports; + } + +} diff --git a/libs/entitlement/tools/securitymanager-scanner/build.gradle b/libs/entitlement/tools/securitymanager-scanner/build.gradle new file mode 100644 index 0000000000000..ebb671e5487ef --- /dev/null +++ b/libs/entitlement/tools/securitymanager-scanner/build.gradle @@ -0,0 +1,61 @@ +plugins { + id 'application' +} + +apply plugin: 'elasticsearch.build' +apply plugin: 'elasticsearch.publish' + +tasks.named("dependencyLicenses").configure { + mapping from: /asm-.*/, to: 'asm' +} + +group = 'org.elasticsearch.entitlement.tools' + +ext { + javaMainClass = "org.elasticsearch.entitlement.tools.securitymanager.scanner.Main" +} + +application { + mainClass.set(javaMainClass) + applicationDefaultJvmArgs = [ + '--add-exports', 'java.base/sun.security.util=ALL-UNNAMED', + '--add-opens', 'java.base/java.lang=ALL-UNNAMED', + '--add-opens', 'java.base/java.net=ALL-UNNAMED', + '--add-opens', 'java.base/java.net.spi=ALL-UNNAMED', + '--add-opens', 'java.base/java.util.concurrent=ALL-UNNAMED', + '--add-opens', 'java.base/javax.crypto=ALL-UNNAMED', + '--add-opens', 'java.base/javax.security.auth=ALL-UNNAMED', + '--add-opens', 'java.base/jdk.internal.logger=ALL-UNNAMED', + '--add-opens', 'java.base/sun.nio.ch=ALL-UNNAMED', + '--add-opens', 'jdk.management.jfr/jdk.management.jfr=ALL-UNNAMED', + '--add-opens', 'java.logging/java.util.logging=ALL-UNNAMED', + '--add-opens', 'java.logging/sun.util.logging.internal=ALL-UNNAMED', + '--add-opens', 'java.naming/javax.naming.ldap.spi=ALL-UNNAMED', + '--add-opens', 'java.rmi/sun.rmi.runtime=ALL-UNNAMED', + '--add-opens', 'jdk.dynalink/jdk.dynalink=ALL-UNNAMED', + '--add-opens', 'jdk.dynalink/jdk.dynalink.linker=ALL-UNNAMED', + '--add-opens', 'java.desktop/sun.awt=ALL-UNNAMED', + '--add-opens', 'java.sql.rowset/javax.sql.rowset.spi=ALL-UNNAMED', + '--add-opens', 'java.sql/java.sql=ALL-UNNAMED', + '--add-opens', 'java.xml.crypto/com.sun.org.apache.xml.internal.security.utils=ALL-UNNAMED' + ] +} + +repositories { + mavenCentral() +} + +dependencies { + compileOnly(project(':libs:core')) + implementation 'org.ow2.asm:asm:9.7.1' + implementation 'org.ow2.asm:asm-util:9.7.1' + implementation(project(':libs:entitlement:tools:common')) +} + +tasks.named('forbiddenApisMain').configure { + replaceSignatureFiles 'jdk-signatures' +} + +tasks.named("thirdPartyAudit").configure { + ignoreMissingClasses() +} diff --git a/libs/entitlement/tools/securitymanager-scanner/licenses/asm-LICENSE.txt b/libs/entitlement/tools/securitymanager-scanner/licenses/asm-LICENSE.txt new file mode 100644 index 0000000000000..afb064f2f2666 --- /dev/null +++ b/libs/entitlement/tools/securitymanager-scanner/licenses/asm-LICENSE.txt @@ -0,0 +1,26 @@ +Copyright (c) 2012 France Télécom +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. diff --git a/libs/entitlement/tools/securitymanager-scanner/licenses/asm-NOTICE.txt b/libs/entitlement/tools/securitymanager-scanner/licenses/asm-NOTICE.txt new file mode 100644 index 0000000000000..8d1c8b69c3fce --- /dev/null +++ b/libs/entitlement/tools/securitymanager-scanner/licenses/asm-NOTICE.txt @@ -0,0 +1 @@ + diff --git a/libs/entitlement/tools/securitymanager-scanner/src/README.md b/libs/entitlement/tools/securitymanager-scanner/src/README.md new file mode 100644 index 0000000000000..c01ba1387d1c8 --- /dev/null +++ b/libs/entitlement/tools/securitymanager-scanner/src/README.md @@ -0,0 +1,47 @@ +This tool scans the JDK on which it is running, looking for any location where `SecurityManager` is currently used, thus giving us a list of "entry points" inside the JDK where security checks are currently happening. + +More in detail, the tool scans for calls to any `SecurityManager` method starting with `check` (e.g. `checkWrite`). The tool treats the generic `checkPermission` method a little bit differently: `checkPermission` accepts a generic `Permission` object, it tries to read the permission type and permission name to give more information about it, trying to match two patterns that are used frequently inside the JDK: + +Pattern 1: private static permission field + +```java +private static final RuntimePermission INET_ADDRESS_RESOLVER_PERMISSION = +new RuntimePermission("inetAddressResolverProvider"); +... +sm.checkPermission(INET_ADDRESS_RESOLVER_PERMISSION); +``` +Pattern 2: direct object creation + +```java +sm.checkPermission(new LinkPermission("symbolic")); +``` + +The tool will recognize this pattern, and report the permission type and name alongside the `checkPermission` entry point (type `RuntimePermission` and name `inetAddressResolverProvider` in the first case, type `LinkPermission` and name `symbolic` in the second). + +This allows to give more information (either a specific type like `LinkPermission`, or a specific name like `inetAddressResolverProvider`) to generic `checkPermission` to help in deciding how to classify the permission check. The 2 patterns work quite well and cover roughly 90% of the cases. + +In order to run the tool, use: +```shell +./gradlew :libs:entitlement:tools:securitymanager-scanner:run +``` +The output of the tool is a CSV file, with one line for each entry-point, columns separated by `TAB` + +The columns are: +1. Module name +2. File name (from source root) +3. Line number +4. Fully qualified class name (ASM style, with `/` separators) +5. Method name +6. Method descriptor (ASM signature) +6. Visibility (PUBLIC/PUBLIC-METHOD/PRIVATE) +7. Check detail 1 (method name, or in case of checkPermission, permission name. Might be `MISSING`) +8. Check detail 2 (in case of checkPermission, the argument type (`Permission` subtype). Might be `MISSING`) + +Examples: +``` +java.base sun/nio/ch/DatagramChannelImpl.java 1360 sun/nio/ch/DatagramChannelImpl connect (Ljava/net/SocketAddress;Z)Ljava/nio/channels/DatagramChannel; PRIVATE checkConnect +``` +or +``` +java.base java/net/ResponseCache.java 118 java/net/ResponseCache setDefault (Ljava/net/ResponseCache;)V PUBLIC setResponseCache java/net/NetPermission +``` diff --git a/libs/entitlement/tools/securitymanager-scanner/src/main/java/org/elasticsearch/entitlement/tools/securitymanager/scanner/Main.java b/libs/entitlement/tools/securitymanager-scanner/src/main/java/org/elasticsearch/entitlement/tools/securitymanager/scanner/Main.java new file mode 100644 index 0000000000000..bea49e0296e67 --- /dev/null +++ b/libs/entitlement/tools/securitymanager-scanner/src/main/java/org/elasticsearch/entitlement/tools/securitymanager/scanner/Main.java @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.tools.securitymanager.scanner; + +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.entitlement.tools.Utils; +import org.objectweb.asm.ClassReader; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.List; +import java.util.Set; + +public class Main { + + static final Set excludedModules = Set.of("java.desktop"); + + private static void identifySMChecksEntryPoints() throws IOException { + + FileSystem fs = FileSystems.getFileSystem(URI.create("jrt:/")); + + var moduleExports = Utils.findModuleExports(fs); + + var callers = new HashMap>(); + var visitor = new SecurityCheckClassVisitor(callers); + + try (var stream = Files.walk(fs.getPath("modules"))) { + stream.filter(x -> x.toString().endsWith(".class")).forEach(x -> { + var moduleName = x.subpath(1, 2).toString(); + if (excludedModules.contains(moduleName) == false) { + try { + ClassReader cr = new ClassReader(Files.newInputStream(x)); + visitor.setCurrentModule(moduleName, moduleExports.get(moduleName)); + var path = x.getNameCount() > 3 ? x.subpath(2, x.getNameCount() - 1).toString() : ""; + visitor.setCurrentSourcePath(path); + cr.accept(visitor, 0); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }); + } + + printToStdout(callers); + } + + @SuppressForbidden(reason = "This simple tool just prints to System.out") + private static void printToStdout(HashMap> callers) { + for (var kv : callers.entrySet()) { + for (var e : kv.getValue()) { + System.out.println(toString(kv.getKey(), e)); + } + } + } + + private static final String SEPARATOR = "\t"; + + private static String toString(String calleeName, SecurityCheckClassVisitor.CallerInfo callerInfo) { + var s = callerInfo.moduleName() + SEPARATOR + callerInfo.source() + SEPARATOR + callerInfo.line() + SEPARATOR + callerInfo + .className() + SEPARATOR + callerInfo.methodName() + SEPARATOR + callerInfo.methodDescriptor() + SEPARATOR; + + if (callerInfo.externalAccess().contains(SecurityCheckClassVisitor.ExternalAccess.METHOD) + && callerInfo.externalAccess().contains(SecurityCheckClassVisitor.ExternalAccess.CLASS)) { + s += "PUBLIC"; + } else if (callerInfo.externalAccess().contains(SecurityCheckClassVisitor.ExternalAccess.METHOD)) { + s += "PUBLIC-METHOD"; + } else { + s += "PRIVATE"; + } + + if (callerInfo.runtimePermissionType() != null) { + s += SEPARATOR + callerInfo.runtimePermissionType(); + } else if (calleeName.equals("checkPermission")) { + s += SEPARATOR + "MISSING"; // missing information + } else { + s += SEPARATOR + calleeName; + } + + if (callerInfo.permissionType() != null) { + s += SEPARATOR + callerInfo.permissionType(); + } else if (calleeName.equals("checkPermission")) { + s += SEPARATOR + "MISSING"; // missing information + } else { + s += SEPARATOR; + } + return s; + } + + public static void main(String[] args) throws IOException { + identifySMChecksEntryPoints(); + } +} diff --git a/libs/entitlement/tools/securitymanager-scanner/src/main/java/org/elasticsearch/entitlement/tools/securitymanager/scanner/SecurityCheckClassVisitor.java b/libs/entitlement/tools/securitymanager-scanner/src/main/java/org/elasticsearch/entitlement/tools/securitymanager/scanner/SecurityCheckClassVisitor.java new file mode 100644 index 0000000000000..7b18a0b1be332 --- /dev/null +++ b/libs/entitlement/tools/securitymanager-scanner/src/main/java/org/elasticsearch/entitlement/tools/securitymanager/scanner/SecurityCheckClassVisitor.java @@ -0,0 +1,279 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.tools.securitymanager.scanner; + +import org.elasticsearch.core.SuppressForbidden; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.Label; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Type; + +import java.lang.reflect.InaccessibleObjectException; +import java.lang.reflect.Modifier; +import java.nio.file.Path; +import java.security.Permission; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.objectweb.asm.Opcodes.ACC_PUBLIC; +import static org.objectweb.asm.Opcodes.ASM9; +import static org.objectweb.asm.Opcodes.GETSTATIC; +import static org.objectweb.asm.Opcodes.INVOKEDYNAMIC; +import static org.objectweb.asm.Opcodes.INVOKEINTERFACE; +import static org.objectweb.asm.Opcodes.INVOKESPECIAL; +import static org.objectweb.asm.Opcodes.INVOKESTATIC; +import static org.objectweb.asm.Opcodes.INVOKEVIRTUAL; +import static org.objectweb.asm.Opcodes.NEW; + +class SecurityCheckClassVisitor extends ClassVisitor { + + static final String SECURITY_MANAGER_INTERNAL_NAME = "java/lang/SecurityManager"; + static final Set excludedClasses = Set.of(SECURITY_MANAGER_INTERNAL_NAME); + + enum ExternalAccess { + CLASS, + METHOD + } + + record CallerInfo( + String moduleName, + String source, + int line, + String className, + String methodName, + String methodDescriptor, + EnumSet externalAccess, + String permissionType, + String runtimePermissionType + ) {} + + private final Map> callerInfoByMethod; + private String className; + private int classAccess; + private String source; + private String moduleName; + private String sourcePath; + private Set moduleExports; + + protected SecurityCheckClassVisitor(Map> callerInfoByMethod) { + super(ASM9); + this.callerInfoByMethod = callerInfoByMethod; + } + + @Override + public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) { + super.visit(version, access, name, signature, superName, interfaces); + this.className = name; + this.classAccess = access; + } + + @Override + public void visitSource(String source, String debug) { + super.visitSource(source, debug); + this.source = source; + } + + @Override + public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { + if (excludedClasses.contains(this.className)) { + return super.visitMethod(access, name, descriptor, signature, exceptions); + } + return new SecurityCheckMethodVisitor(super.visitMethod(access, name, descriptor, signature, exceptions), name, access, descriptor); + } + + public void setCurrentModule(String moduleName, Set moduleExports) { + this.moduleName = moduleName; + this.moduleExports = moduleExports; + } + + public void setCurrentSourcePath(String path) { + this.sourcePath = path; + } + + private class SecurityCheckMethodVisitor extends MethodVisitor { + + private final String methodName; + private final String methodDescriptor; + private int line; + private String permissionType; + private String runtimePermissionType; + private final int methodAccess; + + protected SecurityCheckMethodVisitor(MethodVisitor mv, String methodName, int methodAccess, String methodDescriptor) { + super(ASM9, mv); + this.methodName = methodName; + this.methodAccess = methodAccess; + this.methodDescriptor = methodDescriptor; + } + + private static final Set KNOWN_PERMISSIONS = Set.of("jdk.vm.ci.services.JVMCIPermission"); + + @SuppressForbidden(reason = "System.err is OK for this simple command-line tool") + private void handleException(String className, Throwable e) { + System.err.println("Cannot process " + className + ": " + e.getMessage()); + } + + @Override + public void visitTypeInsn(int opcode, String type) { + super.visitTypeInsn(opcode, type); + if (opcode == NEW) { + if (type.endsWith("Permission")) { + var objectType = Type.getObjectType(type); + if (KNOWN_PERMISSIONS.contains(objectType.getClassName())) { + permissionType = type; + } else { + try { + var clazz = Class.forName(objectType.getClassName()); + if (Permission.class.isAssignableFrom(clazz)) { + permissionType = type; + } + } catch (ClassNotFoundException e) { + handleException(objectType.getClassName(), e); + } + } + } + } + } + + @Override + @SuppressForbidden(reason = "We need to violate java's access system to access private parts") + public void visitFieldInsn(int opcode, String owner, String name, String descriptor) { + super.visitFieldInsn(opcode, owner, name, descriptor); + if (opcode == GETSTATIC && descriptor.endsWith("Permission;")) { + var permissionType = Type.getType(descriptor); + if (permissionType.getSort() == Type.ARRAY) { + permissionType = permissionType.getElementType(); + } + try { + var clazz = Class.forName(permissionType.getClassName()); + if (Permission.class.isAssignableFrom(clazz)) { + this.permissionType = permissionType.getInternalName(); + } + } catch (ClassNotFoundException e) { + handleException(permissionType.getClassName(), e); + } + + var objectType = Type.getObjectType(owner); + try { + var clazz = Class.forName(objectType.getClassName()); + Arrays.stream(clazz.getDeclaredFields()) + .filter(f -> Modifier.isStatic(f.getModifiers()) && Modifier.isFinal(f.getModifiers())) + .filter(f -> f.getName().equals(name)) + .findFirst() + .ifPresent(x -> { + if (Permission.class.isAssignableFrom(x.getType())) { + try { + x.setAccessible(true); + var p = (Permission) (x.get(null)); + this.runtimePermissionType = p.getName(); + } catch (IllegalAccessException | InaccessibleObjectException e) { + handleException(x.getName(), e); + } + } + }); + + } catch (ClassNotFoundException | NoClassDefFoundError | UnsatisfiedLinkError e) { + handleException(objectType.getClassName(), e); + } + } + } + + @Override + public void visitLdcInsn(Object value) { + super.visitLdcInsn(value); + if (permissionType != null && permissionType.equals("java/lang/RuntimePermission")) { + this.runtimePermissionType = value.toString(); + } + } + + @Override + public void visitMethodInsn(int opcode, String owner, String name, String descriptor, boolean isInterface) { + super.visitMethodInsn(opcode, owner, name, descriptor, isInterface); + if (opcode == INVOKEVIRTUAL + || opcode == INVOKESPECIAL + || opcode == INVOKESTATIC + || opcode == INVOKEINTERFACE + || opcode == INVOKEDYNAMIC) { + + if (SECURITY_MANAGER_INTERNAL_NAME.equals(owner)) { + EnumSet externalAccesses = EnumSet.noneOf(ExternalAccess.class); + if (moduleExports.contains(getPackageName(className))) { + if ((methodAccess & ACC_PUBLIC) != 0) { + externalAccesses.add(ExternalAccess.METHOD); + } + if ((classAccess & ACC_PUBLIC) != 0) { + externalAccesses.add(ExternalAccess.CLASS); + } + } + + if (name.equals("checkPermission")) { + var callers = callerInfoByMethod.computeIfAbsent(name, ignored -> new ArrayList<>()); + callers.add( + new CallerInfo( + moduleName, + Path.of(sourcePath, source).toString(), + line, + className, + methodName, + methodDescriptor, + externalAccesses, + permissionType, + runtimePermissionType + ) + ); + this.permissionType = null; + this.runtimePermissionType = null; + } else if (name.startsWith("check")) { + // Non-generic methods (named methods that which already tell us the permission type) + var callers = callerInfoByMethod.computeIfAbsent(name, ignored -> new ArrayList<>()); + callers.add( + new CallerInfo( + moduleName, + Path.of(sourcePath, source).toString(), + line, + className, + methodName, + methodDescriptor, + externalAccesses, + null, + null + ) + ); + } + } + } + } + + private String getPackageName(String className) { + int lastSlash = className.lastIndexOf('/'); + return className.substring(0, lastSlash).replace("/", "."); + } + + @Override + public void visitParameter(String name, int access) { + if (name != null) super.visitParameter(name, access); + } + + @Override + public void visitLineNumber(int line, Label start) { + super.visitLineNumber(line, start); + this.line = line; + } + + @Override + public void visitEnd() { + super.visitEnd(); + } + } +} diff --git a/libs/geo/build.gradle b/libs/geo/build.gradle index ef88e8d277f03..c753ba814a5f9 100644 --- a/libs/geo/build.gradle +++ b/libs/geo/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.build' @@ -11,7 +12,7 @@ apply plugin: 'elasticsearch.publish' dependencies { testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-geo' + exclude group: 'org.elasticsearch', module: 'geo' } } diff --git a/libs/geo/src/main/java/module-info.java b/libs/geo/src/main/java/module-info.java index 3f7b1f0c9710f..c85e5f0de72a2 100644 --- a/libs/geo/src/main/java/module-info.java +++ b/libs/geo/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.geo { diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/Circle.java b/libs/geo/src/main/java/org/elasticsearch/geometry/Circle.java index 399711d95aaa9..792d660179f5b 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/Circle.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/Circle.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry; diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/Geometry.java b/libs/geo/src/main/java/org/elasticsearch/geometry/Geometry.java index 71d057dfc8b39..903aa8d2918b1 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/Geometry.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/Geometry.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry; diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/GeometryCollection.java b/libs/geo/src/main/java/org/elasticsearch/geometry/GeometryCollection.java index 80f2e9ea6f822..7fb447555d74d 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/GeometryCollection.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/GeometryCollection.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry; diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/GeometryVisitor.java b/libs/geo/src/main/java/org/elasticsearch/geometry/GeometryVisitor.java index 34a61700c9740..1ec9d249b2268 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/GeometryVisitor.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/GeometryVisitor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry; diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/Line.java b/libs/geo/src/main/java/org/elasticsearch/geometry/Line.java index 904128ef2a4e5..be1a640f1a8d7 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/Line.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/Line.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry; diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/LinearRing.java b/libs/geo/src/main/java/org/elasticsearch/geometry/LinearRing.java index 67bc0392682ec..e76895da47cae 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/LinearRing.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/LinearRing.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry; diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/MultiLine.java b/libs/geo/src/main/java/org/elasticsearch/geometry/MultiLine.java index 2e36cb5999127..5fde9be3d671a 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/MultiLine.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/MultiLine.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry; diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/MultiPoint.java b/libs/geo/src/main/java/org/elasticsearch/geometry/MultiPoint.java index 43871fae6f6b6..287ea9e433c48 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/MultiPoint.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/MultiPoint.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry; diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/MultiPolygon.java b/libs/geo/src/main/java/org/elasticsearch/geometry/MultiPolygon.java index 5618f42b6aa49..6b83374f6e47f 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/MultiPolygon.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/MultiPolygon.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry; diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/Point.java b/libs/geo/src/main/java/org/elasticsearch/geometry/Point.java index 1b2b9ac4124df..dc6a29db87bc0 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/Point.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/Point.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry; diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/Polygon.java b/libs/geo/src/main/java/org/elasticsearch/geometry/Polygon.java index 3360ac4da2b2c..9e0ca7d84ced6 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/Polygon.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/Polygon.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry; diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/Rectangle.java b/libs/geo/src/main/java/org/elasticsearch/geometry/Rectangle.java index fceb27c7515a2..3ea981a96d813 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/Rectangle.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/Rectangle.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry; diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/ShapeType.java b/libs/geo/src/main/java/org/elasticsearch/geometry/ShapeType.java index 12c1d00891cfe..9e3e266837b8e 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/ShapeType.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/ShapeType.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry; diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/package-info.java b/libs/geo/src/main/java/org/elasticsearch/geometry/package-info.java index 182c35fc61809..c0cf5dc7ca8f2 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/package-info.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/simplify/GeometrySimplifier.java b/libs/geo/src/main/java/org/elasticsearch/geometry/simplify/GeometrySimplifier.java index 5ff6fd6048ada..3b1b47c748949 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/simplify/GeometrySimplifier.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/simplify/GeometrySimplifier.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry.simplify; diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/simplify/SimplificationErrorCalculator.java b/libs/geo/src/main/java/org/elasticsearch/geometry/simplify/SimplificationErrorCalculator.java index d90dabcc8cc35..51dfac8ac0a53 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/simplify/SimplificationErrorCalculator.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/simplify/SimplificationErrorCalculator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry.simplify; diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/simplify/SloppyMath.java b/libs/geo/src/main/java/org/elasticsearch/geometry/simplify/SloppyMath.java index 555dccd057174..18581123cddb3 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/simplify/SloppyMath.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/simplify/SloppyMath.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry.simplify; diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/simplify/StreamingGeometrySimplifier.java b/libs/geo/src/main/java/org/elasticsearch/geometry/simplify/StreamingGeometrySimplifier.java index 75862538fa006..d9a27a725ac2e 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/simplify/StreamingGeometrySimplifier.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/simplify/StreamingGeometrySimplifier.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry.simplify; diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/BitUtil.java b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/BitUtil.java index 1522080da3ced..8d5bc1b5816f9 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/BitUtil.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/BitUtil.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry.utils; diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/CircleUtils.java b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/CircleUtils.java index acd5c8234171d..f9a63b0468805 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/CircleUtils.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/CircleUtils.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry.utils; diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/GeographyValidator.java b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/GeographyValidator.java index 08250417d76c1..622159cd69aee 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/GeographyValidator.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/GeographyValidator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry.utils; diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/Geohash.java b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/Geohash.java index 90ae3b76e796d..bbd923939b2a8 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/Geohash.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/Geohash.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry.utils; diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/GeometryValidator.java b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/GeometryValidator.java index 13aa232c34840..d7b88e3cdd6ee 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/GeometryValidator.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/GeometryValidator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry.utils; diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java new file mode 100644 index 0000000000000..696be2808ed1f --- /dev/null +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitor.java @@ -0,0 +1,340 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.geometry.utils; + +import org.elasticsearch.geometry.Circle; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.GeometryCollection; +import org.elasticsearch.geometry.GeometryVisitor; +import org.elasticsearch.geometry.Line; +import org.elasticsearch.geometry.LinearRing; +import org.elasticsearch.geometry.MultiLine; +import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.geometry.MultiPolygon; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Polygon; +import org.elasticsearch.geometry.Rectangle; + +import java.util.Locale; +import java.util.Optional; + +/** + * This visitor is designed to determine the spatial envelope (or BBOX or MBR) of a potentially complex geometry. + * It has two modes: + *
      + *
    • + * Cartesian mode: The envelope is determined by the minimum and maximum x/y coordinates. + * Incoming BBOX geometries with minX > maxX are treated as invalid. + * Resulting BBOX geometries will always have minX <= maxX. + *
    • + *
    • + * Geographic mode: The envelope is determined by the minimum and maximum x/y coordinates, + * considering the possibility of wrapping the longitude around the dateline. + * A bounding box can be determined either by wrapping the longitude around the dateline or not, + * and the smaller bounding box is chosen. It is possible to disable the wrapping of the longitude. + *
    + * Usage of this is as simple as: + * + * Optional<Rectangle> bbox = SpatialEnvelopeVisitor.visit(geometry); + * if (bbox.isPresent()) { + * Rectangle envelope = bbox.get(); + * // Do stuff with the envelope + * } + * + * It is also possible to create the inner PointVisitor separately, as well as use the visitor for multiple geometries. + * + * PointVisitor pointVisitor = new CartesianPointVisitor(); + * SpatialEnvelopeVisitor visitor = new SpatialEnvelopeVisitor(pointVisitor); + * for (Geometry geometry : geometries) { + * geometry.visit(visitor); + * } + * if (visitor.isValid()) { + * Rectangle envelope = visitor.getResult(); + * // Do stuff with the envelope + * } + * + * Code that wishes to modify the behaviour of the visitor can implement the PointVisitor interface, + * or extend the existing implementations. + */ +public class SpatialEnvelopeVisitor implements GeometryVisitor { + + private final PointVisitor pointVisitor; + + public SpatialEnvelopeVisitor(PointVisitor pointVisitor) { + this.pointVisitor = pointVisitor; + } + + /** + * Determine the BBOX without considering the CRS or wrapping of the longitude. + * Note that incoming BBOX's that do cross the dateline (minx>maxx) will be treated as invalid. + */ + public static Optional visitCartesian(Geometry geometry) { + var visitor = new SpatialEnvelopeVisitor(new CartesianPointVisitor()); + if (geometry.visit(visitor)) { + return Optional.of(visitor.getResult()); + } + return Optional.empty(); + } + + public enum WrapLongitude { + NO_WRAP, + WRAP + } + + /** + * Determine the BBOX assuming the CRS is geographic (eg WGS84) and optionally wrapping the longitude around the dateline. + */ + public static Optional visitGeo(Geometry geometry, WrapLongitude wrapLongitude) { + var visitor = new SpatialEnvelopeVisitor(new GeoPointVisitor(wrapLongitude)); + if (geometry.visit(visitor)) { + return Optional.of(visitor.getResult()); + } + return Optional.empty(); + } + + public Rectangle getResult() { + return pointVisitor.getResult(); + } + + /** + * Visitor for visiting points and rectangles. This is where the actual envelope calculation happens. + * There are two implementations, one for cartesian coordinates and one for geographic coordinates. + * The latter can optionally wrap the longitude around the dateline. + */ + public interface PointVisitor { + void visitPoint(double x, double y); + + void visitRectangle(double minX, double maxX, double maxY, double minY); + + boolean isValid(); + + Rectangle getResult(); + } + + /** + * The cartesian point visitor determines the envelope by the minimum and maximum x/y coordinates. + * It also disallows invalid rectangles where minX > maxX. + */ + public static class CartesianPointVisitor implements PointVisitor { + private double minX = Double.POSITIVE_INFINITY; + private double minY = Double.POSITIVE_INFINITY; + private double maxX = Double.NEGATIVE_INFINITY; + private double maxY = Double.NEGATIVE_INFINITY; + + public double getMinX() { + return minX; + } + + public double getMinY() { + return minY; + } + + public double getMaxX() { + return maxX; + } + + public double getMaxY() { + return maxY; + } + + @Override + public void visitPoint(double x, double y) { + minX = Math.min(minX, x); + minY = Math.min(minY, y); + maxX = Math.max(maxX, x); + maxY = Math.max(maxY, y); + } + + @Override + public void visitRectangle(double minX, double maxX, double maxY, double minY) { + if (minX > maxX) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Invalid cartesian rectangle: minX (%s) > maxX (%s)", minX, maxX) + ); + } + this.minX = Math.min(this.minX, minX); + this.minY = Math.min(this.minY, minY); + this.maxX = Math.max(this.maxX, maxX); + this.maxY = Math.max(this.maxY, maxY); + } + + @Override + public boolean isValid() { + return minY != Double.POSITIVE_INFINITY; + } + + @Override + public Rectangle getResult() { + return new Rectangle(minX, maxX, maxY, minY); + } + } + + /** + * The geographic point visitor determines the envelope by the minimum and maximum x/y coordinates, + * while allowing for wrapping the longitude around the dateline. + * When longitude wrapping is enabled, the visitor will determine the smallest bounding box between the two choices: + *
      + *
    • Wrapping around the front of the earth, in which case the result will have minx < maxx
    • + *
    • Wrapping around the back of the earth, crossing the dateline, in which case the result will have minx > maxx
    • + *
    + */ + public static class GeoPointVisitor implements PointVisitor { + protected double minY = Double.POSITIVE_INFINITY; + protected double maxY = Double.NEGATIVE_INFINITY; + protected double minNegX = Double.POSITIVE_INFINITY; + protected double maxNegX = Double.NEGATIVE_INFINITY; + protected double minPosX = Double.POSITIVE_INFINITY; + protected double maxPosX = Double.NEGATIVE_INFINITY; + + private final WrapLongitude wrapLongitude; + + public GeoPointVisitor(WrapLongitude wrapLongitude) { + this.wrapLongitude = wrapLongitude; + } + + @Override + public void visitPoint(double x, double y) { + minY = Math.min(minY, y); + maxY = Math.max(maxY, y); + visitLongitude(x); + } + + @Override + public void visitRectangle(double minX, double maxX, double maxY, double minY) { + this.minY = Math.min(this.minY, minY); + this.maxY = Math.max(this.maxY, maxY); + visitLongitude(minX); + visitLongitude(maxX); + } + + private void visitLongitude(double x) { + if (x >= 0) { + minPosX = Math.min(minPosX, x); + maxPosX = Math.max(maxPosX, x); + } else { + minNegX = Math.min(minNegX, x); + maxNegX = Math.max(maxNegX, x); + } + } + + @Override + public boolean isValid() { + return minY != Double.POSITIVE_INFINITY; + } + + @Override + public Rectangle getResult() { + return getResult(minNegX, minPosX, maxNegX, maxPosX, maxY, minY, wrapLongitude); + } + + protected static Rectangle getResult( + double minNegX, + double minPosX, + double maxNegX, + double maxPosX, + double maxY, + double minY, + WrapLongitude wrapLongitude + ) { + assert Double.isFinite(maxY); + if (Double.isInfinite(minPosX)) { + return new Rectangle(minNegX, maxNegX, maxY, minY); + } else if (Double.isInfinite(minNegX)) { + return new Rectangle(minPosX, maxPosX, maxY, minY); + } else { + return switch (wrapLongitude) { + case NO_WRAP -> new Rectangle(minNegX, maxPosX, maxY, minY); + case WRAP -> maybeWrap(minNegX, minPosX, maxNegX, maxPosX, maxY, minY); + }; + } + } + + private static Rectangle maybeWrap(double minNegX, double minPosX, double maxNegX, double maxPosX, double maxY, double minY) { + double unwrappedWidth = maxPosX - minNegX; + double wrappedWidth = 360 + maxNegX - minPosX; + return unwrappedWidth <= wrappedWidth + ? new Rectangle(minNegX, maxPosX, maxY, minY) + : new Rectangle(minPosX, maxNegX, maxY, minY); + } + } + + private boolean isValid() { + return pointVisitor.isValid(); + } + + @Override + public Boolean visit(Circle circle) throws RuntimeException { + // TODO: Support circle, if given CRS (needed for radius to x/y coordinate transformation) + throw new UnsupportedOperationException("Circle is not supported"); + } + + @Override + public Boolean visit(GeometryCollection collection) throws RuntimeException { + collection.forEach(geometry -> geometry.visit(this)); + return isValid(); + } + + @Override + public Boolean visit(Line line) throws RuntimeException { + for (int i = 0; i < line.length(); i++) { + pointVisitor.visitPoint(line.getX(i), line.getY(i)); + } + return isValid(); + } + + @Override + public Boolean visit(LinearRing ring) throws RuntimeException { + for (int i = 0; i < ring.length(); i++) { + pointVisitor.visitPoint(ring.getX(i), ring.getY(i)); + } + return isValid(); + } + + @Override + public Boolean visit(MultiLine multiLine) throws RuntimeException { + multiLine.forEach(line -> line.visit(this)); + return isValid(); + } + + @Override + public Boolean visit(MultiPoint multiPoint) throws RuntimeException { + for (int i = 0; i < multiPoint.size(); i++) { + visit(multiPoint.get(i)); + } + return isValid(); + } + + @Override + public Boolean visit(MultiPolygon multiPolygon) throws RuntimeException { + multiPolygon.forEach(polygon -> polygon.visit(this)); + return isValid(); + } + + @Override + public Boolean visit(Point point) throws RuntimeException { + pointVisitor.visitPoint(point.getX(), point.getY()); + return isValid(); + } + + @Override + public Boolean visit(Polygon polygon) throws RuntimeException { + visit(polygon.getPolygon()); + for (int i = 0; i < polygon.getNumberOfHoles(); i++) { + visit(polygon.getHole(i)); + } + return isValid(); + } + + @Override + public Boolean visit(Rectangle rectangle) throws RuntimeException { + pointVisitor.visitRectangle(rectangle.getMinX(), rectangle.getMaxX(), rectangle.getMaxY(), rectangle.getMinY()); + return isValid(); + } +} diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/StandardValidator.java b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/StandardValidator.java index a6c992878d14b..44865df5d1f4b 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/StandardValidator.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/StandardValidator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry.utils; diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/WellKnownBinary.java b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/WellKnownBinary.java index 526a621674f6b..d7f02f1006d5f 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/WellKnownBinary.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/WellKnownBinary.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry.utils; diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/WellKnownText.java b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/WellKnownText.java index 1e7ac3f8097e9..9e657013a80dd 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/WellKnownText.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/WellKnownText.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry.utils; diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/BaseGeometryTestCase.java b/libs/geo/src/test/java/org/elasticsearch/geometry/BaseGeometryTestCase.java index dc678abe55121..eb66f8b148c00 100644 --- a/libs/geo/src/test/java/org/elasticsearch/geometry/BaseGeometryTestCase.java +++ b/libs/geo/src/test/java/org/elasticsearch/geometry/BaseGeometryTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry; diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/CircleTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/CircleTests.java index 321dd5ccd4e80..70537184f9da0 100644 --- a/libs/geo/src/test/java/org/elasticsearch/geometry/CircleTests.java +++ b/libs/geo/src/test/java/org/elasticsearch/geometry/CircleTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry; diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/GeometryCollectionTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/GeometryCollectionTests.java index b3f7aa610153b..3d5f3e904ddb4 100644 --- a/libs/geo/src/test/java/org/elasticsearch/geometry/GeometryCollectionTests.java +++ b/libs/geo/src/test/java/org/elasticsearch/geometry/GeometryCollectionTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry; diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/GeometryValidatorTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/GeometryValidatorTests.java index 99e4d5a305ed6..90f89e09d04c8 100644 --- a/libs/geo/src/test/java/org/elasticsearch/geometry/GeometryValidatorTests.java +++ b/libs/geo/src/test/java/org/elasticsearch/geometry/GeometryValidatorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry; diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/LineTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/LineTests.java index d3c386883fe08..48edb1be95e2f 100644 --- a/libs/geo/src/test/java/org/elasticsearch/geometry/LineTests.java +++ b/libs/geo/src/test/java/org/elasticsearch/geometry/LineTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry; diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/LinearRingTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/LinearRingTests.java index f154e339d3dd1..bfa14e930777e 100644 --- a/libs/geo/src/test/java/org/elasticsearch/geometry/LinearRingTests.java +++ b/libs/geo/src/test/java/org/elasticsearch/geometry/LinearRingTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry; diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/MultiLineTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/MultiLineTests.java index 68acba5596266..b09bd2f893650 100644 --- a/libs/geo/src/test/java/org/elasticsearch/geometry/MultiLineTests.java +++ b/libs/geo/src/test/java/org/elasticsearch/geometry/MultiLineTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry; diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/MultiPointTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/MultiPointTests.java index b25057182e958..d5bc9664f8f54 100644 --- a/libs/geo/src/test/java/org/elasticsearch/geometry/MultiPointTests.java +++ b/libs/geo/src/test/java/org/elasticsearch/geometry/MultiPointTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry; diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/MultiPolygonTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/MultiPolygonTests.java index ff4a6047f2632..0a9024652b55f 100644 --- a/libs/geo/src/test/java/org/elasticsearch/geometry/MultiPolygonTests.java +++ b/libs/geo/src/test/java/org/elasticsearch/geometry/MultiPolygonTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry; diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/PointTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/PointTests.java index 136c6b8154b1e..b61c36579bfca 100644 --- a/libs/geo/src/test/java/org/elasticsearch/geometry/PointTests.java +++ b/libs/geo/src/test/java/org/elasticsearch/geometry/PointTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry; diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/PolygonTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/PolygonTests.java index 7c9fdcf7bf169..27cab467135c8 100644 --- a/libs/geo/src/test/java/org/elasticsearch/geometry/PolygonTests.java +++ b/libs/geo/src/test/java/org/elasticsearch/geometry/PolygonTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry; diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/RectangleTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/RectangleTests.java index 88315412524b7..7ec637a123935 100644 --- a/libs/geo/src/test/java/org/elasticsearch/geometry/RectangleTests.java +++ b/libs/geo/src/test/java/org/elasticsearch/geometry/RectangleTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry; diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/GeometrySimplifierCartesianTriangleAreaTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/GeometrySimplifierCartesianTriangleAreaTests.java index 384b360361d98..4e517431b3c6e 100644 --- a/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/GeometrySimplifierCartesianTriangleAreaTests.java +++ b/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/GeometrySimplifierCartesianTriangleAreaTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry.simplify; diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/GeometrySimplifierHeightAndBackpathDistanceTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/GeometrySimplifierHeightAndBackpathDistanceTests.java index 8e925325964d5..83f5196518737 100644 --- a/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/GeometrySimplifierHeightAndBackpathDistanceTests.java +++ b/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/GeometrySimplifierHeightAndBackpathDistanceTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry.simplify; diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/GeometrySimplifierSphericalHeightAndBackpathDistanceTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/GeometrySimplifierSphericalHeightAndBackpathDistanceTests.java index b32fa4cab9ad2..317b5fb0fa910 100644 --- a/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/GeometrySimplifierSphericalHeightAndBackpathDistanceTests.java +++ b/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/GeometrySimplifierSphericalHeightAndBackpathDistanceTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry.simplify; diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/GeometrySimplifierTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/GeometrySimplifierTests.java index 866c9dc724392..fe826e5d510ac 100644 --- a/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/GeometrySimplifierTests.java +++ b/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/GeometrySimplifierTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry.simplify; diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/GeometrySimplifierTriangleAreaTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/GeometrySimplifierTriangleAreaTests.java index 47d3ded851405..28bbb3eaa13a0 100644 --- a/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/GeometrySimplifierTriangleAreaTests.java +++ b/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/GeometrySimplifierTriangleAreaTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry.simplify; diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/GeometrySimplifierTriangleHeightTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/GeometrySimplifierTriangleHeightTests.java index 5eb78cb7f89d8..b69de0f8bac2f 100644 --- a/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/GeometrySimplifierTriangleHeightTests.java +++ b/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/GeometrySimplifierTriangleHeightTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry.simplify; diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/SimplificationErrorCalculatorTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/SimplificationErrorCalculatorTests.java index c581959041c97..97e5de883520e 100644 --- a/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/SimplificationErrorCalculatorTests.java +++ b/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/SimplificationErrorCalculatorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry.simplify; diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/Vector3DTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/Vector3DTests.java index 5a88091cbcd4f..e9afd44a628f0 100644 --- a/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/Vector3DTests.java +++ b/libs/geo/src/test/java/org/elasticsearch/geometry/simplify/Vector3DTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry.simplify; diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/utils/CircleUtilsTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/utils/CircleUtilsTests.java index 8d8d97a8f79f0..f0c639c7b4242 100644 --- a/libs/geo/src/test/java/org/elasticsearch/geometry/utils/CircleUtilsTests.java +++ b/libs/geo/src/test/java/org/elasticsearch/geometry/utils/CircleUtilsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry.utils; diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/utils/GeoHashTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/utils/GeoHashTests.java index 615740763e3c2..e530edc76f905 100644 --- a/libs/geo/src/test/java/org/elasticsearch/geometry/utils/GeoHashTests.java +++ b/libs/geo/src/test/java/org/elasticsearch/geometry/utils/GeoHashTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.geometry.utils; diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitorTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitorTests.java new file mode 100644 index 0000000000000..893a1700616a6 --- /dev/null +++ b/libs/geo/src/test/java/org/elasticsearch/geometry/utils/SpatialEnvelopeVisitorTests.java @@ -0,0 +1,196 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.geometry.utils; + +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geo.ShapeTestUtils; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.geometry.utils.SpatialEnvelopeVisitor.WrapLongitude; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; + +public class SpatialEnvelopeVisitorTests extends ESTestCase { + + public void testVisitCartesianShape() { + for (int i = 0; i < 1000; i++) { + var geometry = ShapeTestUtils.randomGeometryWithoutCircle(0, false); + var bbox = SpatialEnvelopeVisitor.visitCartesian(geometry); + assertNotNull(bbox); + assertTrue(i + ": " + geometry, bbox.isPresent()); + var result = bbox.get(); + assertThat(i + ": " + geometry, result.getMinX(), lessThanOrEqualTo(result.getMaxX())); + assertThat(i + ": " + geometry, result.getMinY(), lessThanOrEqualTo(result.getMaxY())); + } + } + + public void testVisitGeoShapeNoWrap() { + for (int i = 0; i < 1000; i++) { + var geometry = GeometryTestUtils.randomGeometryWithoutCircle(0, false); + var bbox = SpatialEnvelopeVisitor.visitGeo(geometry, WrapLongitude.NO_WRAP); + assertNotNull(bbox); + assertTrue(i + ": " + geometry, bbox.isPresent()); + var result = bbox.get(); + assertThat(i + ": " + geometry, result.getMinX(), lessThanOrEqualTo(result.getMaxX())); + assertThat(i + ": " + geometry, result.getMinY(), lessThanOrEqualTo(result.getMaxY())); + } + } + + public void testVisitGeoShapeWrap() { + for (int i = 0; i < 1000; i++) { + var geometry = GeometryTestUtils.randomGeometryWithoutCircle(0, true); + // TODO this should be WRAP instead + var bbox = SpatialEnvelopeVisitor.visitGeo(geometry, WrapLongitude.NO_WRAP); + assertNotNull(bbox); + assertTrue(i + ": " + geometry, bbox.isPresent()); + var result = bbox.get(); + assertThat(i + ": " + geometry, result.getMinX(), lessThanOrEqualTo(result.getMaxX())); + assertThat(i + ": " + geometry, result.getMinY(), lessThanOrEqualTo(result.getMaxY())); + } + } + + public void testVisitCartesianPoints() { + var visitor = new SpatialEnvelopeVisitor(new SpatialEnvelopeVisitor.CartesianPointVisitor()); + double minX = Double.MAX_VALUE; + double minY = Double.MAX_VALUE; + double maxX = -Double.MAX_VALUE; + double maxY = -Double.MAX_VALUE; + for (int i = 0; i < 1000; i++) { + var x = randomFloat(); + var y = randomFloat(); + var point = new Point(x, y); + visitor.visit(point); + minX = Math.min(minX, x); + minY = Math.min(minY, y); + maxX = Math.max(maxX, x); + maxY = Math.max(maxY, y); + var result = visitor.getResult(); + assertThat(i + ": " + point, result.getMinX(), equalTo(minX)); + assertThat(i + ": " + point, result.getMinY(), equalTo(minY)); + assertThat(i + ": " + point, result.getMaxX(), equalTo(maxX)); + assertThat(i + ": " + point, result.getMaxY(), equalTo(maxY)); + } + } + + public void testVisitGeoPointsNoWrapping() { + var visitor = new SpatialEnvelopeVisitor(new SpatialEnvelopeVisitor.GeoPointVisitor(WrapLongitude.NO_WRAP)); + double minY = Double.MAX_VALUE; + double maxY = -Double.MAX_VALUE; + double minX = Double.MAX_VALUE; + double maxX = -Double.MAX_VALUE; + for (int i = 0; i < 1000; i++) { + var point = GeometryTestUtils.randomPoint(); + visitor.visit(point); + minY = Math.min(minY, point.getY()); + maxY = Math.max(maxY, point.getY()); + minX = Math.min(minX, point.getX()); + maxX = Math.max(maxX, point.getX()); + var result = visitor.getResult(); + assertThat(i + ": " + point, result.getMinX(), lessThanOrEqualTo(result.getMaxX())); + assertThat(i + ": " + point, result.getMinX(), equalTo(minX)); + assertThat(i + ": " + point, result.getMinY(), equalTo(minY)); + assertThat(i + ": " + point, result.getMaxX(), equalTo(maxX)); + assertThat(i + ": " + point, result.getMaxY(), equalTo(maxY)); + } + } + + public void testVisitGeoPointsWrapping() { + var visitor = new SpatialEnvelopeVisitor(new SpatialEnvelopeVisitor.GeoPointVisitor(WrapLongitude.WRAP)); + double minY = Double.POSITIVE_INFINITY; + double maxY = Double.NEGATIVE_INFINITY; + double minNegX = Double.POSITIVE_INFINITY; + double maxNegX = Double.NEGATIVE_INFINITY; + double minPosX = Double.POSITIVE_INFINITY; + double maxPosX = Double.NEGATIVE_INFINITY; + for (int i = 0; i < 1000; i++) { + var point = GeometryTestUtils.randomPoint(); + visitor.visit(point); + minY = Math.min(minY, point.getY()); + maxY = Math.max(maxY, point.getY()); + if (point.getX() >= 0) { + minPosX = Math.min(minPosX, point.getX()); + maxPosX = Math.max(maxPosX, point.getX()); + } else { + minNegX = Math.min(minNegX, point.getX()); + maxNegX = Math.max(maxNegX, point.getX()); + } + var result = visitor.getResult(); + if (Double.isInfinite(minPosX)) { + // Only negative x values were considered + assertRectangleResult(i + ": " + point, result, minNegX, maxNegX, maxY, minY, false); + } else if (Double.isInfinite(minNegX)) { + // Only positive x values were considered + assertRectangleResult(i + ": " + point, result, minPosX, maxPosX, maxY, minY, false); + } else { + // Both positive and negative x values exist, we need to decide which way to wrap the bbox + double unwrappedWidth = maxPosX - minNegX; + double wrappedWidth = (180 - minPosX) - (-180 - maxNegX); + if (unwrappedWidth <= wrappedWidth) { + // The smaller bbox is around the front of the planet, no dateline wrapping required + assertRectangleResult(i + ": " + point, result, minNegX, maxPosX, maxY, minY, false); + } else { + // The smaller bbox is around the back of the planet, dateline wrapping required (minx > maxx) + assertRectangleResult(i + ": " + point, result, minPosX, maxNegX, maxY, minY, true); + } + } + } + } + + public void testWillCrossDateline() { + var visitor = new SpatialEnvelopeVisitor(new SpatialEnvelopeVisitor.GeoPointVisitor(WrapLongitude.WRAP)); + visitor.visit(new Point(-90.0, 0.0)); + visitor.visit(new Point(90.0, 0.0)); + assertCrossesDateline(visitor, false); + visitor.visit(new Point(-89.0, 0.0)); + visitor.visit(new Point(89.0, 0.0)); + assertCrossesDateline(visitor, false); + visitor.visit(new Point(-100.0, 0.0)); + visitor.visit(new Point(100.0, 0.0)); + assertCrossesDateline(visitor, true); + visitor.visit(new Point(-70.0, 0.0)); + visitor.visit(new Point(70.0, 0.0)); + assertCrossesDateline(visitor, false); + visitor.visit(new Point(-120.0, 0.0)); + visitor.visit(new Point(120.0, 0.0)); + assertCrossesDateline(visitor, true); + } + + private void assertCrossesDateline(SpatialEnvelopeVisitor visitor, boolean crossesDateline) { + var result = visitor.getResult(); + if (crossesDateline) { + assertThat("Crosses dateline, minx>maxx", result.getMinX(), greaterThanOrEqualTo(result.getMaxX())); + } else { + assertThat("Does not cross dateline, minx 'http://www.apache.org/licenses/LICENSE-2.0')]) licenseFile.set(rootProject.file('licenses/APACHE-LICENSE-2.0.txt')) tasks.withType(LicenseHeadersTask.class).configureEach { approvedLicenses = ['Apache', 'Generated', 'Vendored'] -} \ No newline at end of file +} diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/BaseCells.java b/libs/h3/src/main/java/org/elasticsearch/h3/BaseCells.java index b15c86c17ab83..24b60686ff224 100644 --- a/libs/h3/src/main/java/org/elasticsearch/h3/BaseCells.java +++ b/libs/h3/src/main/java/org/elasticsearch/h3/BaseCells.java @@ -27,27 +27,14 @@ */ final class BaseCells { - private static class BaseCellData { - // "home" face and normalized ijk coordinates on that face - final int homeFace; - final int homeI; - final int homeJ; - final int homeK; - // is this base cell a pentagon? - final boolean isPentagon; - // if a pentagon, what are its two clockwise offset - final int[] cwOffsetPent; - - /// faces? - BaseCellData(int homeFace, int homeI, int homeJ, int homeK, boolean isPentagon, int[] cwOffsetPent) { - this.homeFace = homeFace; - this.homeI = homeI; - this.homeJ = homeJ; - this.homeK = homeK; - this.isPentagon = isPentagon; - this.cwOffsetPent = cwOffsetPent; - } - } + private record BaseCellData( + int homeFace, // "home" face and normalized ijk coordinates on that face + int homeI, + int homeJ, + int homeK, + boolean isPentagon, // is this base cell a pentagon? + int[] cwOffsetPent // if a pentagon, what are its two clockwise offset + ) {} /** * Resolution 0 base cell data table. @@ -185,16 +172,10 @@ private static class BaseCellData { /** * base cell at a given ijk and required rotations into its system */ - private static class BaseCellRotation { - final int baseCell; // base cell number - final int ccwRot60; // number of ccw 60 degree rotations relative to current - /// face - - BaseCellRotation(int baseCell, int ccwRot60) { - this.baseCell = baseCell; - this.ccwRot60 = ccwRot60; - } - } + record BaseCellRotation( + int baseCell, // base cell number + int ccwRot60 // number of ccw 60 degree rotations relative to current + ) {} /** @brief Resolution 0 base cell lookup table for each face. * diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/CellBoundary.java b/libs/h3/src/main/java/org/elasticsearch/h3/CellBoundary.java index 74115d5a002d6..e0f9df174c2b5 100644 --- a/libs/h3/src/main/java/org/elasticsearch/h3/CellBoundary.java +++ b/libs/h3/src/main/java/org/elasticsearch/h3/CellBoundary.java @@ -22,36 +22,52 @@ */ package org.elasticsearch.h3; +import java.util.Arrays; +import java.util.Objects; + /** * cell boundary points as {@link LatLng} */ public final class CellBoundary { - /** Maximum number of cell boundary vertices; worst case is pentagon: * 5 original verts + 5 edge crossings */ - private static final int MAX_CELL_BNDRY_VERTS = 10; + static final int MAX_CELL_BNDRY_VERTS = 10; /** How many points it holds */ - private int numVertext; + private final int numPoints; /** The actual points */ - private final LatLng[] points = new LatLng[MAX_CELL_BNDRY_VERTS]; - - CellBoundary() {} + private final LatLng[] points; - void add(LatLng point) { - points[numVertext++] = point; + CellBoundary(LatLng[] points, int numPoints) { + this.points = points; + this.numPoints = numPoints; } /** Number of points in this boundary */ public int numPoints() { - return numVertext; + return numPoints; } /** Return the point at the given position*/ public LatLng getLatLon(int i) { - if (i >= numVertext) { - throw new IndexOutOfBoundsException(); - } + assert i >= 0 && i < numPoints; return points[i]; } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final CellBoundary that = (CellBoundary) o; + return numPoints == that.numPoints && Arrays.equals(points, that.points); + } + + @Override + public int hashCode() { + return Objects.hash(numPoints, Arrays.hashCode(points)); + } } diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/Constants.java b/libs/h3/src/main/java/org/elasticsearch/h3/Constants.java index 5192fe836e73d..3b3f760c0534f 100644 --- a/libs/h3/src/main/java/org/elasticsearch/h3/Constants.java +++ b/libs/h3/src/main/java/org/elasticsearch/h3/Constants.java @@ -33,11 +33,7 @@ final class Constants { /** * 2.0 * PI */ - public static final double M_2PI = 6.28318530717958647692528676655900576839433; - /** - * max H3 resolution; H3 version 1 has 16 resolutions, numbered 0 through 15 - */ - public static int MAX_H3_RES = 15; + public static final double M_2PI = 2.0 * Math.PI; /** * The number of H3 base cells */ @@ -58,6 +54,11 @@ final class Constants { * square root of 7 */ public static final double M_SQRT7 = 2.6457513110645905905016157536392604257102; + + /** + * 1 / square root of 7 + */ + public static final double M_RSQRT7 = 1.0 / M_SQRT7; /** * scaling factor from hex2d resolution 0 unit length * (or distance between adjacent cell center points diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/CoordIJK.java b/libs/h3/src/main/java/org/elasticsearch/h3/CoordIJK.java index e57f681fc2eae..bfb5f662dee8f 100644 --- a/libs/h3/src/main/java/org/elasticsearch/h3/CoordIJK.java +++ b/libs/h3/src/main/java/org/elasticsearch/h3/CoordIJK.java @@ -39,6 +39,9 @@ */ final class CoordIJK { + /** one seventh (1/7) **/ + private static final double M_ONESEVENTH = 1.0 / 7.0; + /** CoordIJK unit vectors corresponding to the 7 H3 digits. */ private static final int[][] UNIT_VECS = { @@ -106,8 +109,8 @@ void reset(int i, int j, int k) { * Find the center point in 2D cartesian coordinates of a hex. */ public Vec2d ijkToHex2d() { - final int i = Math.subtractExact(this.i, this.k); - final int j = Math.subtractExact(this.j, this.k); + final int i = this.i - this.k; + final int j = this.j - this.k; return new Vec2d(i - 0.5 * j, j * Constants.M_SQRT3_2); } @@ -115,8 +118,8 @@ public Vec2d ijkToHex2d() { * Find the center point in spherical coordinates of a hex on a particular icosahedral face. */ public LatLng ijkToGeo(int face, int res, boolean substrate) { - final int i = Math.subtractExact(this.i, this.k); - final int j = Math.subtractExact(this.j, this.k); + final int i = this.i - this.k; + final int j = this.j - this.k; return Vec2d.hex2dToGeo(i - 0.5 * j, j * Constants.M_SQRT3_2, face, res, substrate); } @@ -129,9 +132,9 @@ public LatLng ijkToGeo(int face, int res, boolean substrate) { */ public void ijkAdd(int i, int j, int k) { - this.i = Math.addExact(this.i, i); - this.j = Math.addExact(this.j, j); - this.k = Math.addExact(this.k, k); + this.i += i; + this.j += j; + this.k += k; } /** @@ -142,9 +145,9 @@ public void ijkAdd(int i, int j, int k) { * @param k the k coordinate */ public void ijkSub(int i, int j, int k) { - this.i = Math.subtractExact(this.i, i); - this.j = Math.subtractExact(this.j, j); - this.k = Math.subtractExact(this.k, k); + this.i -= i; + this.j -= j; + this.k -= k; } /** @@ -165,9 +168,9 @@ public void downAp7() { // iVec (3, 0, 1) // jVec (1, 3, 0) // kVec (0, 1, 3) - final int i = Math.addExact(Math.multiplyExact(this.i, 3), this.j); - final int j = Math.addExact(Math.multiplyExact(this.j, 3), this.k); - final int k = Math.addExact(Math.multiplyExact(this.k, 3), this.i); + final int i = this.i * 3 + this.j; + final int j = this.j * 3 + this.k; + final int k = this.k * 3 + this.i; this.i = i; this.j = j; this.k = k; @@ -182,9 +185,9 @@ public void downAp7r() { // iVec (3, 1, 0) // jVec (0, 3, 1) // kVec (1, 0, 3) - final int i = Math.addExact(Math.multiplyExact(this.i, 3), this.k); - final int j = Math.addExact(Math.multiplyExact(this.j, 3), this.i); - final int k = Math.addExact(Math.multiplyExact(this.k, 3), this.j); + final int i = this.i * 3 + this.k; + final int j = this.j * 3 + this.i; + final int k = this.k * 3 + this.j; this.i = i; this.j = j; this.k = k; @@ -200,9 +203,9 @@ public void downAp3() { // iVec (2, 0, 1) // jVec (1, 2, 0) // kVec (0, 1, 2) - final int i = Math.addExact(Math.multiplyExact(this.i, 2), this.j); - final int j = Math.addExact(Math.multiplyExact(this.j, 2), this.k); - final int k = Math.addExact(Math.multiplyExact(this.k, 2), this.i); + final int i = this.i * 2 + this.j; + final int j = this.j * 2 + this.k; + final int k = this.k * 2 + this.i; this.i = i; this.j = j; this.k = k; @@ -218,9 +221,9 @@ public void downAp3r() { // iVec (2, 1, 0) // jVec (0, 2, 1) // kVec (1, 0, 2) - final int i = Math.addExact(Math.multiplyExact(this.i, 2), this.k); - final int j = Math.addExact(Math.multiplyExact(this.j, 2), this.i); - final int k = Math.addExact(Math.multiplyExact(this.k, 2), this.j); + final int i = this.i * 2 + this.k; + final int j = this.j * 2 + this.i; + final int k = this.k * 2 + this.j; this.i = i; this.j = j; this.k = k; @@ -236,9 +239,9 @@ public void ijkRotate60cw() { // iVec (1, 0, 1) // jVec (1, 1, 0) // kVec (0, 1, 1) - final int i = Math.addExact(this.i, this.j); - final int j = Math.addExact(this.j, this.k); - final int k = Math.addExact(this.i, this.k); + final int i = this.i + this.j; + final int j = this.j + this.k; + final int k = this.i + this.k; this.i = i; this.j = j; this.k = k; @@ -253,9 +256,9 @@ public void ijkRotate60ccw() { // iVec (1, 1, 0) // jVec (0, 1, 1) // kVec (1, 0, 1) - final int i = Math.addExact(this.i, this.k); - final int j = Math.addExact(this.i, this.j); - final int k = Math.addExact(this.j, this.k); + final int i = this.i + this.k; + final int j = this.i + this.j; + final int k = this.j + this.k; this.i = i; this.j = j; this.k = k; @@ -279,10 +282,10 @@ public void neighbor(int digit) { * clockwise aperture 7 grid. */ public void upAp7r() { - final int i = Math.subtractExact(this.i, this.k); - final int j = Math.subtractExact(this.j, this.k); - this.i = (int) Math.round((Math.addExact(Math.multiplyExact(2, i), j)) / 7.0); - this.j = (int) Math.round((Math.subtractExact(Math.multiplyExact(3, j), i)) / 7.0); + final int i = this.i - this.k; + final int j = this.j - this.k; + this.i = (int) Math.round((2 * i + j) * M_ONESEVENTH); + this.j = (int) Math.round((3 * j - i) * M_ONESEVENTH); this.k = 0; ijkNormalize(); } @@ -293,10 +296,10 @@ public void upAp7r() { * */ public void upAp7() { - final int i = Math.subtractExact(this.i, this.k); - final int j = Math.subtractExact(this.j, this.k); - this.i = (int) Math.round((Math.subtractExact(Math.multiplyExact(3, i), j)) / 7.0); - this.j = (int) Math.round((Math.addExact(Math.multiplyExact(2, j), i)) / 7.0); + final int i = this.i - this.k; + final int j = this.j - this.k; + this.i = (int) Math.round((3 * i - j) * M_ONESEVENTH); + this.j = (int) Math.round((2 * j + i) * M_ONESEVENTH); this.k = 0; ijkNormalize(); } @@ -360,5 +363,4 @@ public static int rotate60ccw(int digit) { default -> digit; }; } - } diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/FaceIJK.java b/libs/h3/src/main/java/org/elasticsearch/h3/FaceIJK.java index df2ab26ca0686..a5744ed5eb6bc 100644 --- a/libs/h3/src/main/java/org/elasticsearch/h3/FaceIJK.java +++ b/libs/h3/src/main/java/org/elasticsearch/h3/FaceIJK.java @@ -149,25 +149,13 @@ enum Overage { /** * Information to transform into an adjacent face IJK system */ - private static class FaceOrientIJK { - // face number - final int face; - // res 0 translation relative to primary face - final int translateI; - final int translateJ; - final int translateK; - // number of 60 degree ccw rotations relative to primary - final int ccwRot60; - - // face - FaceOrientIJK(int face, int translateI, int translateJ, int translateK, int ccwRot60) { - this.face = face; - this.translateI = translateI; - this.translateJ = translateJ; - this.translateK = translateK; - this.ccwRot60 = ccwRot60; - } - } + private record FaceOrientIJK( + int face, // face number + int translateI, // res 0 translation relative to primary face + int translateJ, + int translateK, + int ccwRot60// number of 60 degree ccw rotations relative to primary + ) {} /** * Definition of which faces neighbor each other. @@ -429,42 +417,64 @@ public LatLng faceIjkToGeo(int res) { * for this FaceIJK address at a specified resolution. * * @param res The H3 resolution of the cell. - * @param start The first topological vertex to return. - * @param length The number of topological vertexes to return. */ - public CellBoundary faceIjkPentToCellBoundary(int res, int start, int length) { + public CellBoundary faceIjkPentToCellBoundary(int res) { // adjust the center point to be in an aperture 33r substrate grid // these should be composed for speed this.coord.downAp3(); this.coord.downAp3r(); // if res is Class III we need to add a cw aperture 7 to get to // icosahedral Class II - int adjRes = res; - if (H3Index.isResolutionClassIII(res)) { - this.coord.downAp7r(); - adjRes += 1; - } + final int adjRes = adjustRes(this.coord, res); + // If we're returning the entire loop, we need one more iteration in case // of a distortion vertex on the last edge - final int additionalIteration = length == Constants.NUM_PENT_VERTS ? 1 : 0; - final boolean isResolutionClassIII = H3Index.isResolutionClassIII(res); - // convert each vertex to lat/lng - // adjust the face of each vertex as appropriate and introduce - // edge-crossing vertices as needed - final CellBoundary boundary = new CellBoundary(); - final CoordIJK scratch = new CoordIJK(0, 0, 0); - final FaceIJK fijk = new FaceIJK(this.face, scratch); - final int[][] coord = isResolutionClassIII ? VERTEX_CLASSIII : VERTEX_CLASSII; + if (H3Index.isResolutionClassIII(res)) { + return faceIjkPentToCellBoundaryClassIII(adjRes); + } else { + return faceIjkPentToCellBoundaryClassII(adjRes); + } + } + + private CellBoundary faceIjkPentToCellBoundaryClassII(int adjRes) { + final LatLng[] points = new LatLng[Constants.NUM_PENT_VERTS]; + final FaceIJK fijk = new FaceIJK(this.face, new CoordIJK(0, 0, 0)); + for (int vert = 0; vert < Constants.NUM_PENT_VERTS; vert++) { + // The center point is now in the same substrate grid as the origin + // cell vertices. Add the center point substate coordinates + // to each vertex to translate the vertices to that cell. + fijk.coord.reset( + VERTEX_CLASSII[vert][0] + this.coord.i, + VERTEX_CLASSII[vert][1] + this.coord.j, + VERTEX_CLASSII[vert][2] + this.coord.k + ); + fijk.coord.ijkNormalize(); + fijk.face = this.face; + + fijk.adjustPentVertOverage(adjRes); + + points[vert] = fijk.coord.ijkToGeo(fijk.face, adjRes, true); + } + return new CellBoundary(points, Constants.NUM_PENT_VERTS); + } + + private CellBoundary faceIjkPentToCellBoundaryClassIII(int adjRes) { + final LatLng[] points = new LatLng[CellBoundary.MAX_CELL_BNDRY_VERTS]; + int numPoints = 0; + final FaceIJK fijk = new FaceIJK(this.face, new CoordIJK(0, 0, 0)); final CoordIJK lastCoord = new CoordIJK(0, 0, 0); int lastFace = this.face; - for (int vert = start; vert < start + length + additionalIteration; vert++) { + for (int vert = 0; vert < Constants.NUM_PENT_VERTS + 1; vert++) { final int v = vert % Constants.NUM_PENT_VERTS; // The center point is now in the same substrate grid as the origin // cell vertices. Add the center point substate coordinates // to each vertex to translate the vertices to that cell. - scratch.reset(coord[v][0], coord[v][1], coord[v][2]); - scratch.ijkAdd(this.coord.i, this.coord.j, this.coord.k); - scratch.ijkNormalize(); + fijk.coord.reset( + VERTEX_CLASSIII[v][0] + this.coord.i, + VERTEX_CLASSIII[v][1] + this.coord.j, + VERTEX_CLASSIII[v][2] + this.coord.k + ); + fijk.coord.ijkNormalize(); fijk.face = this.face; fijk.adjustPentVertOverage(adjRes); @@ -472,7 +482,7 @@ public CellBoundary faceIjkPentToCellBoundary(int res, int start, int length) { // all Class III pentagon edges cross icosa edges // note that Class II pentagons have vertices on the edge, // not edge intersections - if (isResolutionClassIII && vert > start) { + if (vert > 0) { // find hex2d of the two vertexes on the last face final Vec2d orig2d0 = lastCoord.ijkToHex2d(); @@ -486,52 +496,28 @@ public CellBoundary faceIjkPentToCellBoundary(int res, int start, int length) { } final int unitScale = unitScaleByCIIres[adjRes] * 3; - lastCoord.ijkAdd( - Math.multiplyExact(fijkOrient.translateI, unitScale), - Math.multiplyExact(fijkOrient.translateJ, unitScale), - Math.multiplyExact(fijkOrient.translateK, unitScale) - ); + lastCoord.ijkAdd(fijkOrient.translateI * unitScale, fijkOrient.translateJ * unitScale, fijkOrient.translateK * unitScale); lastCoord.ijkNormalize(); final Vec2d orig2d1 = lastCoord.ijkToHex2d(); - // find the appropriate icosa face edge vertexes - final Vec2d edge0; - final Vec2d edge1; - switch (adjacentFaceDir[fijkOrient.face][fijk.face]) { - case IJ -> { - edge0 = maxDimByCIIVec2d[adjRes][0]; - edge1 = maxDimByCIIVec2d[adjRes][1]; - } - case JK -> { - edge0 = maxDimByCIIVec2d[adjRes][1]; - edge1 = maxDimByCIIVec2d[adjRes][2]; - } - // case KI: - default -> { - assert (adjacentFaceDir[fijkOrient.face][fijk.face] == KI); - edge0 = maxDimByCIIVec2d[adjRes][2]; - edge1 = maxDimByCIIVec2d[adjRes][0]; - } - } - // find the intersection and add the lat/lng point to the result - final Vec2d inter = Vec2d.v2dIntersect(orig2d0, orig2d1, edge0, edge1); - final LatLng point = inter.hex2dToGeo(fijkOrient.face, adjRes, true); - boundary.add(point); + final Vec2d inter = findIntersectionPoint(orig2d0, orig2d1, adjRes, adjacentFaceDir[fijkOrient.face][fijk.face]); + if (inter != null) { + points[numPoints++] = inter.hex2dToGeo(fijkOrient.face, adjRes, true); + } } // convert vertex to lat/lng and add to the result // vert == start + NUM_PENT_VERTS is only used to test for possible // intersection on last edge - if (vert < start + Constants.NUM_PENT_VERTS) { - final LatLng point = fijk.coord.ijkToGeo(fijk.face, adjRes, true); - boundary.add(point); + if (vert < Constants.NUM_PENT_VERTS) { + points[numPoints++] = fijk.coord.ijkToGeo(fijk.face, adjRes, true); } lastFace = fijk.face; lastCoord.reset(fijk.coord.i, fijk.coord.j, fijk.coord.k); } - return boundary; + return new CellBoundary(points, numPoints); } /** @@ -539,10 +525,8 @@ public CellBoundary faceIjkPentToCellBoundary(int res, int start, int length) { * FaceIJK address at a specified resolution. * * @param res The H3 resolution of the cell. - * @param start The first topological vertex to return. - * @param length The number of topological vertexes to return. */ - public CellBoundary faceIjkToCellBoundary(final int res, final int start, final int length) { + public CellBoundary faceIjkToCellBoundary(final int res) { // adjust the center point to be in an aperture 33r substrate grid // these should be composed for speed this.coord.downAp3(); @@ -550,31 +534,63 @@ public CellBoundary faceIjkToCellBoundary(final int res, final int start, final // if res is Class III we need to add a cw aperture 7 to get to // icosahedral Class II - int adjRes = res; - if (H3Index.isResolutionClassIII(res)) { - this.coord.downAp7r(); - adjRes += 1; - } + final int adjRes = adjustRes(this.coord, res); - // If we're returning the entire loop, we need one more iteration in case - // of a distortion vertex on the last edge - final int additionalIteration = length == Constants.NUM_HEX_VERTS ? 1 : 0; - final boolean isResolutionClassIII = H3Index.isResolutionClassIII(res); // convert each vertex to lat/lng // adjust the face of each vertex as appropriate and introduce // edge-crossing vertices as needed - final CellBoundary boundary = new CellBoundary(); - final CoordIJK scratch1 = new CoordIJK(0, 0, 0); - final FaceIJK fijk = new FaceIJK(this.face, scratch1); - final CoordIJK scratch2 = isResolutionClassIII ? new CoordIJK(0, 0, 0) : null; - final int[][] verts = isResolutionClassIII ? VERTEX_CLASSIII : VERTEX_CLASSII; + if (H3Index.isResolutionClassIII(res)) { + return faceIjkToCellBoundaryClassIII(adjRes); + } else { + return faceIjkToCellBoundaryClassII(adjRes); + } + } + + private static int adjustRes(CoordIJK coord, int res) { + if (H3Index.isResolutionClassIII(res)) { + coord.downAp7r(); + res += 1; + } + return res; + } + + private CellBoundary faceIjkToCellBoundaryClassII(int adjRes) { + final LatLng[] points = new LatLng[Constants.NUM_HEX_VERTS]; + final FaceIJK fijk = new FaceIJK(this.face, new CoordIJK(0, 0, 0)); + for (int vert = 0; vert < Constants.NUM_HEX_VERTS; vert++) { + fijk.coord.reset( + VERTEX_CLASSII[vert][0] + this.coord.i, + VERTEX_CLASSII[vert][1] + this.coord.j, + VERTEX_CLASSII[vert][2] + this.coord.k + ); + fijk.coord.ijkNormalize(); + fijk.face = this.face; + + fijk.adjustOverageClassII(adjRes, false, true); + + // convert vertex to lat/lng and add to the result + // vert == start + NUM_HEX_VERTS is only used to test for possible + // intersection on last edge + points[vert] = fijk.coord.ijkToGeo(fijk.face, adjRes, true); + } + return new CellBoundary(points, Constants.NUM_HEX_VERTS); + } + + private CellBoundary faceIjkToCellBoundaryClassIII(int adjRes) { + final LatLng[] points = new LatLng[CellBoundary.MAX_CELL_BNDRY_VERTS]; + int numPoints = 0; + final FaceIJK fijk = new FaceIJK(this.face, new CoordIJK(0, 0, 0)); + final CoordIJK scratch = new CoordIJK(0, 0, 0); int lastFace = -1; Overage lastOverage = Overage.NO_OVERAGE; - for (int vert = start; vert < start + length + additionalIteration; vert++) { - int v = vert % Constants.NUM_HEX_VERTS; - scratch1.reset(verts[v][0], verts[v][1], verts[v][2]); - scratch1.ijkAdd(this.coord.i, this.coord.j, this.coord.k); - scratch1.ijkNormalize(); + for (int vert = 0; vert < Constants.NUM_HEX_VERTS + 1; vert++) { + final int v = vert % Constants.NUM_HEX_VERTS; + fijk.coord.reset( + VERTEX_CLASSIII[v][0] + this.coord.i, + VERTEX_CLASSIII[v][1] + this.coord.j, + VERTEX_CLASSIII[v][2] + this.coord.k + ); + fijk.coord.ijkNormalize(); fijk.face = this.face; final Overage overage = fijk.adjustOverageClassII(adjRes, false, true); @@ -588,74 +604,70 @@ public CellBoundary faceIjkToCellBoundary(final int res, final int start, final projection. Note that Class II cell edges have vertices on the face edge, with no edge line intersections. */ - if (isResolutionClassIII && vert > start && fijk.face != lastFace && lastOverage != Overage.FACE_EDGE) { + if (vert > 0 && fijk.face != lastFace && lastOverage != Overage.FACE_EDGE) { // find hex2d of the two vertexes on original face final int lastV = (v + 5) % Constants.NUM_HEX_VERTS; // The center point is now in the same substrate grid as the origin // cell vertices. Add the center point substate coordinates // to each vertex to translate the vertices to that cell. - final int[] vertexLast = verts[lastV]; - final int[] vertexV = verts[v]; - scratch2.reset( - Math.addExact(vertexLast[0], this.coord.i), - Math.addExact(vertexLast[1], this.coord.j), - Math.addExact(vertexLast[2], this.coord.k) - ); - scratch2.ijkNormalize(); - final Vec2d orig2d0 = scratch2.ijkToHex2d(); - scratch2.reset( - Math.addExact(vertexV[0], this.coord.i), - Math.addExact(vertexV[1], this.coord.j), - Math.addExact(vertexV[2], this.coord.k) - ); - scratch2.ijkNormalize(); - final Vec2d orig2d1 = scratch2.ijkToHex2d(); + final Vec2d orig2d0 = orig(scratch, VERTEX_CLASSIII[lastV]); + final Vec2d orig2d1 = orig(scratch, VERTEX_CLASSIII[v]); // find the appropriate icosa face edge vertexes final int face2 = ((lastFace == this.face) ? fijk.face : lastFace); - final Vec2d edge0; - final Vec2d edge1; - switch (adjacentFaceDir[this.face][face2]) { - case IJ -> { - edge0 = maxDimByCIIVec2d[adjRes][0]; - edge1 = maxDimByCIIVec2d[adjRes][1]; - } - case JK -> { - edge0 = maxDimByCIIVec2d[adjRes][1]; - edge1 = maxDimByCIIVec2d[adjRes][2]; - } - // case KI: - default -> { - assert (adjacentFaceDir[this.face][face2] == KI); - edge0 = maxDimByCIIVec2d[adjRes][2]; - edge1 = maxDimByCIIVec2d[adjRes][0]; - } - } // find the intersection and add the lat/lng point to the result - final Vec2d inter = Vec2d.v2dIntersect(orig2d0, orig2d1, edge0, edge1); - /* - If a point of intersection occurs at a hexagon vertex, then each - adjacent hexagon edge will lie completely on a single icosahedron - face, and no additional vertex is required. - */ - final boolean isIntersectionAtVertex = orig2d0.numericallyIdentical(inter) || orig2d1.numericallyIdentical(inter); - if (isIntersectionAtVertex == false) { - final LatLng point = inter.hex2dToGeo(this.face, adjRes, true); - boundary.add(point); + final Vec2d inter = findIntersectionPoint(orig2d0, orig2d1, adjRes, adjacentFaceDir[this.face][face2]); + if (inter != null) { + points[numPoints++] = inter.hex2dToGeo(this.face, adjRes, true); } } // convert vertex to lat/lng and add to the result // vert == start + NUM_HEX_VERTS is only used to test for possible // intersection on last edge - if (vert < start + Constants.NUM_HEX_VERTS) { - final LatLng point = fijk.coord.ijkToGeo(fijk.face, adjRes, true); - boundary.add(point); + if (vert < Constants.NUM_HEX_VERTS) { + points[numPoints++] = fijk.coord.ijkToGeo(fijk.face, adjRes, true); } lastFace = fijk.face; lastOverage = overage; } - return boundary; + return new CellBoundary(points, numPoints); + } + + private Vec2d orig(CoordIJK scratch, int[] vertexLast) { + scratch.reset(vertexLast[0] + this.coord.i, vertexLast[1] + this.coord.j, vertexLast[2] + this.coord.k); + scratch.ijkNormalize(); + return scratch.ijkToHex2d(); + } + + private Vec2d findIntersectionPoint(Vec2d orig2d0, Vec2d orig2d1, int adjRes, int faceDir) { + // find the appropriate icosa face edge vertexes + final Vec2d edge0; + final Vec2d edge1; + switch (faceDir) { + case IJ -> { + edge0 = maxDimByCIIVec2d[adjRes][0]; + edge1 = maxDimByCIIVec2d[adjRes][1]; + } + case JK -> { + edge0 = maxDimByCIIVec2d[adjRes][1]; + edge1 = maxDimByCIIVec2d[adjRes][2]; + } + // case KI: + default -> { + assert (faceDir == KI); + edge0 = maxDimByCIIVec2d[adjRes][2]; + edge1 = maxDimByCIIVec2d[adjRes][0]; + } + } + // find the intersection and add the lat/lng point to the result + final Vec2d inter = Vec2d.v2dIntersect(orig2d0, orig2d1, edge0, edge1); + /* + If a point of intersection occurs at a hexagon vertex, then each + adjacent hexagon edge will lie completely on a single icosahedron + face, and no additional vertex is required. + */ + return orig2d0.numericallyIdentical(inter) || orig2d1.numericallyIdentical(inter) ? null : inter; } /** @@ -677,7 +689,6 @@ static long faceIjkToH3(int res, int face, CoordIJK coord) { // out of range input throw new IllegalArgumentException(" out of range input"); } - return H3Index.H3_set_base_cell(h, BaseCells.getBaseCell(face, coord)); } @@ -704,7 +715,7 @@ static long faceIjkToH3(int res, int face, CoordIJK coord) { scratch.reset(coord.i, coord.j, coord.k); scratch.downAp7r(); } - scratch.reset(Math.subtractExact(lastI, scratch.i), Math.subtractExact(lastJ, scratch.j), Math.subtractExact(lastK, scratch.k)); + scratch.reset(lastI - scratch.i, lastJ - scratch.j, lastK - scratch.k); scratch.ijkNormalize(); h = H3Index.H3_set_index_digit(h, r, scratch.unitIjkToDigit()); } diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/FastMath.java b/libs/h3/src/main/java/org/elasticsearch/h3/FastMath.java index 61d767901ae0c..760fa75535487 100644 --- a/libs/h3/src/main/java/org/elasticsearch/h3/FastMath.java +++ b/libs/h3/src/main/java/org/elasticsearch/h3/FastMath.java @@ -102,6 +102,15 @@ final class FastMath { private static final int MIN_DOUBLE_EXPONENT = -1074; private static final int MAX_DOUBLE_EXPONENT = 1023; + /** + * PI / 2.0 + */ + private static final double M_HALF_PI = Math.PI * 0.5; + /** + * PI / 4.0 + */ + private static final double M_QUARTER_PI = Math.PI * 0.25; + // -------------------------------------------------------------------------- // CONSTANTS FOR NORMALIZATIONS // -------------------------------------------------------------------------- @@ -335,7 +344,7 @@ public static double cos(double angle) { // Faster than using normalizeZeroTwoPi. angle = remainderTwoPi(angle); if (angle < 0.0) { - angle += 2 * Math.PI; + angle += Constants.M_2PI; } } // index: possibly outside tables range. @@ -366,7 +375,7 @@ public static double sin(double angle) { // Faster than using normalizeZeroTwoPi. angle = remainderTwoPi(angle); if (angle < 0.0) { - angle += 2 * Math.PI; + angle += Constants.M_2PI; } } int index = (int) (angle * SIN_COS_INDEXER + 0.5); @@ -387,9 +396,9 @@ public static double tan(double angle) { if (Math.abs(angle) > TAN_MAX_VALUE_FOR_INT_MODULO) { // Faster than using normalizeMinusHalfPiHalfPi. angle = remainderTwoPi(angle); - if (angle < -Math.PI / 2) { + if (angle < -M_HALF_PI) { angle += Math.PI; - } else if (angle > Math.PI / 2) { + } else if (angle > M_HALF_PI) { angle -= Math.PI; } } @@ -428,7 +437,7 @@ public static double tan(double angle) { * @return Value arccosine, in radians, in [0,PI]. */ public static double acos(double value) { - return Math.PI / 2 - FastMath.asin(value); + return M_HALF_PI - FastMath.asin(value); } /** @@ -468,7 +477,7 @@ public static double asin(double value) { return negateResult ? -result : result; } else { // value >= 1.0, or value is NaN if (value == 1.0) { - return negateResult ? -Math.PI / 2 : Math.PI / 2; + return negateResult ? -M_HALF_PI : M_HALF_PI; } else { return Double.NaN; } @@ -490,7 +499,7 @@ public static double atan(double value) { } if (value == 1.0) { // We want "exact" result for 1.0. - return negateResult ? -Math.PI / 4 : Math.PI / 4; + return negateResult ? -M_QUARTER_PI : M_QUARTER_PI; } else if (value <= ATAN_MAX_VALUE_FOR_TABS) { int index = (int) (value * ATAN_INDEXER + 0.5); double delta = value - index * ATAN_DELTA; @@ -511,7 +520,7 @@ public static double atan(double value) { if (Double.isNaN(value)) { return Double.NaN; } else { - return negateResult ? -Math.PI / 2 : Math.PI / 2; + return negateResult ? -M_HALF_PI : M_HALF_PI; } } } @@ -532,9 +541,9 @@ public static double atan2(double y, double x) { } if (x == Double.POSITIVE_INFINITY) { if (y == Double.POSITIVE_INFINITY) { - return Math.PI / 4; + return M_QUARTER_PI; } else if (y == Double.NEGATIVE_INFINITY) { - return -Math.PI / 4; + return -M_QUARTER_PI; } else if (y > 0.0) { return 0.0; } else if (y < 0.0) { @@ -551,9 +560,9 @@ public static double atan2(double y, double x) { } if (x == Double.NEGATIVE_INFINITY) { if (y == Double.POSITIVE_INFINITY) { - return 3 * Math.PI / 4; + return 3 * M_QUARTER_PI; } else if (y == Double.NEGATIVE_INFINITY) { - return -3 * Math.PI / 4; + return -3 * M_QUARTER_PI; } else if (y > 0.0) { return Math.PI; } else if (y < 0.0) { @@ -562,9 +571,9 @@ public static double atan2(double y, double x) { return Double.NaN; } } else if (y > 0.0) { - return Math.PI / 2 + FastMath.atan(-x / y); + return M_HALF_PI + FastMath.atan(-x / y); } else if (y < 0.0) { - return -Math.PI / 2 - FastMath.atan(x / y); + return -M_HALF_PI - FastMath.atan(x / y); } else { return Double.NaN; } @@ -577,9 +586,9 @@ public static double atan2(double y, double x) { } } if (y > 0.0) { - return Math.PI / 2; + return M_HALF_PI; } else if (y < 0.0) { - return -Math.PI / 2; + return -M_HALF_PI; } else { return Double.NaN; } diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/H3.java b/libs/h3/src/main/java/org/elasticsearch/h3/H3.java index 46bcc3f141dde..08031088728ba 100644 --- a/libs/h3/src/main/java/org/elasticsearch/h3/H3.java +++ b/libs/h3/src/main/java/org/elasticsearch/h3/H3.java @@ -30,8 +30,10 @@ * Defines the public API of the H3 library. */ public final class H3 { - - public static int MAX_H3_RES = Constants.MAX_H3_RES; + /** + * max H3 resolution; H3 version 1 has 16 resolutions, numbered 0 through 15 + */ + public static int MAX_H3_RES = 15; private static final long[] NORTH = new long[MAX_H3_RES + 1]; private static final long[] SOUTH = new long[MAX_H3_RES + 1]; @@ -97,7 +99,7 @@ public static boolean h3IsValid(long h3) { } int res = H3Index.H3_get_resolution(h3); - if (res < 0 || res > Constants.MAX_H3_RES) { // LCOV_EXCL_BR_LINE + if (res < 0 || res > MAX_H3_RES) { // LCOV_EXCL_BR_LINE // Resolutions less than zero can not be represented in an index return false; } @@ -118,7 +120,7 @@ public static boolean h3IsValid(long h3) { } } - for (int r = res + 1; r <= Constants.MAX_H3_RES; r++) { + for (int r = res + 1; r <= MAX_H3_RES; r++) { int digit = H3Index.H3_get_index_digit(h3, r); if (digit != CoordIJK.Direction.INVALID_DIGIT.digit()) { return false; @@ -172,11 +174,11 @@ public static LatLng h3ToLatLng(String h3Address) { * Find the cell {@link CellBoundary} coordinates for the cell */ public static CellBoundary h3ToGeoBoundary(long h3) { - FaceIJK fijk = H3Index.h3ToFaceIjk(h3); + final FaceIJK fijk = H3Index.h3ToFaceIjk(h3); if (H3Index.H3_is_pentagon(h3)) { - return fijk.faceIjkPentToCellBoundary(H3Index.H3_get_resolution(h3), 0, Constants.NUM_PENT_VERTS); + return fijk.faceIjkPentToCellBoundary(H3Index.H3_get_resolution(h3)); } else { - return fijk.faceIjkToCellBoundary(H3Index.H3_get_resolution(h3), 0, Constants.NUM_HEX_VERTS); + return fijk.faceIjkToCellBoundary(H3Index.H3_get_resolution(h3)); } } @@ -601,7 +603,7 @@ private static String[] h3ToStringList(long[] h3s) { * @throws IllegalArgumentException res is not a valid H3 resolution. */ private static void checkResolution(int res) { - if (res < 0 || res > Constants.MAX_H3_RES) { + if (res < 0 || res > MAX_H3_RES) { throw new IllegalArgumentException("resolution [" + res + "] is out of range (must be 0 <= res <= 15)"); } } diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/H3Index.java b/libs/h3/src/main/java/org/elasticsearch/h3/H3Index.java index 6d7af86a9a537..2b1b9cade21a4 100644 --- a/libs/h3/src/main/java/org/elasticsearch/h3/H3Index.java +++ b/libs/h3/src/main/java/org/elasticsearch/h3/H3Index.java @@ -160,14 +160,14 @@ public static int H3_get_resolution(long h3) { * Gets the resolution res integer digit (0-7) of h3. */ public static int H3_get_index_digit(long h3, int res) { - return ((int) ((((h3) >> ((Constants.MAX_H3_RES - (res)) * H3_PER_DIGIT_OFFSET)) & H3_DIGIT_MASK))); + return ((int) ((((h3) >> ((H3.MAX_H3_RES - (res)) * H3_PER_DIGIT_OFFSET)) & H3_DIGIT_MASK))); } /** * Sets the resolution res digit of h3 to the integer digit (0-7) */ public static long H3_set_index_digit(long h3, int res, long digit) { - int x = (Constants.MAX_H3_RES - res) * H3_PER_DIGIT_OFFSET; + int x = (H3.MAX_H3_RES - res) * H3_PER_DIGIT_OFFSET; return (((h3) & ~((H3_DIGIT_MASK << (x)))) | (((digit)) << x)); } @@ -325,7 +325,9 @@ public static long h3RotatePent60ccw(long h) { foundFirstNonZeroDigit = true; // adjust for deleted k-axes sequence - if (h3LeadingNonZeroDigit(h) == CoordIJK.Direction.K_AXES_DIGIT.digit()) h = h3Rotate60ccw(h); + if (h3LeadingNonZeroDigit(h) == CoordIJK.Direction.K_AXES_DIGIT.digit()) { + h = h3Rotate60ccw(h); + } } } return h; diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/HexRing.java b/libs/h3/src/main/java/org/elasticsearch/h3/HexRing.java index d7011aa4d48ce..936f636e6a5ce 100644 --- a/libs/h3/src/main/java/org/elasticsearch/h3/HexRing.java +++ b/libs/h3/src/main/java/org/elasticsearch/h3/HexRing.java @@ -290,11 +290,6 @@ final class HexRing { { 0, 0, 1, 0, 1, 5, 1 }, // base cell 121 }; - private static final int E_SUCCESS = 0; // Success (no error) - private static final int E_PENTAGON = 9; // Pentagon distortion was encountered which the algorithm - private static final int E_CELL_INVALID = 5; // `H3Index` cell argument was not valid - private static final int E_FAILED = 1; // The operation failed but a more specific error is not available - /** * Directions used for traversing a hexagonal ring counterclockwise around * {1, 0, 0} diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/Vec2d.java b/libs/h3/src/main/java/org/elasticsearch/h3/Vec2d.java index 12ce728a99967..3b6f26aa6357a 100644 --- a/libs/h3/src/main/java/org/elasticsearch/h3/Vec2d.java +++ b/libs/h3/src/main/java/org/elasticsearch/h3/Vec2d.java @@ -27,10 +27,16 @@ /** * 2D floating-point vector */ -final class Vec2d { +record Vec2d( + double x, // x component + double y // y component +) { - /** sin(60') */ - private static final double M_SIN60 = Constants.M_SQRT3_2; + /** 1/sin(60') **/ + private static final double M_RSIN60 = 1.0 / Constants.M_SQRT3_2; + + /** one third **/ + private static final double M_ONETHIRD = 1.0 / 3.0; private static final double VEC2D_RESOLUTION = 1e-7; @@ -87,14 +93,6 @@ final class Vec2d { { 2.361378999196363184, 0.266983896803167583, 4.455774101589558636 }, // face 19 }; - private final double x; /// < x component - private final double y; /// < y component - - Vec2d(double x, double y) { - this.x = x; - this.y = y; - } - /** * Determines the center point in spherical coordinates of a cell given by this 2D * hex coordinates on a particular icosahedral face. @@ -133,14 +131,14 @@ static LatLng hex2dToGeo(double x, double y, int face, int res, boolean substrat // scale for current resolution length u for (int i = 0; i < res; i++) { - r /= Constants.M_SQRT7; + r *= Constants.M_RSQRT7; } // scale accordingly if this is a substrate grid if (substrate) { - r /= 3.0; + r *= M_ONETHIRD; if (H3Index.isResolutionClassIII(res)) { - r /= Constants.M_SQRT7; + r *= Constants.M_RSQRT7; } } @@ -181,8 +179,8 @@ static CoordIJK hex2dToCoordIJK(double x, double y) { a2 = Math.abs(y); // first do a reverse conversion - x2 = a2 / M_SIN60; - x1 = a1 + x2 / 2.0; + x2 = a2 * M_RSIN60; + x1 = a1 + x2 * 0.5; // check if we have the center of a hex m1 = (int) x1; @@ -193,47 +191,47 @@ static CoordIJK hex2dToCoordIJK(double x, double y) { r2 = x2 - m2; if (r1 < 0.5) { - if (r1 < 1.0 / 3.0) { - if (r2 < (1.0 + r1) / 2.0) { + if (r1 < M_ONETHIRD) { + if (r2 < (1.0 + r1) * 0.5) { i = m1; j = m2; } else { i = m1; - j = Math.incrementExact(m2); + j = m2 + 1; } } else { if (r2 < (1.0 - r1)) { j = m2; } else { - j = Math.incrementExact(m2); + j = m2 + 1; } if ((1.0 - r1) <= r2 && r2 < (2.0 * r1)) { - i = Math.incrementExact(m1); + i = m1 + 1; } else { i = m1; } } } else { - if (r1 < 2.0 / 3.0) { + if (r1 < 2.0 * M_ONETHIRD) { if (r2 < (1.0 - r1)) { j = m2; } else { - j = Math.addExact(m2, 1); + j = m2 + 1; } if ((2.0 * r1 - 1.0) < r2 && r2 < (1.0 - r1)) { i = m1; } else { - i = Math.incrementExact(m1); + i = m1 + 1; } } else { - if (r2 < (r1 / 2.0)) { - i = Math.incrementExact(m1); + if (r2 < (r1 * 0.5)) { + i = m1 + 1; j = m2; } else { - i = Math.incrementExact(m1); - j = Math.incrementExact(m2); + i = m1 + 1; + j = m2 + 1; } } } @@ -244,18 +242,19 @@ static CoordIJK hex2dToCoordIJK(double x, double y) { if ((j % 2) == 0) // even { final int axisi = j / 2; - final int diff = Math.subtractExact(i, axisi); - i = Math.subtractExact(i, Math.multiplyExact(2, diff)); + final int diff = i - axisi; + i = i - (2 * diff); } else { - final int axisi = Math.addExact(j, 1) / 2; - final int diff = Math.subtractExact(i, axisi); - i = Math.subtractExact(i, Math.addExact(Math.multiplyExact(2, diff), 1)); + final int axisi = (j + 1) / 2; + final int diff = i - axisi; + i = i - ((2 * diff) + 1); } } if (y < 0.0) { - i = Math.subtractExact(i, Math.addExact(Math.multiplyExact(2, j), 1) / 2); - j = Math.multiplyExact(-1, j); + + i = i - ((2 * j + 1) / 2); + j *= -1; } final CoordIJK coordIJK = new CoordIJK(i, j, k); coordIJK.ijkNormalize(); diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/Vec3d.java b/libs/h3/src/main/java/org/elasticsearch/h3/Vec3d.java index c5c4f8975597c..05f504d8e031d 100644 --- a/libs/h3/src/main/java/org/elasticsearch/h3/Vec3d.java +++ b/libs/h3/src/main/java/org/elasticsearch/h3/Vec3d.java @@ -26,7 +26,7 @@ /** * 3D floating-point vector */ -final class Vec3d { +record Vec3d(double x, double y, double z) { /** icosahedron face centers in x/y/z on the unit sphere */ public static final Vec3d[] faceCenterPoint = new Vec3d[] { @@ -52,14 +52,6 @@ final class Vec3d { new Vec3d(-0.1092625278784796, 0.4811951572873210, -0.8697775121287253) // face 19 }; - private final double x, y, z; - - private Vec3d(double x, double y, double z) { - this.x = x; - this.y = y; - this.z = z; - } - /** * Calculate the square of the distance between two 3D coordinates. * @@ -96,7 +88,7 @@ static long geoToH3(int res, double lat, double lon) { } } // cos(r) = 1 - 2 * sin^2(r/2) = 1 - 2 * (sqd / 4) = 1 - sqd/2 - double r = FastMath.acos(1 - sqd / 2); + double r = FastMath.acos(1 - sqd * 0.5); if (r < Constants.EPSILON) { return FaceIJK.faceIjkToH3(res, face, new CoordIJK(0, 0, 0)); @@ -238,5 +230,4 @@ private static double dotProduct(double x1, double y1, double z1, double x2, dou private static double magnitude(double x, double y, double z) { return Math.sqrt(square(x) + square(y) + square(z)); } - } diff --git a/libs/h3/src/test/java/org/elasticsearch/h3/CellBoundaryTests.java b/libs/h3/src/test/java/org/elasticsearch/h3/CellBoundaryTests.java index 903e4ed40ec16..00ca6f7021e3d 100644 --- a/libs/h3/src/test/java/org/elasticsearch/h3/CellBoundaryTests.java +++ b/libs/h3/src/test/java/org/elasticsearch/h3/CellBoundaryTests.java @@ -218,4 +218,22 @@ private boolean isSharedBoundary(int clon1, int clat1, int clon2, int clat2, Cel } return false; } + + public void testEqualsAndHashCode() { + final long h3 = H3.geoToH3(GeoTestUtil.nextLatitude(), GeoTestUtil.nextLongitude(), randomIntBetween(0, 15)); + final CellBoundary boundary1 = H3.h3ToGeoBoundary(h3); + final CellBoundary boundary2 = H3.h3ToGeoBoundary(h3); + assertEquals(boundary1, boundary2); + assertEquals(boundary1.hashCode(), boundary2.hashCode()); + + final long otherH3 = H3.geoToH3(GeoTestUtil.nextLatitude(), GeoTestUtil.nextLongitude(), randomIntBetween(0, 15)); + final CellBoundary otherCellBoundary = H3.h3ToGeoBoundary(otherH3); + if (otherH3 != h3) { + assertNotEquals(boundary1, otherCellBoundary); + assertNotEquals(boundary1.hashCode(), otherCellBoundary.hashCode()); + } else { + assertEquals(boundary1, otherCellBoundary); + assertEquals(boundary1.hashCode(), otherCellBoundary.hashCode()); + } + } } diff --git a/libs/h3/src/test/java/org/elasticsearch/h3/GeoToH3Tests.java b/libs/h3/src/test/java/org/elasticsearch/h3/GeoToH3Tests.java index cb7d416a5a9d3..3f2c329d9ff3c 100644 --- a/libs/h3/src/test/java/org/elasticsearch/h3/GeoToH3Tests.java +++ b/libs/h3/src/test/java/org/elasticsearch/h3/GeoToH3Tests.java @@ -38,7 +38,7 @@ public void testRandomPoints() { private void testPoint(double lat, double lon) { GeoPoint point = new GeoPoint(PlanetModel.SPHERE, Math.toRadians(lat), Math.toRadians(lon)); - for (int res = 0; res < Constants.MAX_H3_RES; res++) { + for (int res = 0; res < H3.MAX_H3_RES; res++) { String h3Address = H3.geoToH3Address(lat, lon, res); assertEquals(res, H3.getResolution(h3Address)); GeoPolygon polygon = getGeoPolygon(h3Address); diff --git a/libs/h3/src/test/java/org/elasticsearch/h3/HexRingTests.java b/libs/h3/src/test/java/org/elasticsearch/h3/HexRingTests.java index 8fe5c6206fff8..864c0322cac90 100644 --- a/libs/h3/src/test/java/org/elasticsearch/h3/HexRingTests.java +++ b/libs/h3/src/test/java/org/elasticsearch/h3/HexRingTests.java @@ -38,7 +38,7 @@ public void testHexRing() { for (int i = 0; i < 500; i++) { double lat = GeoTestUtil.nextLatitude(); double lon = GeoTestUtil.nextLongitude(); - for (int res = 0; res <= Constants.MAX_H3_RES; res++) { + for (int res = 0; res <= H3.MAX_H3_RES; res++) { String origin = H3.geoToH3Address(lat, lon, res); assertFalse(H3.areNeighborCells(origin, origin)); String[] ring = H3.hexRing(origin); diff --git a/libs/logging/build.gradle b/libs/logging/build.gradle index 3004af029cb55..f52c2629176a7 100644 --- a/libs/logging/build.gradle +++ b/libs/logging/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.publish' @@ -13,12 +14,12 @@ tasks.named("loggerUsageCheck").configure {enabled = false } dependencies { testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-logging' + exclude group: 'org.elasticsearch', module: 'logging' } } tasks.named('forbiddenApisMain').configure { - // :libs:elasticsearch-logging does not depend on server + // :libs:logging does not depend on server replaceSignatureFiles 'jdk-signatures' } diff --git a/libs/logging/src/main/java/module-info.java b/libs/logging/src/main/java/module-info.java index 8edd98e943043..3ff21cefd14df 100644 --- a/libs/logging/src/main/java/module-info.java +++ b/libs/logging/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.logging { diff --git a/libs/logging/src/main/java/org/elasticsearch/logging/Level.java b/libs/logging/src/main/java/org/elasticsearch/logging/Level.java index 4ac9e3e3e754a..3cad8f2c16697 100644 --- a/libs/logging/src/main/java/org/elasticsearch/logging/Level.java +++ b/libs/logging/src/main/java/org/elasticsearch/logging/Level.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.logging; diff --git a/libs/logging/src/main/java/org/elasticsearch/logging/LogManager.java b/libs/logging/src/main/java/org/elasticsearch/logging/LogManager.java index 4cd92961f7a33..67932f644c6bf 100644 --- a/libs/logging/src/main/java/org/elasticsearch/logging/LogManager.java +++ b/libs/logging/src/main/java/org/elasticsearch/logging/LogManager.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.logging; diff --git a/libs/logging/src/main/java/org/elasticsearch/logging/Logger.java b/libs/logging/src/main/java/org/elasticsearch/logging/Logger.java index ef9980ce0f6cd..cf014d85bc9fb 100644 --- a/libs/logging/src/main/java/org/elasticsearch/logging/Logger.java +++ b/libs/logging/src/main/java/org/elasticsearch/logging/Logger.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.logging; diff --git a/libs/logging/src/main/java/org/elasticsearch/logging/internal/spi/LoggerFactory.java b/libs/logging/src/main/java/org/elasticsearch/logging/internal/spi/LoggerFactory.java index cbe7e2a55af23..d5db919197307 100644 --- a/libs/logging/src/main/java/org/elasticsearch/logging/internal/spi/LoggerFactory.java +++ b/libs/logging/src/main/java/org/elasticsearch/logging/internal/spi/LoggerFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.logging.internal.spi; diff --git a/libs/logging/src/main/java/org/elasticsearch/logging/package-info.java b/libs/logging/src/main/java/org/elasticsearch/logging/package-info.java index ea608840574da..6595df00332cf 100644 --- a/libs/logging/src/main/java/org/elasticsearch/logging/package-info.java +++ b/libs/logging/src/main/java/org/elasticsearch/logging/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/libs/logstash-bridge/build.gradle b/libs/logstash-bridge/build.gradle index 28fd6149fd7d8..117bed1e98105 100644 --- a/libs/logstash-bridge/build.gradle +++ b/libs/logstash-bridge/build.gradle @@ -1,17 +1,18 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.build' dependencies { compileOnly project(':server') - compileOnly project(':libs:elasticsearch-core') - compileOnly project(':libs:elasticsearch-plugin-api') - compileOnly project(':libs:elasticsearch-x-content') + compileOnly project(':libs:core') + compileOnly project(':libs:plugin-api') + compileOnly project(':libs:x-content') compileOnly project(':modules:lang-painless') compileOnly project(':modules:lang-painless:spi') compileOnly project(':modules:lang-mustache') diff --git a/libs/logstash-bridge/src/main/java/module-info.java b/libs/logstash-bridge/src/main/java/module-info.java index 49b0e13c14cd4..ca345b8c6c4f3 100644 --- a/libs/logstash-bridge/src/main/java/module-info.java +++ b/libs/logstash-bridge/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** Elasticsearch Logstash Bridge. */ diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/StableBridgeAPI.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/StableBridgeAPI.java index cdf2ab4ee7be3..4963adc889147 100644 --- a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/StableBridgeAPI.java +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/StableBridgeAPI.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.logstashbridge; diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/common/SettingsBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/common/SettingsBridge.java index 86fd0fcf75658..304f4c902bcef 100644 --- a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/common/SettingsBridge.java +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/common/SettingsBridge.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.logstashbridge.common; diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/core/IOUtilsBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/core/IOUtilsBridge.java index 810c671e5b8eb..1e300bee6881f 100644 --- a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/core/IOUtilsBridge.java +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/core/IOUtilsBridge.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.logstashbridge.core; diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/env/EnvironmentBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/env/EnvironmentBridge.java index 8ae3ce2d33d28..236b05dd034e9 100644 --- a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/env/EnvironmentBridge.java +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/env/EnvironmentBridge.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.logstashbridge.env; diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/ConfigurationUtilsBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/ConfigurationUtilsBridge.java index 2d7f5c27b16e0..760bd12a5a9da 100644 --- a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/ConfigurationUtilsBridge.java +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/ConfigurationUtilsBridge.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.logstashbridge.ingest; diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/IngestDocumentBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/IngestDocumentBridge.java index 5135034485392..da97be39d97c5 100644 --- a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/IngestDocumentBridge.java +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/IngestDocumentBridge.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.logstashbridge.ingest; diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/PipelineBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/PipelineBridge.java index 835e377c71b31..b75b144ed616d 100644 --- a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/PipelineBridge.java +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/PipelineBridge.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.logstashbridge.ingest; diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/PipelineConfigurationBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/PipelineConfigurationBridge.java index d2aff89d1f236..cb90d10665659 100644 --- a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/PipelineConfigurationBridge.java +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/PipelineConfigurationBridge.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.logstashbridge.ingest; @@ -27,8 +28,12 @@ public String getId() { return delegate.getId(); } - public Map getConfigAsMap() { - return delegate.getConfigAsMap(); + public Map getConfig() { + return delegate.getConfig(); + } + + public Map getConfig(final boolean unmodifiable) { + return delegate.getConfig(unmodifiable); } @Override diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/ProcessorBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/ProcessorBridge.java index 7b88b12eb3c1c..577865f55c752 100644 --- a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/ProcessorBridge.java +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/ProcessorBridge.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.logstashbridge.ingest; diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/plugins/IngestPluginBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/plugins/IngestPluginBridge.java index a27eaa9063dda..49a719e576c41 100644 --- a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/plugins/IngestPluginBridge.java +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/plugins/IngestPluginBridge.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.logstashbridge.plugins; diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/script/MetadataBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/script/MetadataBridge.java index 4f0a712ca3505..e4dbb3bc8b4ee 100644 --- a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/script/MetadataBridge.java +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/script/MetadataBridge.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.logstashbridge.script; diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/script/ScriptServiceBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/script/ScriptServiceBridge.java index ec5af0f7020ac..1f7a19e333308 100644 --- a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/script/ScriptServiceBridge.java +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/script/ScriptServiceBridge.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.logstashbridge.script; @@ -52,7 +53,7 @@ private static ScriptService getScriptService(final Settings settings, final Lon PainlessScriptEngine.NAME, new PainlessScriptEngine(settings, scriptContexts), MustacheScriptEngine.NAME, - new MustacheScriptEngine() + new MustacheScriptEngine(settings) ); return new ScriptService(settings, scriptEngines, ScriptModule.CORE_CONTEXTS, timeProvider); } diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/script/TemplateScriptBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/script/TemplateScriptBridge.java index 715b357a4ee70..a184a51676048 100644 --- a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/script/TemplateScriptBridge.java +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/script/TemplateScriptBridge.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.logstashbridge.script; diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/threadpool/ThreadPoolBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/threadpool/ThreadPoolBridge.java index 30801b4f0b078..d16cca1dd865a 100644 --- a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/threadpool/ThreadPoolBridge.java +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/threadpool/ThreadPoolBridge.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.logstashbridge.threadpool; diff --git a/libs/lz4/build.gradle b/libs/lz4/build.gradle index 367da95ea6577..72e1bb50187a7 100644 --- a/libs/lz4/build.gradle +++ b/libs/lz4/build.gradle @@ -1,18 +1,19 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.publish' dependencies { api 'org.lz4:lz4-java:1.8.0' - api project(':libs:elasticsearch-core') + api project(':libs:core') testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-lz4' + exclude group: 'org.elasticsearch', module: 'lz4' } } diff --git a/libs/lz4/src/main/java/module-info.java b/libs/lz4/src/main/java/module-info.java index 15b37c251a6a8..936910fbffb20 100644 --- a/libs/lz4/src/main/java/module-info.java +++ b/libs/lz4/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.lz4 { diff --git a/libs/lz4/src/test/java/org/elasticsearch/lz4/ESLZ4CompressorTests.java b/libs/lz4/src/test/java/org/elasticsearch/lz4/ESLZ4CompressorTests.java index 10c8c8a3a4566..e02202da40de3 100644 --- a/libs/lz4/src/test/java/org/elasticsearch/lz4/ESLZ4CompressorTests.java +++ b/libs/lz4/src/test/java/org/elasticsearch/lz4/ESLZ4CompressorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.lz4; diff --git a/libs/lz4/src/test/java/org/elasticsearch/lz4/ESLZ4DecompressorTests.java b/libs/lz4/src/test/java/org/elasticsearch/lz4/ESLZ4DecompressorTests.java index 79607de5140bb..b81d3c77ed79e 100644 --- a/libs/lz4/src/test/java/org/elasticsearch/lz4/ESLZ4DecompressorTests.java +++ b/libs/lz4/src/test/java/org/elasticsearch/lz4/ESLZ4DecompressorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.lz4; diff --git a/libs/native/build.gradle b/libs/native/build.gradle index bfcba01b4783a..2f54b3f7a6a82 100644 --- a/libs/native/build.gradle +++ b/libs/native/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.internal.precommit.CheckForbiddenApisTask @@ -14,14 +15,14 @@ apply plugin: 'elasticsearch.mrjar' apply plugin: 'elasticsearch.embedded-providers' embeddedProviders { - impl 'native-access-jna', project(':libs:elasticsearch-native:jna') + impl 'native-access-jna', project(':libs:native:jna') } dependencies { - api project(':libs:elasticsearch-core') - api project(':libs:elasticsearch-logging') + api project(':libs:core') + api project(':libs:logging') testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-native' + exclude group: 'org.elasticsearch', module: 'native' } } diff --git a/libs/native/jna/build.gradle b/libs/native/jna/build.gradle index 679191afbc574..1ce664c4600a7 100644 --- a/libs/native/jna/build.gradle +++ b/libs/native/jna/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.build' @@ -13,12 +14,12 @@ base { } dependencies { - compileOnly project(':libs:elasticsearch-core') - compileOnly project(':libs:elasticsearch-native') + compileOnly project(':libs:core') + compileOnly project(':libs:native') implementation "net.java.dev.jna:jna:${versions.jna}" testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-native' + exclude group: 'org.elasticsearch', module: 'native' } } diff --git a/libs/native/jna/src/main/java/module-info.java b/libs/native/jna/src/main/java/module-info.java index 6e8b0847ce030..8567a6a0edd99 100644 --- a/libs/native/jna/src/main/java/module-info.java +++ b/libs/native/jna/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.nativeaccess.jna.JnaNativeLibraryProvider; diff --git a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaCloseableByteBuffer.java b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaCloseableByteBuffer.java index e987f8042691b..434942ccdcc7f 100644 --- a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaCloseableByteBuffer.java +++ b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaCloseableByteBuffer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.jna; diff --git a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaJavaLibrary.java b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaJavaLibrary.java index 8526968863688..cf816689ae2cc 100644 --- a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaJavaLibrary.java +++ b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaJavaLibrary.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.jna; diff --git a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaKernel32Library.java b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaKernel32Library.java index 1403806c595a7..7d2a6b56fd376 100644 --- a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaKernel32Library.java +++ b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaKernel32Library.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.jna; diff --git a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaLinuxCLibrary.java b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaLinuxCLibrary.java index ca3137ab5df0e..fc023d2b83805 100644 --- a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaLinuxCLibrary.java +++ b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaLinuxCLibrary.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.jna; diff --git a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaMacCLibrary.java b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaMacCLibrary.java index f416cf862b417..5964dbd3fb0ea 100644 --- a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaMacCLibrary.java +++ b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaMacCLibrary.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.jna; diff --git a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaNativeLibraryProvider.java b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaNativeLibraryProvider.java index e0233187425ea..6639304641d74 100644 --- a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaNativeLibraryProvider.java +++ b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaNativeLibraryProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.jna; diff --git a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaPosixCLibrary.java b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaPosixCLibrary.java index 82a69e4864d94..21e5bcb7d65e9 100644 --- a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaPosixCLibrary.java +++ b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaPosixCLibrary.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.jna; diff --git a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaZstdLibrary.java b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaZstdLibrary.java index e52f36f7c8255..d53070be04e8d 100644 --- a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaZstdLibrary.java +++ b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaZstdLibrary.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.jna; diff --git a/libs/native/libraries/build.gradle b/libs/native/libraries/build.gradle index 8f1a12055bd7e..ed37d4a70931f 100644 --- a/libs/native/libraries/build.gradle +++ b/libs/native/libraries/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.transform.UnzipTransform @@ -24,7 +25,7 @@ repositories { exclusiveContent { forRepository { maven { - url "https://artifactory.elastic.dev/artifactory/elasticsearch-native" + url = "https://artifactory.elastic.dev/artifactory/elasticsearch-native" metadataSources { artifact() } diff --git a/libs/native/src/main/java/module-info.java b/libs/native/src/main/java/module-info.java index 226503b24832d..494a1b7ddb846 100644 --- a/libs/native/src/main/java/module-info.java +++ b/libs/native/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.jdk.ModuleQualifiedExportsService; diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/AbstractNativeAccess.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/AbstractNativeAccess.java index c10f57a900ff7..677563194704c 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/AbstractNativeAccess.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/AbstractNativeAccess.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess; diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/CloseableByteBuffer.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/CloseableByteBuffer.java index 6590aff307cc6..44081b644a129 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/CloseableByteBuffer.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/CloseableByteBuffer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess; diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/LinuxNativeAccess.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/LinuxNativeAccess.java index e1ea28e8786f5..b174c0c531714 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/LinuxNativeAccess.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/LinuxNativeAccess.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess; diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/MacNativeAccess.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/MacNativeAccess.java index f277c69de3192..e5d62c960dd79 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/MacNativeAccess.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/MacNativeAccess.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess; diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/NativeAccess.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/NativeAccess.java index 0534bc10e910a..142171c5bbfcc 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/NativeAccess.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/NativeAccess.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess; diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/NativeAccessHolder.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/NativeAccessHolder.java index 562e7163cd098..e1c2b4c4cd8e3 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/NativeAccessHolder.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/NativeAccessHolder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess; diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/NoopNativeAccess.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/NoopNativeAccess.java index ffe65548eeb44..5eaaae6905791 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/NoopNativeAccess.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/NoopNativeAccess.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess; diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/PosixConstants.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/PosixConstants.java index e767e2b3713ec..e34fa9596454a 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/PosixConstants.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/PosixConstants.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess; diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/PosixNativeAccess.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/PosixNativeAccess.java index 1cf08fce0c2af..a2188d0685c3b 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/PosixNativeAccess.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/PosixNativeAccess.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess; diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/ProcessLimits.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/ProcessLimits.java index 41d8c57579eed..0458da8cbfe20 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/ProcessLimits.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/ProcessLimits.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess; diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/Systemd.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/Systemd.java index 058cfe77b1ff3..845c687ccda8b 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/Systemd.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/Systemd.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess; diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/VectorSimilarityFunctions.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/VectorSimilarityFunctions.java index 8da5f1f8052d8..29a298b714fdd 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/VectorSimilarityFunctions.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/VectorSimilarityFunctions.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess; diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/WindowsFunctions.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/WindowsFunctions.java index c57109678a0b8..a79a1451c0141 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/WindowsFunctions.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/WindowsFunctions.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess; diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/WindowsNativeAccess.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/WindowsNativeAccess.java index 5b4a5abad3e0a..aae4938783916 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/WindowsNativeAccess.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/WindowsNativeAccess.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess; diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/Zstd.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/Zstd.java index 60e65383bf9a2..5ddf1b2d85725 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/Zstd.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/Zstd.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess; diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/exports/NativeAccessModuleExportsService.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/exports/NativeAccessModuleExportsService.java index f640b60f8ac76..5c8d61ac918bb 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/exports/NativeAccessModuleExportsService.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/exports/NativeAccessModuleExportsService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.exports; diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/JavaLibrary.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/JavaLibrary.java index 50a3022fa77c2..258ae9923a40e 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/JavaLibrary.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/JavaLibrary.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.lib; diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/Kernel32Library.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/Kernel32Library.java index f35d9fde5950d..8b41473f6accb 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/Kernel32Library.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/Kernel32Library.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.lib; diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/LinuxCLibrary.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/LinuxCLibrary.java index 8a2917e136bde..918a723d6293c 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/LinuxCLibrary.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/LinuxCLibrary.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.lib; diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/LoaderHelper.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/LoaderHelper.java index 42ca60b81a027..d9c725f5a8d3b 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/LoaderHelper.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/LoaderHelper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.lib; diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/MacCLibrary.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/MacCLibrary.java index b2b2db9c71c90..4c6a39a87d38f 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/MacCLibrary.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/MacCLibrary.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.lib; diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibrary.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibrary.java index cdd0a56c52a90..943d8565f0897 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibrary.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibrary.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.lib; diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibraryProvider.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibraryProvider.java index 737d5aecd0d0a..5fc53d4d242d6 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibraryProvider.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibraryProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.lib; diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/PosixCLibrary.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/PosixCLibrary.java index ac34fcb23b3eb..20312bc6e141c 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/PosixCLibrary.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/PosixCLibrary.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.lib; diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/VectorLibrary.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/VectorLibrary.java index 86d1a82b2bdc9..9aa5d36e141b6 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/VectorLibrary.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/VectorLibrary.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.lib; diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/ZstdLibrary.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/ZstdLibrary.java index ea4c8efa5318a..125dcd80d6cfa 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/ZstdLibrary.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/ZstdLibrary.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.lib; diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/ArenaUtil.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/ArenaUtil.java index 0724386cca22c..d57e38214b162 100644 --- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/ArenaUtil.java +++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/ArenaUtil.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.jdk; diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkCloseableByteBuffer.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkCloseableByteBuffer.java index daa012d35598e..56b5b7c8beae4 100644 --- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkCloseableByteBuffer.java +++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkCloseableByteBuffer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.jdk; diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkJavaLibrary.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkJavaLibrary.java index 60a3966463a77..0a67fc70737e6 100644 --- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkJavaLibrary.java +++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkJavaLibrary.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.jdk; diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkKernel32Library.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkKernel32Library.java index 0294b721aa6a8..551a07c14c6b7 100644 --- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkKernel32Library.java +++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkKernel32Library.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.jdk; diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkLinuxCLibrary.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkLinuxCLibrary.java index a31f212eab382..b718f63f213d5 100644 --- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkLinuxCLibrary.java +++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkLinuxCLibrary.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.jdk; diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkMacCLibrary.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkMacCLibrary.java index b946ca3ca4353..2cf916120612f 100644 --- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkMacCLibrary.java +++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkMacCLibrary.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.jdk; diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkNativeLibraryProvider.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkNativeLibraryProvider.java index 1ac7d6c6f897d..1ac0780fcc460 100644 --- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkNativeLibraryProvider.java +++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkNativeLibraryProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.jdk; diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkPosixCLibrary.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkPosixCLibrary.java index f5e3132b76b56..727f271d3a0c0 100644 --- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkPosixCLibrary.java +++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkPosixCLibrary.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.jdk; diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkVectorLibrary.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkVectorLibrary.java index a1032f1381d94..3277cb8f8e6c7 100644 --- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkVectorLibrary.java +++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkVectorLibrary.java @@ -1,13 +1,16 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.jdk; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; import org.elasticsearch.nativeaccess.VectorSimilarityFunctions; import org.elasticsearch.nativeaccess.lib.LoaderHelper; import org.elasticsearch.nativeaccess.lib.VectorLibrary; @@ -24,6 +27,8 @@ public final class JdkVectorLibrary implements VectorLibrary { + static final Logger logger = LogManager.getLogger(JdkVectorLibrary.class); + static final MethodHandle dot7u$mh; static final MethodHandle sqr7u$mh; @@ -35,6 +40,7 @@ public final class JdkVectorLibrary implements VectorLibrary { try { int caps = (int) vecCaps$mh.invokeExact(); + logger.info("vec_caps=" + caps); if (caps != 0) { if (caps == 2) { dot7u$mh = downcallHandle( diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkZstdLibrary.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkZstdLibrary.java index 284ac134d2036..192776e5dd36a 100644 --- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkZstdLibrary.java +++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkZstdLibrary.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.jdk; diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/LinkerHelper.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/LinkerHelper.java index 2f13cb1324e56..eeff2e8a1be79 100644 --- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/LinkerHelper.java +++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/LinkerHelper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.jdk; diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/LinkerHelperUtil.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/LinkerHelperUtil.java index 8befc4bec1275..180910e651128 100644 --- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/LinkerHelperUtil.java +++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/LinkerHelperUtil.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.jdk; diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/MemorySegmentUtil.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/MemorySegmentUtil.java index 6c4c9bd0111c0..380f20f5a382b 100644 --- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/MemorySegmentUtil.java +++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/MemorySegmentUtil.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.jdk; diff --git a/libs/native/src/main22/java/org/elasticsearch/nativeaccess/jdk/ArenaUtil.java b/libs/native/src/main22/java/org/elasticsearch/nativeaccess/jdk/ArenaUtil.java index 387473e23a561..6a7b2b83c385d 100644 --- a/libs/native/src/main22/java/org/elasticsearch/nativeaccess/jdk/ArenaUtil.java +++ b/libs/native/src/main22/java/org/elasticsearch/nativeaccess/jdk/ArenaUtil.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.jdk; diff --git a/libs/native/src/main22/java/org/elasticsearch/nativeaccess/jdk/LinkerHelperUtil.java b/libs/native/src/main22/java/org/elasticsearch/nativeaccess/jdk/LinkerHelperUtil.java index 6ca3aeaa301c7..60620e415c891 100644 --- a/libs/native/src/main22/java/org/elasticsearch/nativeaccess/jdk/LinkerHelperUtil.java +++ b/libs/native/src/main22/java/org/elasticsearch/nativeaccess/jdk/LinkerHelperUtil.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.jdk; diff --git a/libs/native/src/main22/java/org/elasticsearch/nativeaccess/jdk/MemorySegmentUtil.java b/libs/native/src/main22/java/org/elasticsearch/nativeaccess/jdk/MemorySegmentUtil.java index 23d9919603ab4..2b2941c1b2967 100644 --- a/libs/native/src/main22/java/org/elasticsearch/nativeaccess/jdk/MemorySegmentUtil.java +++ b/libs/native/src/main22/java/org/elasticsearch/nativeaccess/jdk/MemorySegmentUtil.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.jdk; diff --git a/libs/native/src/test/java/org/elasticsearch/nativeaccess/PreallocateTests.java b/libs/native/src/test/java/org/elasticsearch/nativeaccess/PreallocateTests.java index a95f4df6aef6f..46613cdf4545a 100644 --- a/libs/native/src/test/java/org/elasticsearch/nativeaccess/PreallocateTests.java +++ b/libs/native/src/test/java/org/elasticsearch/nativeaccess/PreallocateTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess; diff --git a/libs/native/src/test/java/org/elasticsearch/nativeaccess/ProcessLimitsTests.java b/libs/native/src/test/java/org/elasticsearch/nativeaccess/ProcessLimitsTests.java index 5750c95e8892d..6ca9d8c7a6eec 100644 --- a/libs/native/src/test/java/org/elasticsearch/nativeaccess/ProcessLimitsTests.java +++ b/libs/native/src/test/java/org/elasticsearch/nativeaccess/ProcessLimitsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess; diff --git a/libs/native/src/test/java/org/elasticsearch/nativeaccess/SystemCallFilterTests.java b/libs/native/src/test/java/org/elasticsearch/nativeaccess/SystemCallFilterTests.java index d4bac13990898..d730c695c37d0 100644 --- a/libs/native/src/test/java/org/elasticsearch/nativeaccess/SystemCallFilterTests.java +++ b/libs/native/src/test/java/org/elasticsearch/nativeaccess/SystemCallFilterTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess; diff --git a/libs/native/src/test/java/org/elasticsearch/nativeaccess/VectorSimilarityFunctionsTests.java b/libs/native/src/test/java/org/elasticsearch/nativeaccess/VectorSimilarityFunctionsTests.java index 8c4cbb688abcd..53fd6c7f1fa6b 100644 --- a/libs/native/src/test/java/org/elasticsearch/nativeaccess/VectorSimilarityFunctionsTests.java +++ b/libs/native/src/test/java/org/elasticsearch/nativeaccess/VectorSimilarityFunctionsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess; diff --git a/libs/native/src/test/java/org/elasticsearch/nativeaccess/VectorSystemPropertyTests.java b/libs/native/src/test/java/org/elasticsearch/nativeaccess/VectorSystemPropertyTests.java index cda4fc8c55444..4cdf04fe5a294 100644 --- a/libs/native/src/test/java/org/elasticsearch/nativeaccess/VectorSystemPropertyTests.java +++ b/libs/native/src/test/java/org/elasticsearch/nativeaccess/VectorSystemPropertyTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess; diff --git a/libs/native/src/test/java/org/elasticsearch/nativeaccess/ZstdTests.java b/libs/native/src/test/java/org/elasticsearch/nativeaccess/ZstdTests.java index 1282b1fee9206..b6c84d02201c4 100644 --- a/libs/native/src/test/java/org/elasticsearch/nativeaccess/ZstdTests.java +++ b/libs/native/src/test/java/org/elasticsearch/nativeaccess/ZstdTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess; diff --git a/libs/native/src/test21/java/org/elasticsearch/nativeaccess/jdk/JDKVectorLibraryTests.java b/libs/native/src/test21/java/org/elasticsearch/nativeaccess/jdk/JDKVectorLibraryTests.java index cb68dca14cc46..82cefc93f2036 100644 --- a/libs/native/src/test21/java/org/elasticsearch/nativeaccess/jdk/JDKVectorLibraryTests.java +++ b/libs/native/src/test21/java/org/elasticsearch/nativeaccess/jdk/JDKVectorLibraryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nativeaccess.jdk; diff --git a/libs/plugin-analysis-api/build.gradle b/libs/plugin-analysis-api/build.gradle index bda0745a718a3..3f1670d76a0c1 100644 --- a/libs/plugin-analysis-api/build.gradle +++ b/libs/plugin-analysis-api/build.gradle @@ -2,10 +2,11 @@ import org.elasticsearch.gradle.internal.info.BuildParams /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.publish' @@ -17,12 +18,12 @@ tasks.named("loggerUsageCheck").configure {enabled = false } dependencies { api "org.apache.lucene:lucene-core:${versions.lucene}" - api project(':libs:elasticsearch-plugin-api') + api project(':libs:plugin-api') } tasks.named('forbiddenApisMain').configure { - // :libs:elasticsearch-logging does not depend on server + // :libs:logging does not depend on server replaceSignatureFiles 'jdk-signatures' } diff --git a/libs/plugin-analysis-api/src/main/java/module-info.java b/libs/plugin-analysis-api/src/main/java/module-info.java index 4652e6b77f7c5..6237213a91181 100644 --- a/libs/plugin-analysis-api/src/main/java/module-info.java +++ b/libs/plugin-analysis-api/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.plugin.analysis { diff --git a/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/AnalysisMode.java b/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/AnalysisMode.java index 5180b3168fcb5..676750273491d 100644 --- a/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/AnalysisMode.java +++ b/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/AnalysisMode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis; diff --git a/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/AnalyzerFactory.java b/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/AnalyzerFactory.java index dfda8f2929103..29eecd28e46bd 100644 --- a/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/AnalyzerFactory.java +++ b/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/AnalyzerFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis; diff --git a/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/CharFilterFactory.java b/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/CharFilterFactory.java index cb8cbeba01b2e..fa991d7d73dbb 100644 --- a/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/CharFilterFactory.java +++ b/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/CharFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis; diff --git a/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/TokenFilterFactory.java b/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/TokenFilterFactory.java index 671390f995393..7da8df44751c0 100644 --- a/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/TokenFilterFactory.java +++ b/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/TokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis; diff --git a/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/TokenizerFactory.java b/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/TokenizerFactory.java index 9222c296ebd95..1b8414a12ed08 100644 --- a/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/TokenizerFactory.java +++ b/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/TokenizerFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis; diff --git a/libs/plugin-api/build.gradle b/libs/plugin-api/build.gradle index 8dfe7ce9c7937..2d349cd959eb7 100644 --- a/libs/plugin-api/build.gradle +++ b/libs/plugin-api/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.publish' diff --git a/libs/plugin-api/src/main/java/module-info.java b/libs/plugin-api/src/main/java/module-info.java index 03946a839f7b0..b5d0749eff2c1 100644 --- a/libs/plugin-api/src/main/java/module-info.java +++ b/libs/plugin-api/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.plugin { diff --git a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/Extensible.java b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/Extensible.java index feb65d625e48d..1048f8de04e05 100644 --- a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/Extensible.java +++ b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/Extensible.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin; diff --git a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/Inject.java b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/Inject.java index c9b1f4b3c50a7..d9d7448a94f71 100644 --- a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/Inject.java +++ b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/Inject.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin; diff --git a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/Nameable.java b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/Nameable.java index 373aa94b7b8c3..4ec7c54d3f811 100644 --- a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/Nameable.java +++ b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/Nameable.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin; diff --git a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/NamedComponent.java b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/NamedComponent.java index 44f8e6dd1317f..5cc26ab37801d 100644 --- a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/NamedComponent.java +++ b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/NamedComponent.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin; diff --git a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/package-info.java b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/package-info.java index 9e8ad40ef0286..4b0a0e45c14d8 100644 --- a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/package-info.java +++ b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/AnalysisSettings.java b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/AnalysisSettings.java index c0e6a59e04255..20b4d4ca5fb20 100644 --- a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/AnalysisSettings.java +++ b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/AnalysisSettings.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.settings; diff --git a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/BooleanSetting.java b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/BooleanSetting.java index 58749bea7d573..929ca37854017 100644 --- a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/BooleanSetting.java +++ b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/BooleanSetting.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.settings; diff --git a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/IntSetting.java b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/IntSetting.java index 83f4ef40b423e..97003690d1dae 100644 --- a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/IntSetting.java +++ b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/IntSetting.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.settings; diff --git a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/ListSetting.java b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/ListSetting.java index 5d561164bfb46..1dd3927483316 100644 --- a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/ListSetting.java +++ b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/ListSetting.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.settings; diff --git a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/LongSetting.java b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/LongSetting.java index 67af5baf94483..9f88e93f2c291 100644 --- a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/LongSetting.java +++ b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/LongSetting.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.settings; diff --git a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/StringSetting.java b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/StringSetting.java index e16c3af328770..e2826633f729c 100644 --- a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/StringSetting.java +++ b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/StringSetting.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.settings; diff --git a/libs/plugin-classloader/build.gradle b/libs/plugin-classloader/build.gradle deleted file mode 100644 index 102d3ac9c59b7..0000000000000 --- a/libs/plugin-classloader/build.gradle +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -// This is only required because :server needs this at runtime. -// We'll be removing this in 8.0 so for now just publish the JAR to make dependency resolution work. -apply plugin: 'elasticsearch.publish' - -tasks.named("test").configure { enabled = false } - -// test depend on ES core... -tasks.named('forbiddenApisMain').configure { enabled = false} -tasks.named("jarHell").configure { enabled = false } diff --git a/libs/plugin-classloader/src/main/java/module-info.java b/libs/plugin-classloader/src/main/java/module-info.java deleted file mode 100644 index 5129f6cf64224..0000000000000 --- a/libs/plugin-classloader/src/main/java/module-info.java +++ /dev/null @@ -1,11 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -module org.elasticsearch.pluginclassloader { - exports org.elasticsearch.plugins.loader; -} diff --git a/libs/plugin-classloader/src/main/java/org/elasticsearch/plugins/loader/ExtendedPluginsClassLoader.java b/libs/plugin-classloader/src/main/java/org/elasticsearch/plugins/loader/ExtendedPluginsClassLoader.java deleted file mode 100644 index 379cbef9efcbb..0000000000000 --- a/libs/plugin-classloader/src/main/java/org/elasticsearch/plugins/loader/ExtendedPluginsClassLoader.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.plugins.loader; - -import java.security.AccessController; -import java.security.PrivilegedAction; -import java.util.Collections; -import java.util.List; - -/** - * A classloader that is a union over the parent core classloader and classloaders of extended plugins. - */ -public class ExtendedPluginsClassLoader extends ClassLoader { - - /** Loaders of plugins extended by a plugin. */ - private final List extendedLoaders; - - private ExtendedPluginsClassLoader(ClassLoader parent, List extendedLoaders) { - super(parent); - this.extendedLoaders = Collections.unmodifiableList(extendedLoaders); - } - - @Override - protected Class findClass(String name) throws ClassNotFoundException { - for (ClassLoader loader : extendedLoaders) { - try { - return loader.loadClass(name); - } catch (ClassNotFoundException e) { - // continue - } - } - throw new ClassNotFoundException(name); - } - - /** - * Return a new classloader across the parent and extended loaders. - */ - public static ExtendedPluginsClassLoader create(ClassLoader parent, List extendedLoaders) { - return AccessController.doPrivileged( - (PrivilegedAction) () -> new ExtendedPluginsClassLoader(parent, extendedLoaders) - ); - } -} diff --git a/libs/plugin-scanner/build.gradle b/libs/plugin-scanner/build.gradle index 2f7ab5c22b967..44e6853140a5b 100644 --- a/libs/plugin-scanner/build.gradle +++ b/libs/plugin-scanner/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.publish' @@ -15,16 +16,16 @@ tasks.named("dependencyLicenses").configure { } dependencies { - api project(':libs:elasticsearch-core') - api project(':libs:elasticsearch-plugin-api') - api project(":libs:elasticsearch-x-content") + api project(':libs:core') + api project(':libs:plugin-api') + api project(":libs:x-content") - api 'org.ow2.asm:asm:9.7' - api 'org.ow2.asm:asm-tree:9.7' + api 'org.ow2.asm:asm:9.7.1' + api 'org.ow2.asm:asm-tree:9.7.1' testImplementation "junit:junit:${versions.junit}" testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-plugin-scanner' + exclude group: 'org.elasticsearch', module: 'plugin-scanner' } } tasks.named('forbiddenApisMain').configure { diff --git a/libs/plugin-scanner/src/main/java/module-info.java b/libs/plugin-scanner/src/main/java/module-info.java index 31f0782892140..e15dc14a84411 100644 --- a/libs/plugin-scanner/src/main/java/module-info.java +++ b/libs/plugin-scanner/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.plugin.scanner { diff --git a/libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/AnnotatedHierarchyVisitor.java b/libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/AnnotatedHierarchyVisitor.java index a372ff10683ac..2cd26807663df 100644 --- a/libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/AnnotatedHierarchyVisitor.java +++ b/libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/AnnotatedHierarchyVisitor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.scanner; diff --git a/libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/ClassReaders.java b/libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/ClassReaders.java index c4490b0c27a31..bbad11b2b06e6 100644 --- a/libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/ClassReaders.java +++ b/libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/ClassReaders.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.scanner; diff --git a/libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/ClassScanner.java b/libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/ClassScanner.java index 4341ac729cba4..684fdb9edd4ba 100644 --- a/libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/ClassScanner.java +++ b/libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/ClassScanner.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.scanner; diff --git a/libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/NamedComponentScanner.java b/libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/NamedComponentScanner.java index 7a55a6749846a..ac465a64053ae 100644 --- a/libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/NamedComponentScanner.java +++ b/libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/NamedComponentScanner.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.scanner; diff --git a/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/AnnotatedHierarchyVisitorTests.java b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/AnnotatedHierarchyVisitorTests.java index 8aaa36b7f9a04..c6a8fd4c966a9 100644 --- a/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/AnnotatedHierarchyVisitorTests.java +++ b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/AnnotatedHierarchyVisitorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.scanner; diff --git a/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/ClassReadersTests.java b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/ClassReadersTests.java index 764065bc462be..905282b5d48b3 100644 --- a/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/ClassReadersTests.java +++ b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/ClassReadersTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.scanner; diff --git a/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/ClassScannerTests.java b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/ClassScannerTests.java index b8b35dfb568aa..ead12e66cfd1e 100644 --- a/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/ClassScannerTests.java +++ b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/ClassScannerTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.scanner; diff --git a/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/NamedComponentScannerTests.java b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/NamedComponentScannerTests.java index 1b3781c61cf8f..d8a3b13288b16 100644 --- a/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/NamedComponentScannerTests.java +++ b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/NamedComponentScannerTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.scanner; diff --git a/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/ExtensibleClass.java b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/ExtensibleClass.java index ad055a47629fe..26a52841ecf8f 100644 --- a/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/ExtensibleClass.java +++ b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/ExtensibleClass.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.scanner.test_model; diff --git a/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/ExtensibleInterface.java b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/ExtensibleInterface.java index a99f0a1c8da48..ab6fd6ab9d199 100644 --- a/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/ExtensibleInterface.java +++ b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/ExtensibleInterface.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.scanner.test_model; diff --git a/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/ImplementingExtensible.java b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/ImplementingExtensible.java index 2b82972624f53..ef6ac2064806d 100644 --- a/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/ImplementingExtensible.java +++ b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/ImplementingExtensible.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.scanner.test_model; diff --git a/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/SubClass.java b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/SubClass.java index b117090235940..0d6b446e4c7e2 100644 --- a/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/SubClass.java +++ b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/SubClass.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.scanner.test_model; diff --git a/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/TestNamedComponent.java b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/TestNamedComponent.java index 71db778ca4e1c..01d636fa12360 100644 --- a/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/TestNamedComponent.java +++ b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/TestNamedComponent.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.scanner.test_model; diff --git a/libs/secure-sm/build.gradle b/libs/secure-sm/build.gradle index fa1da5541fc9b..473a86215e91e 100644 --- a/libs/secure-sm/build.gradle +++ b/libs/secure-sm/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.publish' @@ -15,7 +16,7 @@ dependencies { testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}" testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-secure-sm' + exclude group: 'org.elasticsearch', module: 'secure-sm' } } diff --git a/libs/secure-sm/src/main/java/module-info.java b/libs/secure-sm/src/main/java/module-info.java index 15c27cbe6ca02..fb3b6e357a1e5 100644 --- a/libs/secure-sm/src/main/java/module-info.java +++ b/libs/secure-sm/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.securesm { diff --git a/libs/secure-sm/src/main/java/org/elasticsearch/secure_sm/SecureSM.java b/libs/secure-sm/src/main/java/org/elasticsearch/secure_sm/SecureSM.java index 68d2f45386c59..02d0491118dc7 100644 --- a/libs/secure-sm/src/main/java/org/elasticsearch/secure_sm/SecureSM.java +++ b/libs/secure-sm/src/main/java/org/elasticsearch/secure_sm/SecureSM.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.secure_sm; @@ -156,7 +157,9 @@ private static void debugThreadGroups(final ThreadGroup caller, final ThreadGrou // Returns true if the given thread is an instance of the JDK's InnocuousThread. private static boolean isInnocuousThread(Thread t) { final Class c = t.getClass(); - return c.getModule() == Object.class.getModule() && c.getName().equals("jdk.internal.misc.InnocuousThread"); + return c.getModule() == Object.class.getModule() + && (c.getName().equals("jdk.internal.misc.InnocuousThread") + || c.getName().equals("java.util.concurrent.ForkJoinWorkerThread$InnocuousForkJoinWorkerThread")); } protected void checkThreadAccess(Thread t) { @@ -183,11 +186,21 @@ protected void checkThreadAccess(Thread t) { private static final Permission MODIFY_THREADGROUP_PERMISSION = new RuntimePermission("modifyThreadGroup"); private static final Permission MODIFY_ARBITRARY_THREADGROUP_PERMISSION = new ThreadPermission("modifyArbitraryThreadGroup"); + // Returns true if the given thread is an instance of the JDK's InnocuousThread. + private static boolean isInnocuousThreadGroup(ThreadGroup t) { + final Class c = t.getClass(); + return c.getModule() == Object.class.getModule() && t.getName().equals("InnocuousForkJoinWorkerThreadGroup"); + } + protected void checkThreadGroupAccess(ThreadGroup g) { Objects.requireNonNull(g); + boolean targetThreadGroupIsInnocuous = isInnocuousThreadGroup(g); + // first, check if we can modify thread groups at all. - checkPermission(MODIFY_THREADGROUP_PERMISSION); + if (targetThreadGroupIsInnocuous == false) { + checkPermission(MODIFY_THREADGROUP_PERMISSION); + } // check the threadgroup, if its our thread group or an ancestor, its fine. final ThreadGroup source = Thread.currentThread().getThreadGroup(); @@ -195,7 +208,7 @@ protected void checkThreadGroupAccess(ThreadGroup g) { if (source == null) { return; // we are a dead thread, do nothing - } else if (source.parentOf(target) == false) { + } else if (source.parentOf(target) == false && targetThreadGroupIsInnocuous == false) { checkPermission(MODIFY_ARBITRARY_THREADGROUP_PERMISSION); } } diff --git a/libs/secure-sm/src/main/java/org/elasticsearch/secure_sm/SuppressForbidden.java b/libs/secure-sm/src/main/java/org/elasticsearch/secure_sm/SuppressForbidden.java index 848aace193dfc..6586097b5ddf4 100644 --- a/libs/secure-sm/src/main/java/org/elasticsearch/secure_sm/SuppressForbidden.java +++ b/libs/secure-sm/src/main/java/org/elasticsearch/secure_sm/SuppressForbidden.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.secure_sm; diff --git a/libs/secure-sm/src/main/java/org/elasticsearch/secure_sm/ThreadPermission.java b/libs/secure-sm/src/main/java/org/elasticsearch/secure_sm/ThreadPermission.java index cc29ab90dc416..caae4acd888ef 100644 --- a/libs/secure-sm/src/main/java/org/elasticsearch/secure_sm/ThreadPermission.java +++ b/libs/secure-sm/src/main/java/org/elasticsearch/secure_sm/ThreadPermission.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.secure_sm; diff --git a/libs/secure-sm/src/test/java/org/elasticsearch/secure_sm/SecureSMTests.java b/libs/secure-sm/src/test/java/org/elasticsearch/secure_sm/SecureSMTests.java index 6109bde551f52..69c6973f57cdf 100644 --- a/libs/secure-sm/src/test/java/org/elasticsearch/secure_sm/SecureSMTests.java +++ b/libs/secure-sm/src/test/java/org/elasticsearch/secure_sm/SecureSMTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.secure_sm; @@ -13,7 +14,10 @@ import java.security.Permission; import java.security.Policy; import java.security.ProtectionDomain; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; /** Simple tests for SecureSM */ public class SecureSMTests extends TestCase { @@ -127,4 +131,12 @@ public void run() { t1.join(); assertTrue(interrupted1.get()); } + + public void testParallelStreamThreadGroup() throws Exception { + List list = new ArrayList<>(); + for (int i = 0; i < 100; ++i) { + list.add(i); + } + list.parallelStream().collect(Collectors.toSet()); + } } diff --git a/libs/secure-sm/src/test/java/org/elasticsearch/secure_sm/ThreadPermissionTests.java b/libs/secure-sm/src/test/java/org/elasticsearch/secure_sm/ThreadPermissionTests.java index 733d6bcd8ca64..3a398e324fcc6 100644 --- a/libs/secure-sm/src/test/java/org/elasticsearch/secure_sm/ThreadPermissionTests.java +++ b/libs/secure-sm/src/test/java/org/elasticsearch/secure_sm/ThreadPermissionTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.secure_sm; diff --git a/libs/simdvec/build.gradle b/libs/simdvec/build.gradle index be9826a89be32..ffc50ecb1f6ff 100644 --- a/libs/simdvec/build.gradle +++ b/libs/simdvec/build.gradle @@ -1,11 +1,13 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ +import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.internal.precommit.CheckForbiddenApisTask apply plugin: 'elasticsearch.publish' @@ -13,12 +15,26 @@ apply plugin: 'elasticsearch.build' apply plugin: 'elasticsearch.mrjar' dependencies { - implementation project(':libs:elasticsearch-native') - implementation project(':libs:elasticsearch-logging') + implementation project(':libs:native') + implementation project(':libs:logging') implementation "org.apache.lucene:lucene-core:${versions.lucene}" testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-native' + exclude group: 'org.elasticsearch', module: 'native' + } +} + +// compileMain21Java does not exist within idea (see MrJarPlugin) so we cannot reference directly by name +tasks.matching { it.name == "compileMain21Java" }.configureEach { + options.compilerArgs << '--add-modules=jdk.incubator.vector' + // we remove Werror, since incubating suppression (-Xlint:-incubating) + // is only support since JDK 22 + options.compilerArgs -= '-Werror' +} + +tasks.named('test').configure { + if (buildParams.getRuntimeJavaVersion().map{ it.majorVersion.toInteger() }.get() >= 21) { + jvmArgs '--add-modules=jdk.incubator.vector' } } diff --git a/libs/simdvec/includes.txt b/libs/simdvec/includes.txt index 9c3b1a23f9f6b..55bbb74a24922 100644 --- a/libs/simdvec/includes.txt +++ b/libs/simdvec/includes.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # ### Extracted from: vec.h diff --git a/libs/simdvec/native/build.gradle b/libs/simdvec/native/build.gradle index 073477c3aebf2..848d3b325a03f 100644 --- a/libs/simdvec/native/build.gradle +++ b/libs/simdvec/native/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'c' apply plugin: 'cpp' diff --git a/libs/simdvec/native/publish_vec_binaries.sh b/libs/simdvec/native/publish_vec_binaries.sh index ddb3d2c71e448..b581936e18ed0 100755 --- a/libs/simdvec/native/publish_vec_binaries.sh +++ b/libs/simdvec/native/publish_vec_binaries.sh @@ -1,10 +1,11 @@ #!/usr/bin/env bash # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # set -e diff --git a/libs/simdvec/native/settings.gradle b/libs/simdvec/native/settings.gradle index deacf9f685ae0..c15a2b03c6c12 100644 --- a/libs/simdvec/native/settings.gradle +++ b/libs/simdvec/native/settings.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ rootProject.name = 'vec' diff --git a/libs/simdvec/native/src/vec/c/aarch64/vec.c b/libs/simdvec/native/src/vec/c/aarch64/vec.c index 478e5e84d3859..59c0cdb2ff8ff 100644 --- a/libs/simdvec/native/src/vec/c/aarch64/vec.c +++ b/libs/simdvec/native/src/vec/c/aarch64/vec.c @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ #include diff --git a/libs/simdvec/native/src/vec/c/amd64/vec.c b/libs/simdvec/native/src/vec/c/amd64/vec.c index 0fa17109fac6b..50eee12ff69b6 100644 --- a/libs/simdvec/native/src/vec/c/amd64/vec.c +++ b/libs/simdvec/native/src/vec/c/amd64/vec.c @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ #include diff --git a/libs/simdvec/native/src/vec/c/amd64/vec_2.cpp b/libs/simdvec/native/src/vec/c/amd64/vec_2.cpp index 1606b31907405..f851b2a13a9ea 100644 --- a/libs/simdvec/native/src/vec/c/amd64/vec_2.cpp +++ b/libs/simdvec/native/src/vec/c/amd64/vec_2.cpp @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ #include diff --git a/libs/simdvec/native/src/vec/headers/vec.h b/libs/simdvec/native/src/vec/headers/vec.h index 49fa29ec6fae9..e27e9a3a68083 100644 --- a/libs/simdvec/native/src/vec/headers/vec.h +++ b/libs/simdvec/native/src/vec/headers/vec.h @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ #ifdef _MSC_VER diff --git a/libs/simdvec/output.txt b/libs/simdvec/output.txt new file mode 100644 index 0000000000000..acccaea19b765 --- /dev/null +++ b/libs/simdvec/output.txt @@ -0,0 +1,3809 @@ +Using gradle at '/Users/rene/dev/elastic/elasticsearch/gradlew' to run buildfile '/Users/rene/dev/elastic/elasticsearch/libs/simdvec/build.gradle': + +Initialized native services in: /Users/rene/.gradle/native +Initialized jansi services in: /Users/rene/.gradle/native +Found daemon DaemonInfo{pid=51106, address=[aba1969e-035e-4d64-b357-459afffcd02b port:60032, addresses:[/127.0.0.1]], state=Idle, lastBusy=1734939945982, context=DefaultDaemonContext[uid=2770ac86-e90f-46e0-aca4-eb484b46539b,javaHome=/Users/rene/.gradle-idea2/jdks/oracle_corporation-22-aarch64-os_x/jdk-22.jdk/Contents/Home,javaVersion=22,javaVendor=Oracle Corporation,daemonRegistryDir=/Users/rene/.gradle/daemon,pid=51106,idleTimeout=10800000,priority=NORMAL,applyInstrumentationAgent=true,nativeServicesMode=ENABLED,daemonOpts=-XX:+HeapDumpOnOutOfMemoryError,-Xss2m,--add-exports,jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED,--add-exports,jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED,--add-exports,jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED,--add-exports,jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED,--add-exports,jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED,--add-opens,java.base/java.time=ALL-UNNAMED,-Dfile.encoding=UTF-8,-Duser.country=DE,-Duser.language=en,-Duser.variant]} however its context does not match the desired criteria. +JVM is incompatible. +Wanted: DaemonRequestContext{jvmCriteria=/Users/rene/.sdkman/candidates/java/21.0.5-oracle (no JDK specified, using current Java home), daemonOpts=[-XX:+HeapDumpOnOutOfMemoryError, -Xss2m, --add-exports, jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED, --add-exports, jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED, --add-exports, jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED, --add-exports, jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED, --add-exports, jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED, --add-opens, java.base/java.time=ALL-UNNAMED, -Dfile.encoding=UTF-8, -Duser.country=DE, -Duser.language=en, -Duser.variant], applyInstrumentationAgent=true, nativeServicesMode=ENABLED, priority=NORMAL} +Actual: DefaultDaemonContext[uid=2770ac86-e90f-46e0-aca4-eb484b46539b,javaHome=/Users/rene/.gradle-idea2/jdks/oracle_corporation-22-aarch64-os_x/jdk-22.jdk/Contents/Home,javaVersion=22,javaVendor=Oracle Corporation,daemonRegistryDir=/Users/rene/.gradle/daemon,pid=51106,idleTimeout=10800000,priority=NORMAL,applyInstrumentationAgent=true,nativeServicesMode=ENABLED,daemonOpts=-XX:+HeapDumpOnOutOfMemoryError,-Xss2m,--add-exports,jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED,--add-exports,jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED,--add-exports,jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED,--add-exports,jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED,--add-exports,jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED,--add-opens,java.base/java.time=ALL-UNNAMED,-Dfile.encoding=UTF-8,-Duser.country=DE,-Duser.language=en,-Duser.variant] + + Looking for a different daemon... +The client will now receive all logging from the daemon (pid: 49343). The daemon log file: /Users/rene/.gradle/daemon/8.11.1/daemon-49343.out.log +Starting 7th build in daemon [uptime: 28 mins 11.835 secs, performance: 98%, GC rate: 0.00/s, heap usage: 0% of 8 GiB] +Using 12 worker leases. +Now considering [/Users/rene/dev/elastic/elasticsearch, /Users/rene/dev/elastic/elasticsearch/build-tools-internal, /Users/rene/dev/elastic/elasticsearch/build-tools, /Users/rene/dev/elastic/elasticsearch/build-conventions] as hierarchies to watch +Now considering [/Users/rene/dev/elastic/elasticsearch/build-conventions, /Users/rene/dev/elastic/elasticsearch, /Users/rene/dev/elastic/elasticsearch/build-tools-internal, /Users/rene/dev/elastic/elasticsearch/build-tools] as hierarchies to watch +Now considering [/Users/rene/dev/elastic/elasticsearch/build-tools, /Users/rene/dev/elastic/elasticsearch/build-conventions, /Users/rene/dev/elastic/elasticsearch, /Users/rene/dev/elastic/elasticsearch/build-tools-internal] as hierarchies to watch +Now considering [/Users/rene/dev/elastic/elasticsearch/build-tools-internal, /Users/rene/dev/elastic/elasticsearch/build-tools, /Users/rene/dev/elastic/elasticsearch/build-conventions, /Users/rene/dev/elastic/elasticsearch] as hierarchies to watch +Calculating task graph as configuration cache cannot be reused because directory 'libs/simdvec' has changed. +Now considering [/Users/rene/dev/elastic/elasticsearch, /Users/rene/dev/elastic/elasticsearch/build-tools-internal, /Users/rene/dev/elastic/elasticsearch/build-tools, /Users/rene/dev/elastic/elasticsearch/build-conventions] as hierarchies to watch +Watching the file system is configured to be enabled if available +File system watching is active +Starting Build +Now considering [/Users/rene/dev/elastic/elasticsearch/build-conventions, /Users/rene/dev/elastic/elasticsearch, /Users/rene/dev/elastic/elasticsearch/build-tools-internal, /Users/rene/dev/elastic/elasticsearch/build-tools] as hierarchies to watch +Now considering [/Users/rene/dev/elastic/elasticsearch/build-tools, /Users/rene/dev/elastic/elasticsearch/build-conventions, /Users/rene/dev/elastic/elasticsearch, /Users/rene/dev/elastic/elasticsearch/build-tools-internal] as hierarchies to watch +Now considering [/Users/rene/dev/elastic/elasticsearch/build-tools-internal, /Users/rene/dev/elastic/elasticsearch/build-tools, /Users/rene/dev/elastic/elasticsearch/build-conventions, /Users/rene/dev/elastic/elasticsearch] as hierarchies to watch + +> Configure project :build-conventions +Evaluating project ':build-conventions' using build file '/Users/rene/dev/elastic/elasticsearch/build-conventions/build.gradle'. +Registering project ':build-conventions' in composite build. Will substitute for module 'org.elasticsearch:build-conventions'. +Resolve mutations for :build-conventions:compileJava (Thread[#1322,Execution worker,5,main]) started. +:build-conventions:compileJava (Thread[#1329,Execution worker Thread 8,5,main]) started. + +> Task :build-conventions:compileJava UP-TO-DATE +Caching disabled for task ':build-conventions:compileJava' because: + Build cache is disabled +Skipping task ':build-conventions:compileJava' as it is up-to-date. +Resolve mutations for :build-conventions:pluginDescriptors (Thread[#1329,Execution worker Thread 8,5,main]) started. +:build-conventions:pluginDescriptors (Thread[#1329,Execution worker Thread 8,5,main]) started. + +> Task :build-conventions:pluginDescriptors UP-TO-DATE +Caching disabled for task ':build-conventions:pluginDescriptors' because: + Build cache is disabled + Not worth caching +Skipping task ':build-conventions:pluginDescriptors' as it is up-to-date. +Resolve mutations for :build-conventions:processResources (Thread[#1329,Execution worker Thread 8,5,main]) started. +:build-conventions:processResources (Thread[#1329,Execution worker Thread 8,5,main]) started. + +> Task :build-conventions:processResources UP-TO-DATE +Caching disabled for task ':build-conventions:processResources' because: + Build cache is disabled + Not worth caching +Skipping task ':build-conventions:processResources' as it is up-to-date. +Resolve mutations for :build-conventions:classes (Thread[#1329,Execution worker Thread 8,5,main]) started. +:build-conventions:classes (Thread[#1329,Execution worker Thread 8,5,main]) started. + +> Task :build-conventions:classes UP-TO-DATE +Skipping task ':build-conventions:classes' as it has no actions. +Resolve mutations for :build-conventions:jar (Thread[#1329,Execution worker Thread 8,5,main]) started. +:build-conventions:jar (Thread[#1329,Execution worker Thread 8,5,main]) started. + +> Task :build-conventions:jar UP-TO-DATE +Caching disabled for task ':build-conventions:jar' because: + Build cache is disabled + Not worth caching +Skipping task ':build-conventions:jar' as it is up-to-date. + +> Configure project :build-tools +Evaluating project ':build-tools' using build file '/Users/rene/dev/elastic/elasticsearch/build-tools/build.gradle'. +Transforming shadow-gradle-plugin-8.3.5.jar (com.gradleup.shadow:shadow-gradle-plugin:8.3.5) with InstrumentationAnalysisTransform +Transforming log4j-core-2.24.1.jar (org.apache.logging.log4j:log4j-core:2.24.1) with InstrumentationAnalysisTransform +Transforming maven-model-3.6.2.jar (org.apache.maven:maven-model:3.6.2) with InstrumentationAnalysisTransform +Transforming apache-rat-0.11.jar (org.apache.rat:apache-rat:0.11) with InstrumentationAnalysisTransform +Transforming spotless-plugin-gradle-6.25.0.jar (com.diffplug.spotless:spotless-plugin-gradle:6.25.0) with InstrumentationAnalysisTransform +Transforming spotless-lib-extra-2.45.0.jar (com.diffplug.spotless:spotless-lib-extra:2.45.0) with InstrumentationAnalysisTransform +Transforming solstice-1.7.5.jar (dev.equo.ide:solstice:1.7.5) with InstrumentationAnalysisTransform +Transforming org.eclipse.osgi-3.18.300.jar (org.eclipse.platform:org.eclipse.osgi:3.18.300) with InstrumentationAnalysisTransform +Transforming plexus-utils-4.0.2.jar (org.codehaus.plexus:plexus-utils:4.0.2) with InstrumentationAnalysisTransform +Transforming jdom2-2.0.6.1.jar (org.jdom:jdom2:2.0.6.1) with InstrumentationAnalysisTransform +Transforming asm-commons-9.7.1.jar (org.ow2.asm:asm-commons:9.7.1) with InstrumentationAnalysisTransform +Transforming apache-rat-tasks-0.11.jar (org.apache.rat:apache-rat-tasks:0.11) with InstrumentationAnalysisTransform +Transforming apache-rat-core-0.11.jar (org.apache.rat:apache-rat-core:0.11) with InstrumentationAnalysisTransform +Transforming commons-io-2.17.0.jar (commons-io:commons-io:2.17.0) with InstrumentationAnalysisTransform +Transforming ant-1.10.15.jar (org.apache.ant:ant:1.10.15) with InstrumentationAnalysisTransform +Transforming plexus-xml-4.0.4.jar (org.codehaus.plexus:plexus-xml:4.0.4) with InstrumentationAnalysisTransform +Transforming jdependency-2.11.jar (org.vafer:jdependency:2.11) with InstrumentationAnalysisTransform +Transforming durian-io-1.2.0.jar (com.diffplug.durian:durian-io:1.2.0) with InstrumentationAnalysisTransform +Transforming durian-collect-1.2.0.jar (com.diffplug.durian:durian-collect:1.2.0) with InstrumentationAnalysisTransform +Transforming durian-core-1.2.0.jar (com.diffplug.durian:durian-core:1.2.0) with InstrumentationAnalysisTransform +Transforming org.eclipse.jgit-6.7.0.202309050840-r.jar (org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r) with InstrumentationAnalysisTransform +Transforming spotless-lib-2.45.0.jar (com.diffplug.spotless:spotless-lib:2.45.0) with InstrumentationAnalysisTransform +Transforming asm-tree-9.7.1.jar (org.ow2.asm:asm-tree:9.7.1) with InstrumentationAnalysisTransform +Transforming asm-9.7.1.jar (org.ow2.asm:asm:9.7.1) with InstrumentationAnalysisTransform +Transforming ant-launcher-1.10.15.jar (org.apache.ant:ant-launcher:1.10.15) with InstrumentationAnalysisTransform +Transforming maven-xml-impl-4.0.0-alpha-9.jar (org.apache.maven:maven-xml-impl:4.0.0-alpha-9) with InstrumentationAnalysisTransform +Transforming log4j-api-2.24.1.jar (org.apache.logging.log4j:log4j-api:2.24.1) with InstrumentationAnalysisTransform +Transforming commons-collections-3.2.1.jar (commons-collections:commons-collections:3.2.1) with InstrumentationAnalysisTransform +Transforming commons-lang-2.6.jar (commons-lang:commons-lang:2.6) with InstrumentationAnalysisTransform +Transforming commons-compress-1.5.jar (org.apache.commons:commons-compress:1.5) with InstrumentationAnalysisTransform +Transforming commons-cli-1.2.jar (commons-cli:commons-cli:1.2) with InstrumentationAnalysisTransform +Transforming JavaEWAH-1.2.3.jar (com.googlecode.javaewah:JavaEWAH:1.2.3) with InstrumentationAnalysisTransform +Transforming slf4j-api-1.7.36.jar (org.slf4j:slf4j-api:1.7.36) with InstrumentationAnalysisTransform +Transforming commons-codec-1.16.0.jar (commons-codec:commons-codec:1.16.0) with InstrumentationAnalysisTransform +Transforming concurrent-trees-2.6.1.jar (com.googlecode.concurrent-trees:concurrent-trees:2.6.1) with InstrumentationAnalysisTransform +Transforming maven-api-xml-4.0.0-alpha-9.jar (org.apache.maven:maven-api-xml:4.0.0-alpha-9) with InstrumentationAnalysisTransform +Transforming woodstox-core-6.5.1.jar (com.fasterxml.woodstox:woodstox-core:6.5.1) with InstrumentationAnalysisTransform +Transforming xz-1.9.jar (org.tukaani:xz:1.9) with InstrumentationAnalysisTransform +Transforming okhttp-4.12.0.jar (com.squareup.okhttp3:okhttp:4.12.0) with InstrumentationAnalysisTransform +Transforming durian-swt.os-4.2.2.jar (com.diffplug.durian:durian-swt.os:4.2.2) with InstrumentationAnalysisTransform +Transforming maven-api-meta-4.0.0-alpha-9.jar (org.apache.maven:maven-api-meta:4.0.0-alpha-9) with InstrumentationAnalysisTransform +Transforming stax2-api-4.2.1.jar (org.codehaus.woodstox:stax2-api:4.2.1) with InstrumentationAnalysisTransform +Transforming okio-jvm-3.6.0.jar (com.squareup.okio:okio-jvm:3.6.0) with InstrumentationAnalysisTransform +Transforming kotlin-stdlib-jdk8-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10) with InstrumentationAnalysisTransform +Transforming kotlin-stdlib-jdk7-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10) with InstrumentationAnalysisTransform +Transforming kotlin-stdlib-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib:1.9.10) with InstrumentationAnalysisTransform +Transforming kotlin-stdlib-common-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10) with InstrumentationAnalysisTransform +Transforming annotations-13.0.jar (org.jetbrains:annotations:13.0) with InstrumentationAnalysisTransform +Transforming main (project :build-conventions) with InstrumentationAnalysisTransform +Transforming shadow-gradle-plugin-8.3.5.jar (com.gradleup.shadow:shadow-gradle-plugin:8.3.5) with InstrumentationAnalysisTransform +Transforming shadow-gradle-plugin-8.3.5.jar (com.gradleup.shadow:shadow-gradle-plugin:8.3.5) with MergeInstrumentationAnalysisTransform +Transforming log4j-core-2.24.1.jar (org.apache.logging.log4j:log4j-core:2.24.1) with InstrumentationAnalysisTransform +Transforming log4j-core-2.24.1.jar (org.apache.logging.log4j:log4j-core:2.24.1) with MergeInstrumentationAnalysisTransform +Transforming shadow-gradle-plugin-8.3.5.jar (com.gradleup.shadow:shadow-gradle-plugin:8.3.5) with ExternalDependencyInstrumentingArtifactTransform +Transforming maven-model-3.6.2.jar (org.apache.maven:maven-model:3.6.2) with InstrumentationAnalysisTransform +Transforming log4j-core-2.24.1.jar (org.apache.logging.log4j:log4j-core:2.24.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming maven-model-3.6.2.jar (org.apache.maven:maven-model:3.6.2) with MergeInstrumentationAnalysisTransform +Transforming apache-rat-0.11.jar (org.apache.rat:apache-rat:0.11) with InstrumentationAnalysisTransform +Transforming maven-model-3.6.2.jar (org.apache.maven:maven-model:3.6.2) with ExternalDependencyInstrumentingArtifactTransform +Transforming apache-rat-0.11.jar (org.apache.rat:apache-rat:0.11) with MergeInstrumentationAnalysisTransform +Transforming spotless-plugin-gradle-6.25.0.jar (com.diffplug.spotless:spotless-plugin-gradle:6.25.0) with InstrumentationAnalysisTransform +Transforming spotless-plugin-gradle-6.25.0.jar (com.diffplug.spotless:spotless-plugin-gradle:6.25.0) with MergeInstrumentationAnalysisTransform +Transforming apache-rat-0.11.jar (org.apache.rat:apache-rat:0.11) with ExternalDependencyInstrumentingArtifactTransform +Transforming spotless-lib-extra-2.45.0.jar (com.diffplug.spotless:spotless-lib-extra:2.45.0) with InstrumentationAnalysisTransform +Transforming spotless-lib-extra-2.45.0.jar (com.diffplug.spotless:spotless-lib-extra:2.45.0) with MergeInstrumentationAnalysisTransform +Transforming spotless-plugin-gradle-6.25.0.jar (com.diffplug.spotless:spotless-plugin-gradle:6.25.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming solstice-1.7.5.jar (dev.equo.ide:solstice:1.7.5) with InstrumentationAnalysisTransform +Transforming solstice-1.7.5.jar (dev.equo.ide:solstice:1.7.5) with MergeInstrumentationAnalysisTransform +Transforming spotless-lib-extra-2.45.0.jar (com.diffplug.spotless:spotless-lib-extra:2.45.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming org.eclipse.osgi-3.18.300.jar (org.eclipse.platform:org.eclipse.osgi:3.18.300) with InstrumentationAnalysisTransform +Transforming org.eclipse.osgi-3.18.300.jar (org.eclipse.platform:org.eclipse.osgi:3.18.300) with MergeInstrumentationAnalysisTransform +Transforming solstice-1.7.5.jar (dev.equo.ide:solstice:1.7.5) with ExternalDependencyInstrumentingArtifactTransform +Transforming plexus-utils-4.0.2.jar (org.codehaus.plexus:plexus-utils:4.0.2) with InstrumentationAnalysisTransform +Transforming plexus-utils-4.0.2.jar (org.codehaus.plexus:plexus-utils:4.0.2) with MergeInstrumentationAnalysisTransform +Transforming org.eclipse.osgi-3.18.300.jar (org.eclipse.platform:org.eclipse.osgi:3.18.300) with ExternalDependencyInstrumentingArtifactTransform +Transforming jdom2-2.0.6.1.jar (org.jdom:jdom2:2.0.6.1) with InstrumentationAnalysisTransform +Transforming jdom2-2.0.6.1.jar (org.jdom:jdom2:2.0.6.1) with MergeInstrumentationAnalysisTransform +Transforming plexus-utils-4.0.2.jar (org.codehaus.plexus:plexus-utils:4.0.2) with ExternalDependencyInstrumentingArtifactTransform +Transforming asm-commons-9.7.1.jar (org.ow2.asm:asm-commons:9.7.1) with InstrumentationAnalysisTransform +Transforming jdom2-2.0.6.1.jar (org.jdom:jdom2:2.0.6.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming asm-commons-9.7.1.jar (org.ow2.asm:asm-commons:9.7.1) with MergeInstrumentationAnalysisTransform +Transforming apache-rat-tasks-0.11.jar (org.apache.rat:apache-rat-tasks:0.11) with InstrumentationAnalysisTransform +Transforming asm-commons-9.7.1.jar (org.ow2.asm:asm-commons:9.7.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming apache-rat-tasks-0.11.jar (org.apache.rat:apache-rat-tasks:0.11) with MergeInstrumentationAnalysisTransform +Transforming apache-rat-core-0.11.jar (org.apache.rat:apache-rat-core:0.11) with InstrumentationAnalysisTransform +Transforming apache-rat-core-0.11.jar (org.apache.rat:apache-rat-core:0.11) with MergeInstrumentationAnalysisTransform +Transforming apache-rat-tasks-0.11.jar (org.apache.rat:apache-rat-tasks:0.11) with ExternalDependencyInstrumentingArtifactTransform +Transforming commons-io-2.17.0.jar (commons-io:commons-io:2.17.0) with InstrumentationAnalysisTransform +Transforming apache-rat-core-0.11.jar (org.apache.rat:apache-rat-core:0.11) with ExternalDependencyInstrumentingArtifactTransform +Transforming commons-io-2.17.0.jar (commons-io:commons-io:2.17.0) with MergeInstrumentationAnalysisTransform +Transforming ant-1.10.15.jar (org.apache.ant:ant:1.10.15) with InstrumentationAnalysisTransform +Transforming commons-io-2.17.0.jar (commons-io:commons-io:2.17.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming ant-1.10.15.jar (org.apache.ant:ant:1.10.15) with MergeInstrumentationAnalysisTransform +Transforming plexus-xml-4.0.4.jar (org.codehaus.plexus:plexus-xml:4.0.4) with InstrumentationAnalysisTransform +Transforming plexus-xml-4.0.4.jar (org.codehaus.plexus:plexus-xml:4.0.4) with MergeInstrumentationAnalysisTransform +Transforming ant-1.10.15.jar (org.apache.ant:ant:1.10.15) with ExternalDependencyInstrumentingArtifactTransform +Transforming plexus-xml-4.0.4.jar (org.codehaus.plexus:plexus-xml:4.0.4) with ExternalDependencyInstrumentingArtifactTransform +Transforming jdependency-2.11.jar (org.vafer:jdependency:2.11) with InstrumentationAnalysisTransform +Transforming jdependency-2.11.jar (org.vafer:jdependency:2.11) with MergeInstrumentationAnalysisTransform +Transforming durian-io-1.2.0.jar (com.diffplug.durian:durian-io:1.2.0) with InstrumentationAnalysisTransform +Transforming durian-io-1.2.0.jar (com.diffplug.durian:durian-io:1.2.0) with MergeInstrumentationAnalysisTransform +Transforming jdependency-2.11.jar (org.vafer:jdependency:2.11) with ExternalDependencyInstrumentingArtifactTransform +Transforming durian-collect-1.2.0.jar (com.diffplug.durian:durian-collect:1.2.0) with InstrumentationAnalysisTransform +Transforming durian-io-1.2.0.jar (com.diffplug.durian:durian-io:1.2.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming durian-collect-1.2.0.jar (com.diffplug.durian:durian-collect:1.2.0) with MergeInstrumentationAnalysisTransform +Transforming durian-core-1.2.0.jar (com.diffplug.durian:durian-core:1.2.0) with InstrumentationAnalysisTransform +Transforming durian-core-1.2.0.jar (com.diffplug.durian:durian-core:1.2.0) with MergeInstrumentationAnalysisTransform +Transforming durian-collect-1.2.0.jar (com.diffplug.durian:durian-collect:1.2.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming org.eclipse.jgit-6.7.0.202309050840-r.jar (org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r) with InstrumentationAnalysisTransform +Transforming durian-core-1.2.0.jar (com.diffplug.durian:durian-core:1.2.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming org.eclipse.jgit-6.7.0.202309050840-r.jar (org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r) with MergeInstrumentationAnalysisTransform +Transforming spotless-lib-2.45.0.jar (com.diffplug.spotless:spotless-lib:2.45.0) with InstrumentationAnalysisTransform +Transforming spotless-lib-2.45.0.jar (com.diffplug.spotless:spotless-lib:2.45.0) with MergeInstrumentationAnalysisTransform +Transforming org.eclipse.jgit-6.7.0.202309050840-r.jar (org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r) with ExternalDependencyInstrumentingArtifactTransform +Transforming asm-tree-9.7.1.jar (org.ow2.asm:asm-tree:9.7.1) with InstrumentationAnalysisTransform +Transforming asm-tree-9.7.1.jar (org.ow2.asm:asm-tree:9.7.1) with MergeInstrumentationAnalysisTransform +Transforming spotless-lib-2.45.0.jar (com.diffplug.spotless:spotless-lib:2.45.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming asm-9.7.1.jar (org.ow2.asm:asm:9.7.1) with InstrumentationAnalysisTransform +Transforming asm-tree-9.7.1.jar (org.ow2.asm:asm-tree:9.7.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming asm-9.7.1.jar (org.ow2.asm:asm:9.7.1) with MergeInstrumentationAnalysisTransform +Transforming ant-launcher-1.10.15.jar (org.apache.ant:ant-launcher:1.10.15) with InstrumentationAnalysisTransform +Transforming ant-launcher-1.10.15.jar (org.apache.ant:ant-launcher:1.10.15) with MergeInstrumentationAnalysisTransform +Transforming maven-xml-impl-4.0.0-alpha-9.jar (org.apache.maven:maven-xml-impl:4.0.0-alpha-9) with InstrumentationAnalysisTransform +Transforming asm-9.7.1.jar (org.ow2.asm:asm:9.7.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming ant-launcher-1.10.15.jar (org.apache.ant:ant-launcher:1.10.15) with ExternalDependencyInstrumentingArtifactTransform +Transforming maven-xml-impl-4.0.0-alpha-9.jar (org.apache.maven:maven-xml-impl:4.0.0-alpha-9) with MergeInstrumentationAnalysisTransform +Transforming log4j-api-2.24.1.jar (org.apache.logging.log4j:log4j-api:2.24.1) with InstrumentationAnalysisTransform +Transforming maven-xml-impl-4.0.0-alpha-9.jar (org.apache.maven:maven-xml-impl:4.0.0-alpha-9) with ExternalDependencyInstrumentingArtifactTransform +Transforming log4j-api-2.24.1.jar (org.apache.logging.log4j:log4j-api:2.24.1) with MergeInstrumentationAnalysisTransform +Transforming commons-collections-3.2.1.jar (commons-collections:commons-collections:3.2.1) with InstrumentationAnalysisTransform +Transforming commons-collections-3.2.1.jar (commons-collections:commons-collections:3.2.1) with MergeInstrumentationAnalysisTransform +Transforming commons-lang-2.6.jar (commons-lang:commons-lang:2.6) with InstrumentationAnalysisTransform +Transforming log4j-api-2.24.1.jar (org.apache.logging.log4j:log4j-api:2.24.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming commons-collections-3.2.1.jar (commons-collections:commons-collections:3.2.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming commons-lang-2.6.jar (commons-lang:commons-lang:2.6) with MergeInstrumentationAnalysisTransform +Transforming commons-compress-1.5.jar (org.apache.commons:commons-compress:1.5) with InstrumentationAnalysisTransform +Transforming commons-compress-1.5.jar (org.apache.commons:commons-compress:1.5) with MergeInstrumentationAnalysisTransform +Transforming commons-cli-1.2.jar (commons-cli:commons-cli:1.2) with InstrumentationAnalysisTransform +Transforming commons-lang-2.6.jar (commons-lang:commons-lang:2.6) with ExternalDependencyInstrumentingArtifactTransform +Transforming commons-cli-1.2.jar (commons-cli:commons-cli:1.2) with MergeInstrumentationAnalysisTransform +Transforming JavaEWAH-1.2.3.jar (com.googlecode.javaewah:JavaEWAH:1.2.3) with InstrumentationAnalysisTransform +Transforming commons-compress-1.5.jar (org.apache.commons:commons-compress:1.5) with ExternalDependencyInstrumentingArtifactTransform +Transforming JavaEWAH-1.2.3.jar (com.googlecode.javaewah:JavaEWAH:1.2.3) with MergeInstrumentationAnalysisTransform +Transforming commons-cli-1.2.jar (commons-cli:commons-cli:1.2) with ExternalDependencyInstrumentingArtifactTransform +Transforming slf4j-api-1.7.36.jar (org.slf4j:slf4j-api:1.7.36) with InstrumentationAnalysisTransform +Transforming JavaEWAH-1.2.3.jar (com.googlecode.javaewah:JavaEWAH:1.2.3) with ExternalDependencyInstrumentingArtifactTransform +Transforming slf4j-api-1.7.36.jar (org.slf4j:slf4j-api:1.7.36) with MergeInstrumentationAnalysisTransform +Transforming commons-codec-1.16.0.jar (commons-codec:commons-codec:1.16.0) with InstrumentationAnalysisTransform +Transforming slf4j-api-1.7.36.jar (org.slf4j:slf4j-api:1.7.36) with ExternalDependencyInstrumentingArtifactTransform +Transforming commons-codec-1.16.0.jar (commons-codec:commons-codec:1.16.0) with MergeInstrumentationAnalysisTransform +Transforming concurrent-trees-2.6.1.jar (com.googlecode.concurrent-trees:concurrent-trees:2.6.1) with InstrumentationAnalysisTransform +Transforming commons-codec-1.16.0.jar (commons-codec:commons-codec:1.16.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming concurrent-trees-2.6.1.jar (com.googlecode.concurrent-trees:concurrent-trees:2.6.1) with MergeInstrumentationAnalysisTransform +Transforming maven-api-xml-4.0.0-alpha-9.jar (org.apache.maven:maven-api-xml:4.0.0-alpha-9) with InstrumentationAnalysisTransform +Transforming maven-api-xml-4.0.0-alpha-9.jar (org.apache.maven:maven-api-xml:4.0.0-alpha-9) with MergeInstrumentationAnalysisTransform +Transforming concurrent-trees-2.6.1.jar (com.googlecode.concurrent-trees:concurrent-trees:2.6.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming woodstox-core-6.5.1.jar (com.fasterxml.woodstox:woodstox-core:6.5.1) with InstrumentationAnalysisTransform +Transforming maven-api-xml-4.0.0-alpha-9.jar (org.apache.maven:maven-api-xml:4.0.0-alpha-9) with ExternalDependencyInstrumentingArtifactTransform +Transforming woodstox-core-6.5.1.jar (com.fasterxml.woodstox:woodstox-core:6.5.1) with MergeInstrumentationAnalysisTransform +Transforming xz-1.9.jar (org.tukaani:xz:1.9) with InstrumentationAnalysisTransform +Transforming woodstox-core-6.5.1.jar (com.fasterxml.woodstox:woodstox-core:6.5.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming xz-1.9.jar (org.tukaani:xz:1.9) with MergeInstrumentationAnalysisTransform +Transforming okhttp-4.12.0.jar (com.squareup.okhttp3:okhttp:4.12.0) with InstrumentationAnalysisTransform +Transforming okhttp-4.12.0.jar (com.squareup.okhttp3:okhttp:4.12.0) with MergeInstrumentationAnalysisTransform +Transforming xz-1.9.jar (org.tukaani:xz:1.9) with ExternalDependencyInstrumentingArtifactTransform +Transforming durian-swt.os-4.2.2.jar (com.diffplug.durian:durian-swt.os:4.2.2) with InstrumentationAnalysisTransform +Transforming okhttp-4.12.0.jar (com.squareup.okhttp3:okhttp:4.12.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming durian-swt.os-4.2.2.jar (com.diffplug.durian:durian-swt.os:4.2.2) with MergeInstrumentationAnalysisTransform +Transforming maven-api-meta-4.0.0-alpha-9.jar (org.apache.maven:maven-api-meta:4.0.0-alpha-9) with InstrumentationAnalysisTransform +Transforming durian-swt.os-4.2.2.jar (com.diffplug.durian:durian-swt.os:4.2.2) with ExternalDependencyInstrumentingArtifactTransform +Transforming maven-api-meta-4.0.0-alpha-9.jar (org.apache.maven:maven-api-meta:4.0.0-alpha-9) with MergeInstrumentationAnalysisTransform +Transforming stax2-api-4.2.1.jar (org.codehaus.woodstox:stax2-api:4.2.1) with InstrumentationAnalysisTransform +Transforming stax2-api-4.2.1.jar (org.codehaus.woodstox:stax2-api:4.2.1) with MergeInstrumentationAnalysisTransform +Transforming maven-api-meta-4.0.0-alpha-9.jar (org.apache.maven:maven-api-meta:4.0.0-alpha-9) with ExternalDependencyInstrumentingArtifactTransform +Transforming okio-jvm-3.6.0.jar (com.squareup.okio:okio-jvm:3.6.0) with InstrumentationAnalysisTransform +Transforming stax2-api-4.2.1.jar (org.codehaus.woodstox:stax2-api:4.2.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming okio-jvm-3.6.0.jar (com.squareup.okio:okio-jvm:3.6.0) with MergeInstrumentationAnalysisTransform +Transforming kotlin-stdlib-jdk8-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10) with InstrumentationAnalysisTransform +Transforming kotlin-stdlib-jdk8-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10) with MergeInstrumentationAnalysisTransform +Transforming kotlin-stdlib-jdk7-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10) with InstrumentationAnalysisTransform +Transforming okio-jvm-3.6.0.jar (com.squareup.okio:okio-jvm:3.6.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming kotlin-stdlib-jdk7-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10) with MergeInstrumentationAnalysisTransform +Transforming kotlin-stdlib-jdk8-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10) with ExternalDependencyInstrumentingArtifactTransform +Transforming kotlin-stdlib-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib:1.9.10) with InstrumentationAnalysisTransform +Transforming kotlin-stdlib-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib:1.9.10) with MergeInstrumentationAnalysisTransform +Transforming kotlin-stdlib-common-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10) with InstrumentationAnalysisTransform +Transforming kotlin-stdlib-jdk7-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10) with ExternalDependencyInstrumentingArtifactTransform +Transforming kotlin-stdlib-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib:1.9.10) with ExternalDependencyInstrumentingArtifactTransform +Transforming kotlin-stdlib-common-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10) with MergeInstrumentationAnalysisTransform +Transforming annotations-13.0.jar (org.jetbrains:annotations:13.0) with InstrumentationAnalysisTransform +Transforming annotations-13.0.jar (org.jetbrains:annotations:13.0) with MergeInstrumentationAnalysisTransform +Transforming kotlin-stdlib-common-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10) with ExternalDependencyInstrumentingArtifactTransform +Transforming annotations-13.0.jar (org.jetbrains:annotations:13.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming build-conventions.jar (project :build-conventions) with ProjectDependencyInstrumentingArtifactTransform + +> Configure project :build-tools:reaper +Evaluating project ':build-tools:reaper' using build file '/Users/rene/dev/elastic/elasticsearch/build-tools/reaper/build.gradle'. +Registering project ':build-tools' in composite build. Will substitute for module 'org.elasticsearch.gradle:build-tools'. +Registering project ':build-tools:reaper' in composite build. Will substitute for module 'org.elasticsearch.gradle:reaper'. + +> Configure project :build-tools-internal +Evaluating project ':build-tools-internal' using build file '/Users/rene/dev/elastic/elasticsearch/build-tools-internal/build.gradle'. +Transforming shadow-gradle-plugin-8.3.5.jar (com.gradleup.shadow:shadow-gradle-plugin:8.3.5) with InstrumentationAnalysisTransform +Transforming log4j-core-2.24.1.jar (org.apache.logging.log4j:log4j-core:2.24.1) with InstrumentationAnalysisTransform +Transforming maven-model-3.6.2.jar (org.apache.maven:maven-model:3.6.2) with InstrumentationAnalysisTransform +Transforming apache-rat-0.11.jar (org.apache.rat:apache-rat:0.11) with InstrumentationAnalysisTransform +Transforming spotless-plugin-gradle-6.25.0.jar (com.diffplug.spotless:spotless-plugin-gradle:6.25.0) with InstrumentationAnalysisTransform +Transforming spotless-lib-extra-2.45.0.jar (com.diffplug.spotless:spotless-lib-extra:2.45.0) with InstrumentationAnalysisTransform +Transforming solstice-1.7.5.jar (dev.equo.ide:solstice:1.7.5) with InstrumentationAnalysisTransform +Transforming org.eclipse.osgi-3.18.300.jar (org.eclipse.platform:org.eclipse.osgi:3.18.300) with InstrumentationAnalysisTransform +Transforming plexus-utils-4.0.2.jar (org.codehaus.plexus:plexus-utils:4.0.2) with InstrumentationAnalysisTransform +Transforming jdom2-2.0.6.1.jar (org.jdom:jdom2:2.0.6.1) with InstrumentationAnalysisTransform +Transforming asm-commons-9.7.1.jar (org.ow2.asm:asm-commons:9.7.1) with InstrumentationAnalysisTransform +Transforming apache-rat-tasks-0.11.jar (org.apache.rat:apache-rat-tasks:0.11) with InstrumentationAnalysisTransform +Transforming apache-rat-core-0.11.jar (org.apache.rat:apache-rat-core:0.11) with InstrumentationAnalysisTransform +Transforming commons-io-2.17.0.jar (commons-io:commons-io:2.17.0) with InstrumentationAnalysisTransform +Transforming ant-1.10.15.jar (org.apache.ant:ant:1.10.15) with InstrumentationAnalysisTransform +Transforming plexus-xml-4.0.4.jar (org.codehaus.plexus:plexus-xml:4.0.4) with InstrumentationAnalysisTransform +Transforming jdependency-2.11.jar (org.vafer:jdependency:2.11) with InstrumentationAnalysisTransform +Transforming durian-io-1.2.0.jar (com.diffplug.durian:durian-io:1.2.0) with InstrumentationAnalysisTransform +Transforming durian-collect-1.2.0.jar (com.diffplug.durian:durian-collect:1.2.0) with InstrumentationAnalysisTransform +Transforming durian-core-1.2.0.jar (com.diffplug.durian:durian-core:1.2.0) with InstrumentationAnalysisTransform +Transforming org.eclipse.jgit-6.7.0.202309050840-r.jar (org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r) with InstrumentationAnalysisTransform +Transforming spotless-lib-2.45.0.jar (com.diffplug.spotless:spotless-lib:2.45.0) with InstrumentationAnalysisTransform +Transforming asm-tree-9.7.1.jar (org.ow2.asm:asm-tree:9.7.1) with InstrumentationAnalysisTransform +Transforming asm-9.7.1.jar (org.ow2.asm:asm:9.7.1) with InstrumentationAnalysisTransform +Transforming ant-launcher-1.10.15.jar (org.apache.ant:ant-launcher:1.10.15) with InstrumentationAnalysisTransform +Transforming maven-xml-impl-4.0.0-alpha-9.jar (org.apache.maven:maven-xml-impl:4.0.0-alpha-9) with InstrumentationAnalysisTransform +Transforming log4j-api-2.24.1.jar (org.apache.logging.log4j:log4j-api:2.24.1) with InstrumentationAnalysisTransform +Transforming commons-collections-3.2.1.jar (commons-collections:commons-collections:3.2.1) with InstrumentationAnalysisTransform +Transforming commons-lang-2.6.jar (commons-lang:commons-lang:2.6) with InstrumentationAnalysisTransform +Transforming commons-compress-1.5.jar (org.apache.commons:commons-compress:1.5) with InstrumentationAnalysisTransform +Transforming commons-cli-1.2.jar (commons-cli:commons-cli:1.2) with InstrumentationAnalysisTransform +Transforming JavaEWAH-1.2.3.jar (com.googlecode.javaewah:JavaEWAH:1.2.3) with InstrumentationAnalysisTransform +Transforming slf4j-api-1.7.36.jar (org.slf4j:slf4j-api:1.7.36) with InstrumentationAnalysisTransform +Transforming commons-codec-1.16.0.jar (commons-codec:commons-codec:1.16.0) with InstrumentationAnalysisTransform +Transforming concurrent-trees-2.6.1.jar (com.googlecode.concurrent-trees:concurrent-trees:2.6.1) with InstrumentationAnalysisTransform +Transforming maven-api-xml-4.0.0-alpha-9.jar (org.apache.maven:maven-api-xml:4.0.0-alpha-9) with InstrumentationAnalysisTransform +Transforming woodstox-core-6.5.1.jar (com.fasterxml.woodstox:woodstox-core:6.5.1) with InstrumentationAnalysisTransform +Transforming xz-1.9.jar (org.tukaani:xz:1.9) with InstrumentationAnalysisTransform +Transforming okhttp-4.12.0.jar (com.squareup.okhttp3:okhttp:4.12.0) with InstrumentationAnalysisTransform +Transforming durian-swt.os-4.2.2.jar (com.diffplug.durian:durian-swt.os:4.2.2) with InstrumentationAnalysisTransform +Transforming maven-api-meta-4.0.0-alpha-9.jar (org.apache.maven:maven-api-meta:4.0.0-alpha-9) with InstrumentationAnalysisTransform +Transforming stax2-api-4.2.1.jar (org.codehaus.woodstox:stax2-api:4.2.1) with InstrumentationAnalysisTransform +Transforming okio-jvm-3.6.0.jar (com.squareup.okio:okio-jvm:3.6.0) with InstrumentationAnalysisTransform +Transforming kotlin-stdlib-jdk8-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10) with InstrumentationAnalysisTransform +Transforming kotlin-stdlib-jdk7-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10) with InstrumentationAnalysisTransform +Transforming kotlin-stdlib-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib:1.9.10) with InstrumentationAnalysisTransform +Transforming kotlin-stdlib-common-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10) with InstrumentationAnalysisTransform +Transforming annotations-13.0.jar (org.jetbrains:annotations:13.0) with InstrumentationAnalysisTransform +Transforming main (project :build-conventions) with InstrumentationAnalysisTransform +Transforming shadow-gradle-plugin-8.3.5.jar (com.gradleup.shadow:shadow-gradle-plugin:8.3.5) with InstrumentationAnalysisTransform +Transforming shadow-gradle-plugin-8.3.5.jar (com.gradleup.shadow:shadow-gradle-plugin:8.3.5) with MergeInstrumentationAnalysisTransform +Transforming log4j-core-2.24.1.jar (org.apache.logging.log4j:log4j-core:2.24.1) with InstrumentationAnalysisTransform +Transforming log4j-core-2.24.1.jar (org.apache.logging.log4j:log4j-core:2.24.1) with MergeInstrumentationAnalysisTransform +Transforming shadow-gradle-plugin-8.3.5.jar (com.gradleup.shadow:shadow-gradle-plugin:8.3.5) with ExternalDependencyInstrumentingArtifactTransform +Transforming maven-model-3.6.2.jar (org.apache.maven:maven-model:3.6.2) with InstrumentationAnalysisTransform +Transforming maven-model-3.6.2.jar (org.apache.maven:maven-model:3.6.2) with MergeInstrumentationAnalysisTransform +Transforming log4j-core-2.24.1.jar (org.apache.logging.log4j:log4j-core:2.24.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming apache-rat-0.11.jar (org.apache.rat:apache-rat:0.11) with InstrumentationAnalysisTransform +Transforming apache-rat-0.11.jar (org.apache.rat:apache-rat:0.11) with MergeInstrumentationAnalysisTransform +Transforming maven-model-3.6.2.jar (org.apache.maven:maven-model:3.6.2) with ExternalDependencyInstrumentingArtifactTransform +Transforming spotless-plugin-gradle-6.25.0.jar (com.diffplug.spotless:spotless-plugin-gradle:6.25.0) with InstrumentationAnalysisTransform +Transforming spotless-plugin-gradle-6.25.0.jar (com.diffplug.spotless:spotless-plugin-gradle:6.25.0) with MergeInstrumentationAnalysisTransform +Transforming apache-rat-0.11.jar (org.apache.rat:apache-rat:0.11) with ExternalDependencyInstrumentingArtifactTransform +Transforming spotless-lib-extra-2.45.0.jar (com.diffplug.spotless:spotless-lib-extra:2.45.0) with InstrumentationAnalysisTransform +Transforming spotless-lib-extra-2.45.0.jar (com.diffplug.spotless:spotless-lib-extra:2.45.0) with MergeInstrumentationAnalysisTransform +Transforming spotless-plugin-gradle-6.25.0.jar (com.diffplug.spotless:spotless-plugin-gradle:6.25.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming solstice-1.7.5.jar (dev.equo.ide:solstice:1.7.5) with InstrumentationAnalysisTransform +Transforming solstice-1.7.5.jar (dev.equo.ide:solstice:1.7.5) with MergeInstrumentationAnalysisTransform +Transforming spotless-lib-extra-2.45.0.jar (com.diffplug.spotless:spotless-lib-extra:2.45.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming org.eclipse.osgi-3.18.300.jar (org.eclipse.platform:org.eclipse.osgi:3.18.300) with InstrumentationAnalysisTransform +Transforming org.eclipse.osgi-3.18.300.jar (org.eclipse.platform:org.eclipse.osgi:3.18.300) with MergeInstrumentationAnalysisTransform +Transforming solstice-1.7.5.jar (dev.equo.ide:solstice:1.7.5) with ExternalDependencyInstrumentingArtifactTransform +Transforming plexus-utils-4.0.2.jar (org.codehaus.plexus:plexus-utils:4.0.2) with InstrumentationAnalysisTransform +Transforming plexus-utils-4.0.2.jar (org.codehaus.plexus:plexus-utils:4.0.2) with MergeInstrumentationAnalysisTransform +Transforming org.eclipse.osgi-3.18.300.jar (org.eclipse.platform:org.eclipse.osgi:3.18.300) with ExternalDependencyInstrumentingArtifactTransform +Transforming jdom2-2.0.6.1.jar (org.jdom:jdom2:2.0.6.1) with InstrumentationAnalysisTransform +Transforming jdom2-2.0.6.1.jar (org.jdom:jdom2:2.0.6.1) with MergeInstrumentationAnalysisTransform +Transforming plexus-utils-4.0.2.jar (org.codehaus.plexus:plexus-utils:4.0.2) with ExternalDependencyInstrumentingArtifactTransform +Transforming asm-commons-9.7.1.jar (org.ow2.asm:asm-commons:9.7.1) with InstrumentationAnalysisTransform +Transforming asm-commons-9.7.1.jar (org.ow2.asm:asm-commons:9.7.1) with MergeInstrumentationAnalysisTransform +Transforming jdom2-2.0.6.1.jar (org.jdom:jdom2:2.0.6.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming apache-rat-tasks-0.11.jar (org.apache.rat:apache-rat-tasks:0.11) with InstrumentationAnalysisTransform +Transforming apache-rat-tasks-0.11.jar (org.apache.rat:apache-rat-tasks:0.11) with MergeInstrumentationAnalysisTransform +Transforming apache-rat-core-0.11.jar (org.apache.rat:apache-rat-core:0.11) with InstrumentationAnalysisTransform +Transforming apache-rat-core-0.11.jar (org.apache.rat:apache-rat-core:0.11) with MergeInstrumentationAnalysisTransform +Transforming commons-io-2.17.0.jar (commons-io:commons-io:2.17.0) with InstrumentationAnalysisTransform +Transforming asm-commons-9.7.1.jar (org.ow2.asm:asm-commons:9.7.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming apache-rat-tasks-0.11.jar (org.apache.rat:apache-rat-tasks:0.11) with ExternalDependencyInstrumentingArtifactTransform +Transforming commons-io-2.17.0.jar (commons-io:commons-io:2.17.0) with MergeInstrumentationAnalysisTransform +Transforming ant-1.10.15.jar (org.apache.ant:ant:1.10.15) with InstrumentationAnalysisTransform +Transforming ant-1.10.15.jar (org.apache.ant:ant:1.10.15) with MergeInstrumentationAnalysisTransform +Transforming apache-rat-core-0.11.jar (org.apache.rat:apache-rat-core:0.11) with ExternalDependencyInstrumentingArtifactTransform +Transforming commons-io-2.17.0.jar (commons-io:commons-io:2.17.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming plexus-xml-4.0.4.jar (org.codehaus.plexus:plexus-xml:4.0.4) with InstrumentationAnalysisTransform +Transforming plexus-xml-4.0.4.jar (org.codehaus.plexus:plexus-xml:4.0.4) with MergeInstrumentationAnalysisTransform +Transforming jdependency-2.11.jar (org.vafer:jdependency:2.11) with InstrumentationAnalysisTransform +Transforming ant-1.10.15.jar (org.apache.ant:ant:1.10.15) with ExternalDependencyInstrumentingArtifactTransform +Transforming jdependency-2.11.jar (org.vafer:jdependency:2.11) with MergeInstrumentationAnalysisTransform +Transforming plexus-xml-4.0.4.jar (org.codehaus.plexus:plexus-xml:4.0.4) with ExternalDependencyInstrumentingArtifactTransform +Transforming durian-io-1.2.0.jar (com.diffplug.durian:durian-io:1.2.0) with InstrumentationAnalysisTransform +Transforming durian-io-1.2.0.jar (com.diffplug.durian:durian-io:1.2.0) with MergeInstrumentationAnalysisTransform +Transforming jdependency-2.11.jar (org.vafer:jdependency:2.11) with ExternalDependencyInstrumentingArtifactTransform +Transforming durian-collect-1.2.0.jar (com.diffplug.durian:durian-collect:1.2.0) with InstrumentationAnalysisTransform +Transforming durian-io-1.2.0.jar (com.diffplug.durian:durian-io:1.2.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming durian-collect-1.2.0.jar (com.diffplug.durian:durian-collect:1.2.0) with MergeInstrumentationAnalysisTransform +Transforming durian-core-1.2.0.jar (com.diffplug.durian:durian-core:1.2.0) with InstrumentationAnalysisTransform +Transforming durian-core-1.2.0.jar (com.diffplug.durian:durian-core:1.2.0) with MergeInstrumentationAnalysisTransform +Transforming durian-collect-1.2.0.jar (com.diffplug.durian:durian-collect:1.2.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming org.eclipse.jgit-6.7.0.202309050840-r.jar (org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r) with InstrumentationAnalysisTransform +Transforming durian-core-1.2.0.jar (com.diffplug.durian:durian-core:1.2.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming org.eclipse.jgit-6.7.0.202309050840-r.jar (org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r) with MergeInstrumentationAnalysisTransform +Transforming spotless-lib-2.45.0.jar (com.diffplug.spotless:spotless-lib:2.45.0) with InstrumentationAnalysisTransform +Transforming spotless-lib-2.45.0.jar (com.diffplug.spotless:spotless-lib:2.45.0) with MergeInstrumentationAnalysisTransform +Transforming org.eclipse.jgit-6.7.0.202309050840-r.jar (org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r) with ExternalDependencyInstrumentingArtifactTransform +Transforming asm-tree-9.7.1.jar (org.ow2.asm:asm-tree:9.7.1) with InstrumentationAnalysisTransform +Transforming spotless-lib-2.45.0.jar (com.diffplug.spotless:spotless-lib:2.45.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming asm-tree-9.7.1.jar (org.ow2.asm:asm-tree:9.7.1) with MergeInstrumentationAnalysisTransform +Transforming asm-9.7.1.jar (org.ow2.asm:asm:9.7.1) with InstrumentationAnalysisTransform +Transforming asm-tree-9.7.1.jar (org.ow2.asm:asm-tree:9.7.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming asm-9.7.1.jar (org.ow2.asm:asm:9.7.1) with MergeInstrumentationAnalysisTransform +Transforming ant-launcher-1.10.15.jar (org.apache.ant:ant-launcher:1.10.15) with InstrumentationAnalysisTransform +Transforming ant-launcher-1.10.15.jar (org.apache.ant:ant-launcher:1.10.15) with MergeInstrumentationAnalysisTransform +Transforming maven-xml-impl-4.0.0-alpha-9.jar (org.apache.maven:maven-xml-impl:4.0.0-alpha-9) with InstrumentationAnalysisTransform +Transforming asm-9.7.1.jar (org.ow2.asm:asm:9.7.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming maven-xml-impl-4.0.0-alpha-9.jar (org.apache.maven:maven-xml-impl:4.0.0-alpha-9) with MergeInstrumentationAnalysisTransform +Transforming ant-launcher-1.10.15.jar (org.apache.ant:ant-launcher:1.10.15) with ExternalDependencyInstrumentingArtifactTransform +Transforming log4j-api-2.24.1.jar (org.apache.logging.log4j:log4j-api:2.24.1) with InstrumentationAnalysisTransform +Transforming maven-xml-impl-4.0.0-alpha-9.jar (org.apache.maven:maven-xml-impl:4.0.0-alpha-9) with ExternalDependencyInstrumentingArtifactTransform +Transforming log4j-api-2.24.1.jar (org.apache.logging.log4j:log4j-api:2.24.1) with MergeInstrumentationAnalysisTransform +Transforming commons-collections-3.2.1.jar (commons-collections:commons-collections:3.2.1) with InstrumentationAnalysisTransform +Transforming commons-collections-3.2.1.jar (commons-collections:commons-collections:3.2.1) with MergeInstrumentationAnalysisTransform +Transforming log4j-api-2.24.1.jar (org.apache.logging.log4j:log4j-api:2.24.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming commons-lang-2.6.jar (commons-lang:commons-lang:2.6) with InstrumentationAnalysisTransform +Transforming commons-lang-2.6.jar (commons-lang:commons-lang:2.6) with MergeInstrumentationAnalysisTransform +Transforming commons-collections-3.2.1.jar (commons-collections:commons-collections:3.2.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming commons-compress-1.5.jar (org.apache.commons:commons-compress:1.5) with InstrumentationAnalysisTransform +Transforming commons-compress-1.5.jar (org.apache.commons:commons-compress:1.5) with MergeInstrumentationAnalysisTransform +Transforming commons-lang-2.6.jar (commons-lang:commons-lang:2.6) with ExternalDependencyInstrumentingArtifactTransform +Transforming commons-cli-1.2.jar (commons-cli:commons-cli:1.2) with InstrumentationAnalysisTransform +Transforming commons-compress-1.5.jar (org.apache.commons:commons-compress:1.5) with ExternalDependencyInstrumentingArtifactTransform +Transforming commons-cli-1.2.jar (commons-cli:commons-cli:1.2) with MergeInstrumentationAnalysisTransform +Transforming JavaEWAH-1.2.3.jar (com.googlecode.javaewah:JavaEWAH:1.2.3) with InstrumentationAnalysisTransform +Transforming JavaEWAH-1.2.3.jar (com.googlecode.javaewah:JavaEWAH:1.2.3) with MergeInstrumentationAnalysisTransform +Transforming slf4j-api-1.7.36.jar (org.slf4j:slf4j-api:1.7.36) with InstrumentationAnalysisTransform +Transforming commons-cli-1.2.jar (commons-cli:commons-cli:1.2) with ExternalDependencyInstrumentingArtifactTransform +Transforming JavaEWAH-1.2.3.jar (com.googlecode.javaewah:JavaEWAH:1.2.3) with ExternalDependencyInstrumentingArtifactTransform +Transforming slf4j-api-1.7.36.jar (org.slf4j:slf4j-api:1.7.36) with MergeInstrumentationAnalysisTransform +Transforming commons-codec-1.16.0.jar (commons-codec:commons-codec:1.16.0) with InstrumentationAnalysisTransform +Transforming commons-codec-1.16.0.jar (commons-codec:commons-codec:1.16.0) with MergeInstrumentationAnalysisTransform +Transforming concurrent-trees-2.6.1.jar (com.googlecode.concurrent-trees:concurrent-trees:2.6.1) with InstrumentationAnalysisTransform +Transforming concurrent-trees-2.6.1.jar (com.googlecode.concurrent-trees:concurrent-trees:2.6.1) with MergeInstrumentationAnalysisTransform +Transforming slf4j-api-1.7.36.jar (org.slf4j:slf4j-api:1.7.36) with ExternalDependencyInstrumentingArtifactTransform +Transforming commons-codec-1.16.0.jar (commons-codec:commons-codec:1.16.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming maven-api-xml-4.0.0-alpha-9.jar (org.apache.maven:maven-api-xml:4.0.0-alpha-9) with InstrumentationAnalysisTransform +Transforming concurrent-trees-2.6.1.jar (com.googlecode.concurrent-trees:concurrent-trees:2.6.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming maven-api-xml-4.0.0-alpha-9.jar (org.apache.maven:maven-api-xml:4.0.0-alpha-9) with MergeInstrumentationAnalysisTransform +Transforming woodstox-core-6.5.1.jar (com.fasterxml.woodstox:woodstox-core:6.5.1) with InstrumentationAnalysisTransform +Transforming maven-api-xml-4.0.0-alpha-9.jar (org.apache.maven:maven-api-xml:4.0.0-alpha-9) with ExternalDependencyInstrumentingArtifactTransform +Transforming woodstox-core-6.5.1.jar (com.fasterxml.woodstox:woodstox-core:6.5.1) with MergeInstrumentationAnalysisTransform +Transforming xz-1.9.jar (org.tukaani:xz:1.9) with InstrumentationAnalysisTransform +Transforming xz-1.9.jar (org.tukaani:xz:1.9) with MergeInstrumentationAnalysisTransform +Transforming woodstox-core-6.5.1.jar (com.fasterxml.woodstox:woodstox-core:6.5.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming xz-1.9.jar (org.tukaani:xz:1.9) with ExternalDependencyInstrumentingArtifactTransform +Transforming okhttp-4.12.0.jar (com.squareup.okhttp3:okhttp:4.12.0) with InstrumentationAnalysisTransform +Transforming okhttp-4.12.0.jar (com.squareup.okhttp3:okhttp:4.12.0) with MergeInstrumentationAnalysisTransform +Transforming durian-swt.os-4.2.2.jar (com.diffplug.durian:durian-swt.os:4.2.2) with InstrumentationAnalysisTransform +Transforming durian-swt.os-4.2.2.jar (com.diffplug.durian:durian-swt.os:4.2.2) with MergeInstrumentationAnalysisTransform +Transforming okhttp-4.12.0.jar (com.squareup.okhttp3:okhttp:4.12.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming maven-api-meta-4.0.0-alpha-9.jar (org.apache.maven:maven-api-meta:4.0.0-alpha-9) with InstrumentationAnalysisTransform +Transforming durian-swt.os-4.2.2.jar (com.diffplug.durian:durian-swt.os:4.2.2) with ExternalDependencyInstrumentingArtifactTransform +Transforming maven-api-meta-4.0.0-alpha-9.jar (org.apache.maven:maven-api-meta:4.0.0-alpha-9) with MergeInstrumentationAnalysisTransform +Transforming stax2-api-4.2.1.jar (org.codehaus.woodstox:stax2-api:4.2.1) with InstrumentationAnalysisTransform +Transforming stax2-api-4.2.1.jar (org.codehaus.woodstox:stax2-api:4.2.1) with MergeInstrumentationAnalysisTransform +Transforming maven-api-meta-4.0.0-alpha-9.jar (org.apache.maven:maven-api-meta:4.0.0-alpha-9) with ExternalDependencyInstrumentingArtifactTransform +Transforming okio-jvm-3.6.0.jar (com.squareup.okio:okio-jvm:3.6.0) with InstrumentationAnalysisTransform +Transforming stax2-api-4.2.1.jar (org.codehaus.woodstox:stax2-api:4.2.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming okio-jvm-3.6.0.jar (com.squareup.okio:okio-jvm:3.6.0) with MergeInstrumentationAnalysisTransform +Transforming kotlin-stdlib-jdk8-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10) with InstrumentationAnalysisTransform +Transforming okio-jvm-3.6.0.jar (com.squareup.okio:okio-jvm:3.6.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming kotlin-stdlib-jdk8-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10) with MergeInstrumentationAnalysisTransform +Transforming kotlin-stdlib-jdk7-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10) with InstrumentationAnalysisTransform +Transforming kotlin-stdlib-jdk7-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10) with MergeInstrumentationAnalysisTransform +Transforming kotlin-stdlib-jdk8-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10) with ExternalDependencyInstrumentingArtifactTransform +Transforming kotlin-stdlib-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib:1.9.10) with InstrumentationAnalysisTransform +Transforming kotlin-stdlib-jdk7-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10) with ExternalDependencyInstrumentingArtifactTransform +Transforming kotlin-stdlib-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib:1.9.10) with MergeInstrumentationAnalysisTransform +Transforming kotlin-stdlib-common-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10) with InstrumentationAnalysisTransform +Transforming kotlin-stdlib-common-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10) with MergeInstrumentationAnalysisTransform +Transforming kotlin-stdlib-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib:1.9.10) with ExternalDependencyInstrumentingArtifactTransform +Transforming annotations-13.0.jar (org.jetbrains:annotations:13.0) with InstrumentationAnalysisTransform +Transforming annotations-13.0.jar (org.jetbrains:annotations:13.0) with MergeInstrumentationAnalysisTransform +Transforming kotlin-stdlib-common-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10) with ExternalDependencyInstrumentingArtifactTransform +Transforming annotations-13.0.jar (org.jetbrains:annotations:13.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming build-conventions.jar (project :build-conventions) with ProjectDependencyInstrumentingArtifactTransform +Found project 'project :build-conventions' as substitute for module 'org.elasticsearch:build-conventions'. +Found project 'project :build-tools' as substitute for module 'org.elasticsearch.gradle:build-tools'. +Found project 'project :build-tools:reaper' as substitute for module 'org.elasticsearch.gradle:reaper'. +Found project 'project :build-conventions' as substitute for module 'org.elasticsearch:build-conventions'. +Found project 'project :build-tools' as substitute for module 'org.elasticsearch.gradle:build-tools'. +Resolve mutations for :build-tools:compileJava (Thread[#1328,Execution worker Thread 7,5,main]) started. +Resolve mutations for :build-tools:reaper:compileJava (Thread[#1329,Execution worker Thread 8,5,main]) started. +Resolve mutations for :build-tools-internal:extractPluginRequests (Thread[#1331,Execution worker Thread 10,5,main]) started. +work action resolve build-conventions.jar (project :build-conventions) (Thread[#1330,Execution worker Thread 9,5,main]) started. +:build-tools-internal:extractPluginRequests (Thread[#1331,Execution worker Thread 10,5,main]) started. +:build-tools:reaper:compileJava (Thread[#1329,Execution worker Thread 8,5,main]) started. +:build-tools:compileJava (Thread[#1328,Execution worker Thread 7,5,main]) started. + +> Task :build-tools-internal:extractPluginRequests UP-TO-DATE +Caching disabled for task ':build-tools-internal:extractPluginRequests' because: + Build cache is disabled +Skipping task ':build-tools-internal:extractPluginRequests' as it is up-to-date. +Resolve mutations for :build-tools-internal:generatePluginAdapters (Thread[#1331,Execution worker Thread 10,5,main]) started. + +> Task :build-tools:reaper:compileJava UP-TO-DATE +Caching disabled for task ':build-tools:reaper:compileJava' because: + Build cache is disabled +Skipping task ':build-tools:reaper:compileJava' as it is up-to-date. +No compile result for :build-tools:reaper:compileJava +:build-tools-internal:generatePluginAdapters (Thread[#1325,Execution worker Thread 4,5,main]) started. +No compile result for :build-tools:reaper:compileJava +Resolve mutations for :build-tools:reaper:processResources (Thread[#1329,Execution worker Thread 8,5,main]) started. +:build-tools:reaper:processResources (Thread[#1329,Execution worker Thread 8,5,main]) started. + +> Task :build-tools-internal:generatePluginAdapters UP-TO-DATE +Caching disabled for task ':build-tools-internal:generatePluginAdapters' because: + Build cache is disabled +Skipping task ':build-tools-internal:generatePluginAdapters' as it is up-to-date. + +> Task :build-tools:reaper:processResources NO-SOURCE +Skipping task ':build-tools:reaper:processResources' as it has no source files and no previous output files. +Resolve mutations for :build-tools:reaper:classes (Thread[#1329,Execution worker Thread 8,5,main]) started. +:build-tools:reaper:classes (Thread[#1329,Execution worker Thread 8,5,main]) started. + +> Task :build-tools:reaper:classes UP-TO-DATE +Skipping task ':build-tools:reaper:classes' as it has no actions. +Resolve mutations for :build-tools-internal:pluginDescriptors (Thread[#1325,Execution worker Thread 4,5,main]) started. +Resolve mutations for :build-tools:reaper:jar (Thread[#1330,Execution worker Thread 9,5,main]) started. +:build-tools-internal:pluginDescriptors (Thread[#1325,Execution worker Thread 4,5,main]) started. +:build-tools:reaper:jar (Thread[#1330,Execution worker Thread 9,5,main]) started. + +> Task :build-tools-internal:pluginDescriptors UP-TO-DATE +Caching disabled for task ':build-tools-internal:pluginDescriptors' because: + Build cache is disabled + Not worth caching +Skipping task ':build-tools-internal:pluginDescriptors' as it is up-to-date. +Resolve mutations for :build-tools-internal:processResources (Thread[#1325,Execution worker Thread 4,5,main]) started. +:build-tools-internal:processResources (Thread[#1325,Execution worker Thread 4,5,main]) started. + +> Task :build-tools:reaper:jar UP-TO-DATE +Caching disabled for task ':build-tools:reaper:jar' because: + Build cache is disabled + Not worth caching +Skipping task ':build-tools:reaper:jar' as it is up-to-date. +work action resolve reaper.jar (project :build-tools:reaper) (Thread[#1330,Execution worker Thread 9,5,main]) started. + +> Task :build-tools-internal:processResources UP-TO-DATE +Caching disabled for task ':build-tools-internal:processResources' because: + Build cache is disabled + Not worth caching +Skipping task ':build-tools-internal:processResources' as it is up-to-date. + +> Task :build-tools:compileJava UP-TO-DATE +Caching disabled for task ':build-tools:compileJava' because: + Build cache is disabled +Skipping task ':build-tools:compileJava' as it is up-to-date. +No compile result for :build-tools:compileJava +No compile result for :build-tools:compileJava +Resolve mutations for :build-tools:compileGroovy (Thread[#1328,Execution worker Thread 7,5,main]) started. +:build-tools:compileGroovy (Thread[#1328,Execution worker Thread 7,5,main]) started. + +> Task :build-tools:compileGroovy NO-SOURCE +Skipping task ':build-tools:compileGroovy' as it has no source files and no previous output files. +Resolve mutations for :build-tools:generateVersionProperties (Thread[#1328,Execution worker Thread 7,5,main]) started. +:build-tools:generateVersionProperties (Thread[#1328,Execution worker Thread 7,5,main]) started. + +> Task :build-tools:generateVersionProperties UP-TO-DATE +Caching disabled for task ':build-tools:generateVersionProperties' because: + Build cache is disabled +Skipping task ':build-tools:generateVersionProperties' as it is up-to-date. +Resolve mutations for :build-tools:pluginDescriptors (Thread[#1328,Execution worker Thread 7,5,main]) started. +:build-tools:pluginDescriptors (Thread[#1328,Execution worker Thread 7,5,main]) started. + +> Task :build-tools:pluginDescriptors UP-TO-DATE +Caching disabled for task ':build-tools:pluginDescriptors' because: + Build cache is disabled + Not worth caching +Skipping task ':build-tools:pluginDescriptors' as it is up-to-date. +Resolve mutations for :build-tools:processResources (Thread[#1328,Execution worker Thread 7,5,main]) started. +:build-tools:processResources (Thread[#1328,Execution worker Thread 7,5,main]) started. + +> Task :build-tools:processResources UP-TO-DATE +Caching disabled for task ':build-tools:processResources' because: + Build cache is disabled + Not worth caching +Skipping task ':build-tools:processResources' as it is up-to-date. +Resolve mutations for :build-tools:classes (Thread[#1328,Execution worker Thread 7,5,main]) started. +:build-tools:classes (Thread[#1328,Execution worker Thread 7,5,main]) started. + +> Task :build-tools:classes UP-TO-DATE +Skipping task ':build-tools:classes' as it has no actions. +Resolve mutations for :build-tools:jar (Thread[#1328,Execution worker Thread 7,5,main]) started. +:build-tools:jar (Thread[#1328,Execution worker Thread 7,5,main]) started. + +> Task :build-tools:jar UP-TO-DATE +Caching disabled for task ':build-tools:jar' because: + Build cache is disabled + Not worth caching +Skipping task ':build-tools:jar' as it is up-to-date. +other build task :build-tools:jar (Thread[#1328,Execution worker Thread 7,5,main]) started. +work action resolve build-tools-9.0.0-SNAPSHOT.jar (project :build-tools) (Thread[#1328,Execution worker Thread 7,5,main]) started. +Resolve mutations for :build-tools-internal:compileJava (Thread[#1331,Execution worker Thread 10,5,main]) started. +:build-tools-internal:compileJava (Thread[#1331,Execution worker Thread 10,5,main]) started. + +> Task :build-tools-internal:compileJava UP-TO-DATE +Caching disabled for task ':build-tools-internal:compileJava' because: + Build cache is disabled +Skipping task ':build-tools-internal:compileJava' as it is up-to-date. +No compile result for :build-tools-internal:compileJava +No compile result for :build-tools-internal:compileJava +Resolve mutations for :build-tools-internal:compileGroovy (Thread[#1331,Execution worker Thread 10,5,main]) started. +:build-tools-internal:compileGroovy (Thread[#1331,Execution worker Thread 10,5,main]) started. + +> Task :build-tools-internal:compileGroovy UP-TO-DATE +Caching disabled for task ':build-tools-internal:compileGroovy' because: + Build cache is disabled +Skipping task ':build-tools-internal:compileGroovy' as it is up-to-date. +Resolve mutations for :build-tools-internal:compileGroovyPlugins (Thread[#1331,Execution worker Thread 10,5,main]) started. +:build-tools-internal:compileGroovyPlugins (Thread[#1331,Execution worker Thread 10,5,main]) started. + +> Task :build-tools-internal:compileGroovyPlugins UP-TO-DATE +Caching disabled for task ':build-tools-internal:compileGroovyPlugins' because: + Build cache is disabled +Skipping task ':build-tools-internal:compileGroovyPlugins' as it is up-to-date. +Resolve mutations for :build-tools-internal:classes (Thread[#1331,Execution worker Thread 10,5,main]) started. +:build-tools-internal:classes (Thread[#1331,Execution worker Thread 10,5,main]) started. + +> Task :build-tools-internal:classes UP-TO-DATE +Skipping task ':build-tools-internal:classes' as it has no actions. +Resolve mutations for :build-tools-internal:jar (Thread[#1331,Execution worker Thread 10,5,main]) started. +:build-tools-internal:jar (Thread[#1330,Execution worker Thread 9,5,main]) started. + +> Task :build-tools-internal:jar UP-TO-DATE +Caching disabled for task ':build-tools-internal:jar' because: + Build cache is disabled + Not worth caching +Skipping task ':build-tools-internal:jar' as it is up-to-date. +Transforming shadow-gradle-plugin-8.3.5.jar (com.gradleup.shadow:shadow-gradle-plugin:8.3.5) with InstrumentationAnalysisTransform +Transforming log4j-core-2.24.1.jar (org.apache.logging.log4j:log4j-core:2.24.1) with InstrumentationAnalysisTransform +Transforming develocity-gradle-plugin-3.18.1.jar (com.gradle:develocity-gradle-plugin:3.18.1) with InstrumentationAnalysisTransform +Transforming jackson-dataformat-yaml-2.15.0.jar (com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.15.0) with InstrumentationAnalysisTransform +Transforming json-schema-validator-1.0.72.jar (com.networknt:json-schema-validator:1.0.72) with InstrumentationAnalysisTransform +Transforming jackson-databind-2.15.0.jar (com.fasterxml.jackson.core:jackson-databind:2.15.0) with InstrumentationAnalysisTransform +Transforming jackson-core-2.15.0.jar (com.fasterxml.jackson.core:jackson-core:2.15.0) with InstrumentationAnalysisTransform +Transforming jackson-annotations-2.15.0.jar (com.fasterxml.jackson.core:jackson-annotations:2.15.0) with InstrumentationAnalysisTransform +Transforming apache-rat-0.11.jar (org.apache.rat:apache-rat:0.11) with InstrumentationAnalysisTransform +Transforming apache-rat-tasks-0.11.jar (org.apache.rat:apache-rat-tasks:0.11) with InstrumentationAnalysisTransform +Transforming apache-rat-core-0.11.jar (org.apache.rat:apache-rat-core:0.11) with InstrumentationAnalysisTransform +Transforming commons-compress-1.26.1.jar (org.apache.commons:commons-compress:1.26.1) with InstrumentationAnalysisTransform +Transforming httpclient-4.5.14.jar (org.apache.httpcomponents:httpclient:4.5.14) with InstrumentationAnalysisTransform +Transforming gradle-info-plugin-11.3.3.jar (com.netflix.nebula:gradle-info-plugin:11.3.3) with InstrumentationAnalysisTransform +Transforming spotless-plugin-gradle-6.25.0.jar (com.diffplug.spotless:spotless-plugin-gradle:6.25.0) with InstrumentationAnalysisTransform +Transforming spotless-lib-extra-2.45.0.jar (com.diffplug.spotless:spotless-lib-extra:2.45.0) with InstrumentationAnalysisTransform +Transforming org.eclipse.jgit-6.7.0.202309050840-r.jar (org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r) with InstrumentationAnalysisTransform +Transforming commons-codec-1.16.1.jar (commons-codec:commons-codec:1.16.1) with InstrumentationAnalysisTransform +Transforming svnkit-1.8.12.jar (org.tmatesoft.svnkit:svnkit:1.8.12) with InstrumentationAnalysisTransform +Transforming jna-platform-5.7.0.jar (net.java.dev.jna:jna-platform:5.7.0) with InstrumentationAnalysisTransform +Transforming jsch.agentproxy.connector-factory-0.0.7.jar (com.jcraft:jsch.agentproxy.connector-factory:0.0.7) with InstrumentationAnalysisTransform +Transforming jsch.agentproxy.usocket-jna-0.0.7.jar (com.jcraft:jsch.agentproxy.usocket-jna:0.0.7) with InstrumentationAnalysisTransform +Transforming jsch.agentproxy.pageant-0.0.7.jar (com.jcraft:jsch.agentproxy.pageant:0.0.7) with InstrumentationAnalysisTransform +Transforming jna-5.10.0.jar (net.java.dev.jna:jna:5.10.0) with InstrumentationAnalysisTransform +Transforming gradle-idea-ext-1.1.4.jar (gradle.plugin.org.jetbrains.gradle.plugin.idea-ext:gradle-idea-ext:1.1.4) with InstrumentationAnalysisTransform +Transforming forbiddenapis-3.8.jar (de.thetaphi:forbiddenapis:3.8) with InstrumentationAnalysisTransform +Transforming gradle-docker-compose-plugin-0.17.5.jar (com.avast.gradle:gradle-docker-compose-plugin:0.17.5) with InstrumentationAnalysisTransform +Transforming maven-model-3.6.2.jar (org.apache.maven:maven-model:3.6.2) with InstrumentationAnalysisTransform +Transforming ST4-4.3.4.jar (org.antlr:ST4:4.3.4) with InstrumentationAnalysisTransform +Transforming asm-commons-9.7.1.jar (org.ow2.asm:asm-commons:9.7.1) with InstrumentationAnalysisTransform +Transforming asm-tree-9.7.1.jar (org.ow2.asm:asm-tree:9.7.1) with InstrumentationAnalysisTransform +Transforming asm-9.7.1.jar (org.ow2.asm:asm:9.7.1) with InstrumentationAnalysisTransform +Transforming httpcore-4.4.16.jar (org.apache.httpcomponents:httpcore:4.4.16) with InstrumentationAnalysisTransform +Transforming snakeyaml-2.0.jar (org.yaml:snakeyaml:2.0) with InstrumentationAnalysisTransform +Transforming javaparser-core-3.18.0.jar (com.github.javaparser:javaparser-core:3.18.0) with InstrumentationAnalysisTransform +Transforming solstice-1.7.5.jar (dev.equo.ide:solstice:1.7.5) with InstrumentationAnalysisTransform +Transforming org.eclipse.osgi-3.18.300.jar (org.eclipse.platform:org.eclipse.osgi:3.18.300) with InstrumentationAnalysisTransform +Transforming ant-1.10.15.jar (org.apache.ant:ant:1.10.15) with InstrumentationAnalysisTransform +Transforming commons-io-2.17.0.jar (commons-io:commons-io:2.17.0) with InstrumentationAnalysisTransform +Transforming commons-lang3-3.14.0.jar (org.apache.commons:commons-lang3:3.14.0) with InstrumentationAnalysisTransform +Transforming p4java-2015.2.1365273.jar (com.perforce:p4java:2015.2.1365273) with InstrumentationAnalysisTransform +Transforming nebula-gradle-interop-2.0.0.jar (com.netflix.nebula:nebula-gradle-interop:2.0.0) with InstrumentationAnalysisTransform +Transforming gradle-contacts-plugin-6.0.0.jar (com.netflix.nebula:gradle-contacts-plugin:6.0.0) with InstrumentationAnalysisTransform +Transforming jdom2-2.0.6.1.jar (org.jdom:jdom2:2.0.6.1) with InstrumentationAnalysisTransform +Transforming plexus-utils-4.0.2.jar (org.codehaus.plexus:plexus-utils:4.0.2) with InstrumentationAnalysisTransform +Transforming plexus-xml-4.0.4.jar (org.codehaus.plexus:plexus-xml:4.0.4) with InstrumentationAnalysisTransform +Transforming jdependency-2.11.jar (org.vafer:jdependency:2.11) with InstrumentationAnalysisTransform +Transforming gson-2.8.6.jar (com.google.code.gson:gson:2.8.6) with InstrumentationAnalysisTransform +Transforming guava-28.2-jre.jar (com.google.guava:guava:28.2-jre) with InstrumentationAnalysisTransform +Transforming slf4j-api-1.7.36.jar (org.slf4j:slf4j-api:1.7.36) with InstrumentationAnalysisTransform +Transforming itu-1.7.0.jar (com.ethlo.time:itu:1.7.0) with InstrumentationAnalysisTransform +Transforming sqljet-1.1.10.jar (org.tmatesoft.sqljet:sqljet:1.1.10) with InstrumentationAnalysisTransform +Transforming antlr-runtime-3.5.3.jar (org.antlr:antlr-runtime:3.5.3) with InstrumentationAnalysisTransform +Transforming commons-logging-1.2.jar (commons-logging:commons-logging:1.2) with InstrumentationAnalysisTransform +Transforming durian-io-1.2.0.jar (com.diffplug.durian:durian-io:1.2.0) with InstrumentationAnalysisTransform +Transforming durian-collect-1.2.0.jar (com.diffplug.durian:durian-collect:1.2.0) with InstrumentationAnalysisTransform +Transforming durian-core-1.2.0.jar (com.diffplug.durian:durian-core:1.2.0) with InstrumentationAnalysisTransform +Transforming spotless-lib-2.45.0.jar (com.diffplug.spotless:spotless-lib:2.45.0) with InstrumentationAnalysisTransform +Transforming fastdoubleparser-0.8.0.jar (ch.randelshofer:fastdoubleparser:0.8.0) with InstrumentationAnalysisTransform +Transforming jzlib-1.1.2.jar (com.jcraft:jzlib:1.1.2) with InstrumentationAnalysisTransform +Transforming okhttp-4.12.0.jar (com.squareup.okhttp3:okhttp:4.12.0) with InstrumentationAnalysisTransform +Transforming okio-jvm-3.6.0.jar (com.squareup.okio:okio-jvm:3.6.0) with InstrumentationAnalysisTransform +Transforming kotlin-stdlib-jdk8-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10) with InstrumentationAnalysisTransform +Transforming jsch.agentproxy.svnkit-trilead-ssh2-0.0.7.jar (com.jcraft:jsch.agentproxy.svnkit-trilead-ssh2:0.0.7) with InstrumentationAnalysisTransform +Transforming trilead-ssh2-1.0.0-build220.jar (com.trilead:trilead-ssh2:1.0.0-build220) with InstrumentationAnalysisTransform +Transforming sequence-library-1.0.3.jar (de.regnis.q.sequence:sequence-library:1.0.3) with InstrumentationAnalysisTransform +Transforming commons-collections-3.2.1.jar (commons-collections:commons-collections:3.2.1) with InstrumentationAnalysisTransform +Transforming commons-lang-2.6.jar (commons-lang:commons-lang:2.6) with InstrumentationAnalysisTransform +Transforming commons-cli-1.2.jar (commons-cli:commons-cli:1.2) with InstrumentationAnalysisTransform +Transforming ant-launcher-1.10.15.jar (org.apache.ant:ant-launcher:1.10.15) with InstrumentationAnalysisTransform +Transforming maven-xml-impl-4.0.0-alpha-9.jar (org.apache.maven:maven-xml-impl:4.0.0-alpha-9) with InstrumentationAnalysisTransform +Transforming log4j-api-2.24.1.jar (org.apache.logging.log4j:log4j-api:2.24.1) with InstrumentationAnalysisTransform +Transforming failureaccess-1.0.1.jar (com.google.guava:failureaccess:1.0.1) with InstrumentationAnalysisTransform +Transforming listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar (com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava) with InstrumentationAnalysisTransform +Transforming jsr305-3.0.2.jar (com.google.code.findbugs:jsr305:3.0.2) with InstrumentationAnalysisTransform +Transforming checker-qual-2.10.0.jar (org.checkerframework:checker-qual:2.10.0) with InstrumentationAnalysisTransform +Transforming error_prone_annotations-2.3.4.jar (com.google.errorprone:error_prone_annotations:2.3.4) with InstrumentationAnalysisTransform +Transforming j2objc-annotations-1.3.jar (com.google.j2objc:j2objc-annotations:1.3) with InstrumentationAnalysisTransform +Transforming JavaEWAH-1.2.3.jar (com.googlecode.javaewah:JavaEWAH:1.2.3) with InstrumentationAnalysisTransform +Transforming concurrent-trees-2.6.1.jar (com.googlecode.concurrent-trees:concurrent-trees:2.6.1) with InstrumentationAnalysisTransform +Transforming kotlin-stdlib-jdk7-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10) with InstrumentationAnalysisTransform +Transforming kotlin-stdlib-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib:1.9.10) with InstrumentationAnalysisTransform +Transforming jsch.agentproxy.usocket-nc-0.0.7.jar (com.jcraft:jsch.agentproxy.usocket-nc:0.0.7) with InstrumentationAnalysisTransform +Transforming jsch.agentproxy.sshagent-0.0.7.jar (com.jcraft:jsch.agentproxy.sshagent:0.0.7) with InstrumentationAnalysisTransform +Transforming jsch.agentproxy.core-0.0.7.jar (com.jcraft:jsch.agentproxy.core:0.0.7) with InstrumentationAnalysisTransform +Transforming maven-api-xml-4.0.0-alpha-9.jar (org.apache.maven:maven-api-xml:4.0.0-alpha-9) with InstrumentationAnalysisTransform +Transforming woodstox-core-6.5.1.jar (com.fasterxml.woodstox:woodstox-core:6.5.1) with InstrumentationAnalysisTransform +Transforming xz-1.9.jar (org.tukaani:xz:1.9) with InstrumentationAnalysisTransform +Transforming durian-swt.os-4.2.2.jar (com.diffplug.durian:durian-swt.os:4.2.2) with InstrumentationAnalysisTransform +Transforming kotlin-stdlib-common-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10) with InstrumentationAnalysisTransform +Transforming annotations-13.0.jar (org.jetbrains:annotations:13.0) with InstrumentationAnalysisTransform +Transforming platform-3.4.0.jar (net.java.dev.jna:platform:3.4.0) with InstrumentationAnalysisTransform +Transforming maven-api-meta-4.0.0-alpha-9.jar (org.apache.maven:maven-api-meta:4.0.0-alpha-9) with InstrumentationAnalysisTransform +Transforming stax2-api-4.2.1.jar (org.codehaus.woodstox:stax2-api:4.2.1) with InstrumentationAnalysisTransform +Transforming main (project :build-tools-internal) with InstrumentationAnalysisTransform +Transforming main (project :build-tools-internal) with InstrumentationAnalysisTransform +Transforming main (project :build-conventions) with InstrumentationAnalysisTransform +Transforming main (project :build-tools) with InstrumentationAnalysisTransform +Transforming main (project :build-tools) with InstrumentationAnalysisTransform +Transforming main (project :build-tools:reaper) with InstrumentationAnalysisTransform +Transforming shadow-gradle-plugin-8.3.5.jar (com.gradleup.shadow:shadow-gradle-plugin:8.3.5) with InstrumentationAnalysisTransform +Transforming shadow-gradle-plugin-8.3.5.jar (com.gradleup.shadow:shadow-gradle-plugin:8.3.5) with MergeInstrumentationAnalysisTransform +Transforming log4j-core-2.24.1.jar (org.apache.logging.log4j:log4j-core:2.24.1) with InstrumentationAnalysisTransform +Transforming log4j-core-2.24.1.jar (org.apache.logging.log4j:log4j-core:2.24.1) with MergeInstrumentationAnalysisTransform +Transforming shadow-gradle-plugin-8.3.5.jar (com.gradleup.shadow:shadow-gradle-plugin:8.3.5) with ExternalDependencyInstrumentingArtifactTransform +Transforming develocity-gradle-plugin-3.18.1.jar (com.gradle:develocity-gradle-plugin:3.18.1) with InstrumentationAnalysisTransform +Transforming develocity-gradle-plugin-3.18.1.jar (com.gradle:develocity-gradle-plugin:3.18.1) with MergeInstrumentationAnalysisTransform +Transforming log4j-core-2.24.1.jar (org.apache.logging.log4j:log4j-core:2.24.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming jackson-dataformat-yaml-2.15.0.jar (com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.15.0) with InstrumentationAnalysisTransform +Transforming jackson-dataformat-yaml-2.15.0.jar (com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.15.0) with MergeInstrumentationAnalysisTransform +Transforming develocity-gradle-plugin-3.18.1.jar (com.gradle:develocity-gradle-plugin:3.18.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming json-schema-validator-1.0.72.jar (com.networknt:json-schema-validator:1.0.72) with InstrumentationAnalysisTransform +Transforming json-schema-validator-1.0.72.jar (com.networknt:json-schema-validator:1.0.72) with MergeInstrumentationAnalysisTransform +Transforming jackson-dataformat-yaml-2.15.0.jar (com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.15.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming jackson-databind-2.15.0.jar (com.fasterxml.jackson.core:jackson-databind:2.15.0) with InstrumentationAnalysisTransform +Transforming jackson-databind-2.15.0.jar (com.fasterxml.jackson.core:jackson-databind:2.15.0) with MergeInstrumentationAnalysisTransform +Transforming json-schema-validator-1.0.72.jar (com.networknt:json-schema-validator:1.0.72) with ExternalDependencyInstrumentingArtifactTransform +Transforming jackson-core-2.15.0.jar (com.fasterxml.jackson.core:jackson-core:2.15.0) with InstrumentationAnalysisTransform +Transforming jackson-databind-2.15.0.jar (com.fasterxml.jackson.core:jackson-databind:2.15.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming jackson-core-2.15.0.jar (com.fasterxml.jackson.core:jackson-core:2.15.0) with MergeInstrumentationAnalysisTransform +Transforming jackson-annotations-2.15.0.jar (com.fasterxml.jackson.core:jackson-annotations:2.15.0) with InstrumentationAnalysisTransform +Transforming jackson-annotations-2.15.0.jar (com.fasterxml.jackson.core:jackson-annotations:2.15.0) with MergeInstrumentationAnalysisTransform +Transforming jackson-core-2.15.0.jar (com.fasterxml.jackson.core:jackson-core:2.15.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming apache-rat-0.11.jar (org.apache.rat:apache-rat:0.11) with InstrumentationAnalysisTransform +Transforming jackson-annotations-2.15.0.jar (com.fasterxml.jackson.core:jackson-annotations:2.15.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming apache-rat-0.11.jar (org.apache.rat:apache-rat:0.11) with MergeInstrumentationAnalysisTransform +Transforming apache-rat-tasks-0.11.jar (org.apache.rat:apache-rat-tasks:0.11) with InstrumentationAnalysisTransform +Transforming apache-rat-tasks-0.11.jar (org.apache.rat:apache-rat-tasks:0.11) with MergeInstrumentationAnalysisTransform +Transforming apache-rat-0.11.jar (org.apache.rat:apache-rat:0.11) with ExternalDependencyInstrumentingArtifactTransform +Transforming apache-rat-core-0.11.jar (org.apache.rat:apache-rat-core:0.11) with InstrumentationAnalysisTransform +Transforming apache-rat-core-0.11.jar (org.apache.rat:apache-rat-core:0.11) with MergeInstrumentationAnalysisTransform +Transforming apache-rat-tasks-0.11.jar (org.apache.rat:apache-rat-tasks:0.11) with ExternalDependencyInstrumentingArtifactTransform +Transforming commons-compress-1.26.1.jar (org.apache.commons:commons-compress:1.26.1) with InstrumentationAnalysisTransform +Transforming commons-compress-1.26.1.jar (org.apache.commons:commons-compress:1.26.1) with MergeInstrumentationAnalysisTransform +Transforming httpclient-4.5.14.jar (org.apache.httpcomponents:httpclient:4.5.14) with InstrumentationAnalysisTransform +Transforming apache-rat-core-0.11.jar (org.apache.rat:apache-rat-core:0.11) with ExternalDependencyInstrumentingArtifactTransform +Transforming httpclient-4.5.14.jar (org.apache.httpcomponents:httpclient:4.5.14) with MergeInstrumentationAnalysisTransform +Transforming gradle-info-plugin-11.3.3.jar (com.netflix.nebula:gradle-info-plugin:11.3.3) with InstrumentationAnalysisTransform +Transforming commons-compress-1.26.1.jar (org.apache.commons:commons-compress:1.26.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming httpclient-4.5.14.jar (org.apache.httpcomponents:httpclient:4.5.14) with ExternalDependencyInstrumentingArtifactTransform +Transforming gradle-info-plugin-11.3.3.jar (com.netflix.nebula:gradle-info-plugin:11.3.3) with MergeInstrumentationAnalysisTransform +Transforming spotless-plugin-gradle-6.25.0.jar (com.diffplug.spotless:spotless-plugin-gradle:6.25.0) with InstrumentationAnalysisTransform +Transforming spotless-plugin-gradle-6.25.0.jar (com.diffplug.spotless:spotless-plugin-gradle:6.25.0) with MergeInstrumentationAnalysisTransform +Transforming gradle-info-plugin-11.3.3.jar (com.netflix.nebula:gradle-info-plugin:11.3.3) with ExternalDependencyInstrumentingArtifactTransform +Transforming spotless-lib-extra-2.45.0.jar (com.diffplug.spotless:spotless-lib-extra:2.45.0) with InstrumentationAnalysisTransform +Transforming spotless-lib-extra-2.45.0.jar (com.diffplug.spotless:spotless-lib-extra:2.45.0) with MergeInstrumentationAnalysisTransform +Transforming org.eclipse.jgit-6.7.0.202309050840-r.jar (org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r) with InstrumentationAnalysisTransform +Transforming spotless-plugin-gradle-6.25.0.jar (com.diffplug.spotless:spotless-plugin-gradle:6.25.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming spotless-lib-extra-2.45.0.jar (com.diffplug.spotless:spotless-lib-extra:2.45.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming org.eclipse.jgit-6.7.0.202309050840-r.jar (org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r) with MergeInstrumentationAnalysisTransform +Transforming commons-codec-1.16.1.jar (commons-codec:commons-codec:1.16.1) with InstrumentationAnalysisTransform +Transforming commons-codec-1.16.1.jar (commons-codec:commons-codec:1.16.1) with MergeInstrumentationAnalysisTransform +Transforming org.eclipse.jgit-6.7.0.202309050840-r.jar (org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r) with ExternalDependencyInstrumentingArtifactTransform +Transforming svnkit-1.8.12.jar (org.tmatesoft.svnkit:svnkit:1.8.12) with InstrumentationAnalysisTransform +Transforming commons-codec-1.16.1.jar (commons-codec:commons-codec:1.16.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming svnkit-1.8.12.jar (org.tmatesoft.svnkit:svnkit:1.8.12) with MergeInstrumentationAnalysisTransform +Transforming jna-platform-5.7.0.jar (net.java.dev.jna:jna-platform:5.7.0) with InstrumentationAnalysisTransform +Transforming svnkit-1.8.12.jar (org.tmatesoft.svnkit:svnkit:1.8.12) with ExternalDependencyInstrumentingArtifactTransform +Transforming jna-platform-5.7.0.jar (net.java.dev.jna:jna-platform:5.7.0) with MergeInstrumentationAnalysisTransform +Transforming jsch.agentproxy.connector-factory-0.0.7.jar (com.jcraft:jsch.agentproxy.connector-factory:0.0.7) with InstrumentationAnalysisTransform +Transforming jna-platform-5.7.0.jar (net.java.dev.jna:jna-platform:5.7.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming jsch.agentproxy.connector-factory-0.0.7.jar (com.jcraft:jsch.agentproxy.connector-factory:0.0.7) with MergeInstrumentationAnalysisTransform +Transforming jsch.agentproxy.usocket-jna-0.0.7.jar (com.jcraft:jsch.agentproxy.usocket-jna:0.0.7) with InstrumentationAnalysisTransform +Transforming jsch.agentproxy.connector-factory-0.0.7.jar (com.jcraft:jsch.agentproxy.connector-factory:0.0.7) with ExternalDependencyInstrumentingArtifactTransform +Transforming jsch.agentproxy.usocket-jna-0.0.7.jar (com.jcraft:jsch.agentproxy.usocket-jna:0.0.7) with MergeInstrumentationAnalysisTransform +Transforming jsch.agentproxy.pageant-0.0.7.jar (com.jcraft:jsch.agentproxy.pageant:0.0.7) with InstrumentationAnalysisTransform +Transforming jsch.agentproxy.usocket-jna-0.0.7.jar (com.jcraft:jsch.agentproxy.usocket-jna:0.0.7) with ExternalDependencyInstrumentingArtifactTransform +Transforming jsch.agentproxy.pageant-0.0.7.jar (com.jcraft:jsch.agentproxy.pageant:0.0.7) with MergeInstrumentationAnalysisTransform +Transforming jna-5.10.0.jar (net.java.dev.jna:jna:5.10.0) with InstrumentationAnalysisTransform +Transforming jsch.agentproxy.pageant-0.0.7.jar (com.jcraft:jsch.agentproxy.pageant:0.0.7) with ExternalDependencyInstrumentingArtifactTransform +Transforming jna-5.10.0.jar (net.java.dev.jna:jna:5.10.0) with MergeInstrumentationAnalysisTransform +Transforming gradle-idea-ext-1.1.4.jar (gradle.plugin.org.jetbrains.gradle.plugin.idea-ext:gradle-idea-ext:1.1.4) with InstrumentationAnalysisTransform +Transforming jna-5.10.0.jar (net.java.dev.jna:jna:5.10.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming gradle-idea-ext-1.1.4.jar (gradle.plugin.org.jetbrains.gradle.plugin.idea-ext:gradle-idea-ext:1.1.4) with MergeInstrumentationAnalysisTransform +Transforming forbiddenapis-3.8.jar (de.thetaphi:forbiddenapis:3.8) with InstrumentationAnalysisTransform +Transforming gradle-idea-ext-1.1.4.jar (gradle.plugin.org.jetbrains.gradle.plugin.idea-ext:gradle-idea-ext:1.1.4) with ExternalDependencyInstrumentingArtifactTransform +Transforming forbiddenapis-3.8.jar (de.thetaphi:forbiddenapis:3.8) with MergeInstrumentationAnalysisTransform +Transforming gradle-docker-compose-plugin-0.17.5.jar (com.avast.gradle:gradle-docker-compose-plugin:0.17.5) with InstrumentationAnalysisTransform +Transforming gradle-docker-compose-plugin-0.17.5.jar (com.avast.gradle:gradle-docker-compose-plugin:0.17.5) with MergeInstrumentationAnalysisTransform +Transforming forbiddenapis-3.8.jar (de.thetaphi:forbiddenapis:3.8) with ExternalDependencyInstrumentingArtifactTransform +Transforming maven-model-3.6.2.jar (org.apache.maven:maven-model:3.6.2) with InstrumentationAnalysisTransform +Transforming gradle-docker-compose-plugin-0.17.5.jar (com.avast.gradle:gradle-docker-compose-plugin:0.17.5) with ExternalDependencyInstrumentingArtifactTransform +Transforming maven-model-3.6.2.jar (org.apache.maven:maven-model:3.6.2) with MergeInstrumentationAnalysisTransform +Transforming ST4-4.3.4.jar (org.antlr:ST4:4.3.4) with InstrumentationAnalysisTransform +Transforming ST4-4.3.4.jar (org.antlr:ST4:4.3.4) with MergeInstrumentationAnalysisTransform +Transforming maven-model-3.6.2.jar (org.apache.maven:maven-model:3.6.2) with ExternalDependencyInstrumentingArtifactTransform +Transforming asm-commons-9.7.1.jar (org.ow2.asm:asm-commons:9.7.1) with InstrumentationAnalysisTransform +Transforming ST4-4.3.4.jar (org.antlr:ST4:4.3.4) with ExternalDependencyInstrumentingArtifactTransform +Transforming asm-commons-9.7.1.jar (org.ow2.asm:asm-commons:9.7.1) with MergeInstrumentationAnalysisTransform +Transforming asm-tree-9.7.1.jar (org.ow2.asm:asm-tree:9.7.1) with InstrumentationAnalysisTransform +Transforming asm-commons-9.7.1.jar (org.ow2.asm:asm-commons:9.7.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming asm-tree-9.7.1.jar (org.ow2.asm:asm-tree:9.7.1) with MergeInstrumentationAnalysisTransform +Transforming asm-9.7.1.jar (org.ow2.asm:asm:9.7.1) with InstrumentationAnalysisTransform +Transforming asm-tree-9.7.1.jar (org.ow2.asm:asm-tree:9.7.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming asm-9.7.1.jar (org.ow2.asm:asm:9.7.1) with MergeInstrumentationAnalysisTransform +Transforming httpcore-4.4.16.jar (org.apache.httpcomponents:httpcore:4.4.16) with InstrumentationAnalysisTransform +Transforming asm-9.7.1.jar (org.ow2.asm:asm:9.7.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming httpcore-4.4.16.jar (org.apache.httpcomponents:httpcore:4.4.16) with MergeInstrumentationAnalysisTransform +Transforming snakeyaml-2.0.jar (org.yaml:snakeyaml:2.0) with InstrumentationAnalysisTransform +Transforming snakeyaml-2.0.jar (org.yaml:snakeyaml:2.0) with MergeInstrumentationAnalysisTransform +Transforming javaparser-core-3.18.0.jar (com.github.javaparser:javaparser-core:3.18.0) with InstrumentationAnalysisTransform +Transforming httpcore-4.4.16.jar (org.apache.httpcomponents:httpcore:4.4.16) with ExternalDependencyInstrumentingArtifactTransform +Transforming javaparser-core-3.18.0.jar (com.github.javaparser:javaparser-core:3.18.0) with MergeInstrumentationAnalysisTransform +Transforming solstice-1.7.5.jar (dev.equo.ide:solstice:1.7.5) with InstrumentationAnalysisTransform +Transforming solstice-1.7.5.jar (dev.equo.ide:solstice:1.7.5) with MergeInstrumentationAnalysisTransform +Transforming org.eclipse.osgi-3.18.300.jar (org.eclipse.platform:org.eclipse.osgi:3.18.300) with InstrumentationAnalysisTransform +Transforming snakeyaml-2.0.jar (org.yaml:snakeyaml:2.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming solstice-1.7.5.jar (dev.equo.ide:solstice:1.7.5) with ExternalDependencyInstrumentingArtifactTransform +Transforming org.eclipse.osgi-3.18.300.jar (org.eclipse.platform:org.eclipse.osgi:3.18.300) with MergeInstrumentationAnalysisTransform +Transforming javaparser-core-3.18.0.jar (com.github.javaparser:javaparser-core:3.18.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming ant-1.10.15.jar (org.apache.ant:ant:1.10.15) with InstrumentationAnalysisTransform +Transforming org.eclipse.osgi-3.18.300.jar (org.eclipse.platform:org.eclipse.osgi:3.18.300) with ExternalDependencyInstrumentingArtifactTransform +Transforming ant-1.10.15.jar (org.apache.ant:ant:1.10.15) with MergeInstrumentationAnalysisTransform +Transforming commons-io-2.17.0.jar (commons-io:commons-io:2.17.0) with InstrumentationAnalysisTransform +Transforming commons-io-2.17.0.jar (commons-io:commons-io:2.17.0) with MergeInstrumentationAnalysisTransform +Transforming commons-lang3-3.14.0.jar (org.apache.commons:commons-lang3:3.14.0) with InstrumentationAnalysisTransform +Transforming ant-1.10.15.jar (org.apache.ant:ant:1.10.15) with ExternalDependencyInstrumentingArtifactTransform +Transforming commons-io-2.17.0.jar (commons-io:commons-io:2.17.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming commons-lang3-3.14.0.jar (org.apache.commons:commons-lang3:3.14.0) with MergeInstrumentationAnalysisTransform +Transforming p4java-2015.2.1365273.jar (com.perforce:p4java:2015.2.1365273) with InstrumentationAnalysisTransform +Transforming p4java-2015.2.1365273.jar (com.perforce:p4java:2015.2.1365273) with MergeInstrumentationAnalysisTransform +Transforming nebula-gradle-interop-2.0.0.jar (com.netflix.nebula:nebula-gradle-interop:2.0.0) with InstrumentationAnalysisTransform +Transforming commons-lang3-3.14.0.jar (org.apache.commons:commons-lang3:3.14.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming p4java-2015.2.1365273.jar (com.perforce:p4java:2015.2.1365273) with ExternalDependencyInstrumentingArtifactTransform +Transforming nebula-gradle-interop-2.0.0.jar (com.netflix.nebula:nebula-gradle-interop:2.0.0) with MergeInstrumentationAnalysisTransform +Transforming gradle-contacts-plugin-6.0.0.jar (com.netflix.nebula:gradle-contacts-plugin:6.0.0) with InstrumentationAnalysisTransform +Transforming gradle-contacts-plugin-6.0.0.jar (com.netflix.nebula:gradle-contacts-plugin:6.0.0) with MergeInstrumentationAnalysisTransform +Transforming nebula-gradle-interop-2.0.0.jar (com.netflix.nebula:nebula-gradle-interop:2.0.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming jdom2-2.0.6.1.jar (org.jdom:jdom2:2.0.6.1) with InstrumentationAnalysisTransform +Transforming gradle-contacts-plugin-6.0.0.jar (com.netflix.nebula:gradle-contacts-plugin:6.0.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming jdom2-2.0.6.1.jar (org.jdom:jdom2:2.0.6.1) with MergeInstrumentationAnalysisTransform +Transforming plexus-utils-4.0.2.jar (org.codehaus.plexus:plexus-utils:4.0.2) with InstrumentationAnalysisTransform +Transforming plexus-utils-4.0.2.jar (org.codehaus.plexus:plexus-utils:4.0.2) with MergeInstrumentationAnalysisTransform +Transforming plexus-xml-4.0.4.jar (org.codehaus.plexus:plexus-xml:4.0.4) with InstrumentationAnalysisTransform +Transforming jdom2-2.0.6.1.jar (org.jdom:jdom2:2.0.6.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming plexus-xml-4.0.4.jar (org.codehaus.plexus:plexus-xml:4.0.4) with MergeInstrumentationAnalysisTransform +Transforming jdependency-2.11.jar (org.vafer:jdependency:2.11) with InstrumentationAnalysisTransform +Transforming jdependency-2.11.jar (org.vafer:jdependency:2.11) with MergeInstrumentationAnalysisTransform +Transforming plexus-utils-4.0.2.jar (org.codehaus.plexus:plexus-utils:4.0.2) with ExternalDependencyInstrumentingArtifactTransform +Transforming plexus-xml-4.0.4.jar (org.codehaus.plexus:plexus-xml:4.0.4) with ExternalDependencyInstrumentingArtifactTransform +Transforming gson-2.8.6.jar (com.google.code.gson:gson:2.8.6) with InstrumentationAnalysisTransform +Transforming gson-2.8.6.jar (com.google.code.gson:gson:2.8.6) with MergeInstrumentationAnalysisTransform +Transforming jdependency-2.11.jar (org.vafer:jdependency:2.11) with ExternalDependencyInstrumentingArtifactTransform +Transforming guava-28.2-jre.jar (com.google.guava:guava:28.2-jre) with InstrumentationAnalysisTransform +Transforming gson-2.8.6.jar (com.google.code.gson:gson:2.8.6) with ExternalDependencyInstrumentingArtifactTransform +Transforming guava-28.2-jre.jar (com.google.guava:guava:28.2-jre) with MergeInstrumentationAnalysisTransform +Transforming slf4j-api-1.7.36.jar (org.slf4j:slf4j-api:1.7.36) with InstrumentationAnalysisTransform +Transforming slf4j-api-1.7.36.jar (org.slf4j:slf4j-api:1.7.36) with MergeInstrumentationAnalysisTransform +Transforming itu-1.7.0.jar (com.ethlo.time:itu:1.7.0) with InstrumentationAnalysisTransform +Transforming guava-28.2-jre.jar (com.google.guava:guava:28.2-jre) with ExternalDependencyInstrumentingArtifactTransform +Transforming slf4j-api-1.7.36.jar (org.slf4j:slf4j-api:1.7.36) with ExternalDependencyInstrumentingArtifactTransform +Transforming itu-1.7.0.jar (com.ethlo.time:itu:1.7.0) with MergeInstrumentationAnalysisTransform +Transforming sqljet-1.1.10.jar (org.tmatesoft.sqljet:sqljet:1.1.10) with InstrumentationAnalysisTransform +Transforming sqljet-1.1.10.jar (org.tmatesoft.sqljet:sqljet:1.1.10) with MergeInstrumentationAnalysisTransform +Transforming itu-1.7.0.jar (com.ethlo.time:itu:1.7.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming antlr-runtime-3.5.3.jar (org.antlr:antlr-runtime:3.5.3) with InstrumentationAnalysisTransform +Transforming sqljet-1.1.10.jar (org.tmatesoft.sqljet:sqljet:1.1.10) with ExternalDependencyInstrumentingArtifactTransform +Transforming antlr-runtime-3.5.3.jar (org.antlr:antlr-runtime:3.5.3) with MergeInstrumentationAnalysisTransform +Transforming commons-logging-1.2.jar (commons-logging:commons-logging:1.2) with InstrumentationAnalysisTransform +Transforming commons-logging-1.2.jar (commons-logging:commons-logging:1.2) with MergeInstrumentationAnalysisTransform +Transforming durian-io-1.2.0.jar (com.diffplug.durian:durian-io:1.2.0) with InstrumentationAnalysisTransform +Transforming antlr-runtime-3.5.3.jar (org.antlr:antlr-runtime:3.5.3) with ExternalDependencyInstrumentingArtifactTransform +Transforming commons-logging-1.2.jar (commons-logging:commons-logging:1.2) with ExternalDependencyInstrumentingArtifactTransform +Transforming durian-io-1.2.0.jar (com.diffplug.durian:durian-io:1.2.0) with MergeInstrumentationAnalysisTransform +Transforming durian-collect-1.2.0.jar (com.diffplug.durian:durian-collect:1.2.0) with InstrumentationAnalysisTransform +Transforming durian-collect-1.2.0.jar (com.diffplug.durian:durian-collect:1.2.0) with MergeInstrumentationAnalysisTransform +Transforming durian-io-1.2.0.jar (com.diffplug.durian:durian-io:1.2.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming durian-core-1.2.0.jar (com.diffplug.durian:durian-core:1.2.0) with InstrumentationAnalysisTransform +Transforming durian-collect-1.2.0.jar (com.diffplug.durian:durian-collect:1.2.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming durian-core-1.2.0.jar (com.diffplug.durian:durian-core:1.2.0) with MergeInstrumentationAnalysisTransform +Transforming spotless-lib-2.45.0.jar (com.diffplug.spotless:spotless-lib:2.45.0) with InstrumentationAnalysisTransform +Transforming durian-core-1.2.0.jar (com.diffplug.durian:durian-core:1.2.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming spotless-lib-2.45.0.jar (com.diffplug.spotless:spotless-lib:2.45.0) with MergeInstrumentationAnalysisTransform +Transforming fastdoubleparser-0.8.0.jar (ch.randelshofer:fastdoubleparser:0.8.0) with InstrumentationAnalysisTransform +Transforming fastdoubleparser-0.8.0.jar (ch.randelshofer:fastdoubleparser:0.8.0) with MergeInstrumentationAnalysisTransform +Transforming spotless-lib-2.45.0.jar (com.diffplug.spotless:spotless-lib:2.45.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming jzlib-1.1.2.jar (com.jcraft:jzlib:1.1.2) with InstrumentationAnalysisTransform +Transforming jzlib-1.1.2.jar (com.jcraft:jzlib:1.1.2) with MergeInstrumentationAnalysisTransform +Transforming fastdoubleparser-0.8.0.jar (ch.randelshofer:fastdoubleparser:0.8.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming okhttp-4.12.0.jar (com.squareup.okhttp3:okhttp:4.12.0) with InstrumentationAnalysisTransform +Transforming jzlib-1.1.2.jar (com.jcraft:jzlib:1.1.2) with ExternalDependencyInstrumentingArtifactTransform +Transforming okhttp-4.12.0.jar (com.squareup.okhttp3:okhttp:4.12.0) with MergeInstrumentationAnalysisTransform +Transforming okio-jvm-3.6.0.jar (com.squareup.okio:okio-jvm:3.6.0) with InstrumentationAnalysisTransform +Transforming okio-jvm-3.6.0.jar (com.squareup.okio:okio-jvm:3.6.0) with MergeInstrumentationAnalysisTransform +Transforming okhttp-4.12.0.jar (com.squareup.okhttp3:okhttp:4.12.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming kotlin-stdlib-jdk8-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10) with InstrumentationAnalysisTransform +Transforming okio-jvm-3.6.0.jar (com.squareup.okio:okio-jvm:3.6.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming kotlin-stdlib-jdk8-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10) with MergeInstrumentationAnalysisTransform +Transforming jsch.agentproxy.svnkit-trilead-ssh2-0.0.7.jar (com.jcraft:jsch.agentproxy.svnkit-trilead-ssh2:0.0.7) with InstrumentationAnalysisTransform +Transforming jsch.agentproxy.svnkit-trilead-ssh2-0.0.7.jar (com.jcraft:jsch.agentproxy.svnkit-trilead-ssh2:0.0.7) with MergeInstrumentationAnalysisTransform +Transforming trilead-ssh2-1.0.0-build220.jar (com.trilead:trilead-ssh2:1.0.0-build220) with InstrumentationAnalysisTransform +Transforming kotlin-stdlib-jdk8-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10) with ExternalDependencyInstrumentingArtifactTransform +Transforming jsch.agentproxy.svnkit-trilead-ssh2-0.0.7.jar (com.jcraft:jsch.agentproxy.svnkit-trilead-ssh2:0.0.7) with ExternalDependencyInstrumentingArtifactTransform +Transforming trilead-ssh2-1.0.0-build220.jar (com.trilead:trilead-ssh2:1.0.0-build220) with MergeInstrumentationAnalysisTransform +Transforming sequence-library-1.0.3.jar (de.regnis.q.sequence:sequence-library:1.0.3) with InstrumentationAnalysisTransform +Transforming sequence-library-1.0.3.jar (de.regnis.q.sequence:sequence-library:1.0.3) with MergeInstrumentationAnalysisTransform +Transforming trilead-ssh2-1.0.0-build220.jar (com.trilead:trilead-ssh2:1.0.0-build220) with ExternalDependencyInstrumentingArtifactTransform +Transforming commons-collections-3.2.1.jar (commons-collections:commons-collections:3.2.1) with InstrumentationAnalysisTransform +Transforming commons-collections-3.2.1.jar (commons-collections:commons-collections:3.2.1) with MergeInstrumentationAnalysisTransform +Transforming commons-lang-2.6.jar (commons-lang:commons-lang:2.6) with InstrumentationAnalysisTransform +Transforming sequence-library-1.0.3.jar (de.regnis.q.sequence:sequence-library:1.0.3) with ExternalDependencyInstrumentingArtifactTransform +Transforming commons-lang-2.6.jar (commons-lang:commons-lang:2.6) with MergeInstrumentationAnalysisTransform +Transforming commons-cli-1.2.jar (commons-cli:commons-cli:1.2) with InstrumentationAnalysisTransform +Transforming commons-collections-3.2.1.jar (commons-collections:commons-collections:3.2.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming commons-lang-2.6.jar (commons-lang:commons-lang:2.6) with ExternalDependencyInstrumentingArtifactTransform +Transforming commons-cli-1.2.jar (commons-cli:commons-cli:1.2) with MergeInstrumentationAnalysisTransform +Transforming ant-launcher-1.10.15.jar (org.apache.ant:ant-launcher:1.10.15) with InstrumentationAnalysisTransform +Transforming commons-cli-1.2.jar (commons-cli:commons-cli:1.2) with ExternalDependencyInstrumentingArtifactTransform +Transforming ant-launcher-1.10.15.jar (org.apache.ant:ant-launcher:1.10.15) with MergeInstrumentationAnalysisTransform +Transforming maven-xml-impl-4.0.0-alpha-9.jar (org.apache.maven:maven-xml-impl:4.0.0-alpha-9) with InstrumentationAnalysisTransform +Transforming ant-launcher-1.10.15.jar (org.apache.ant:ant-launcher:1.10.15) with ExternalDependencyInstrumentingArtifactTransform +Transforming maven-xml-impl-4.0.0-alpha-9.jar (org.apache.maven:maven-xml-impl:4.0.0-alpha-9) with MergeInstrumentationAnalysisTransform +Transforming log4j-api-2.24.1.jar (org.apache.logging.log4j:log4j-api:2.24.1) with InstrumentationAnalysisTransform +Transforming log4j-api-2.24.1.jar (org.apache.logging.log4j:log4j-api:2.24.1) with MergeInstrumentationAnalysisTransform +Transforming maven-xml-impl-4.0.0-alpha-9.jar (org.apache.maven:maven-xml-impl:4.0.0-alpha-9) with ExternalDependencyInstrumentingArtifactTransform +Transforming failureaccess-1.0.1.jar (com.google.guava:failureaccess:1.0.1) with InstrumentationAnalysisTransform +Transforming log4j-api-2.24.1.jar (org.apache.logging.log4j:log4j-api:2.24.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming failureaccess-1.0.1.jar (com.google.guava:failureaccess:1.0.1) with MergeInstrumentationAnalysisTransform +Transforming listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar (com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava) with InstrumentationAnalysisTransform +Transforming failureaccess-1.0.1.jar (com.google.guava:failureaccess:1.0.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar (com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava) with MergeInstrumentationAnalysisTransform +Transforming jsr305-3.0.2.jar (com.google.code.findbugs:jsr305:3.0.2) with InstrumentationAnalysisTransform +Transforming listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar (com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava) with ExternalDependencyInstrumentingArtifactTransform +Transforming jsr305-3.0.2.jar (com.google.code.findbugs:jsr305:3.0.2) with MergeInstrumentationAnalysisTransform +Transforming checker-qual-2.10.0.jar (org.checkerframework:checker-qual:2.10.0) with InstrumentationAnalysisTransform +Transforming checker-qual-2.10.0.jar (org.checkerframework:checker-qual:2.10.0) with MergeInstrumentationAnalysisTransform +Transforming error_prone_annotations-2.3.4.jar (com.google.errorprone:error_prone_annotations:2.3.4) with InstrumentationAnalysisTransform +Transforming jsr305-3.0.2.jar (com.google.code.findbugs:jsr305:3.0.2) with ExternalDependencyInstrumentingArtifactTransform +Transforming error_prone_annotations-2.3.4.jar (com.google.errorprone:error_prone_annotations:2.3.4) with MergeInstrumentationAnalysisTransform +Transforming j2objc-annotations-1.3.jar (com.google.j2objc:j2objc-annotations:1.3) with InstrumentationAnalysisTransform +Transforming checker-qual-2.10.0.jar (org.checkerframework:checker-qual:2.10.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming error_prone_annotations-2.3.4.jar (com.google.errorprone:error_prone_annotations:2.3.4) with ExternalDependencyInstrumentingArtifactTransform +Transforming j2objc-annotations-1.3.jar (com.google.j2objc:j2objc-annotations:1.3) with MergeInstrumentationAnalysisTransform +Transforming JavaEWAH-1.2.3.jar (com.googlecode.javaewah:JavaEWAH:1.2.3) with InstrumentationAnalysisTransform +Transforming JavaEWAH-1.2.3.jar (com.googlecode.javaewah:JavaEWAH:1.2.3) with MergeInstrumentationAnalysisTransform +Transforming concurrent-trees-2.6.1.jar (com.googlecode.concurrent-trees:concurrent-trees:2.6.1) with InstrumentationAnalysisTransform +Transforming j2objc-annotations-1.3.jar (com.google.j2objc:j2objc-annotations:1.3) with ExternalDependencyInstrumentingArtifactTransform +Transforming concurrent-trees-2.6.1.jar (com.googlecode.concurrent-trees:concurrent-trees:2.6.1) with MergeInstrumentationAnalysisTransform +Transforming JavaEWAH-1.2.3.jar (com.googlecode.javaewah:JavaEWAH:1.2.3) with ExternalDependencyInstrumentingArtifactTransform +Transforming kotlin-stdlib-jdk7-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10) with InstrumentationAnalysisTransform +Transforming concurrent-trees-2.6.1.jar (com.googlecode.concurrent-trees:concurrent-trees:2.6.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming kotlin-stdlib-jdk7-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10) with MergeInstrumentationAnalysisTransform +Transforming kotlin-stdlib-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib:1.9.10) with InstrumentationAnalysisTransform +Transforming kotlin-stdlib-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib:1.9.10) with MergeInstrumentationAnalysisTransform +Transforming kotlin-stdlib-jdk7-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10) with ExternalDependencyInstrumentingArtifactTransform +Transforming kotlin-stdlib-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib:1.9.10) with ExternalDependencyInstrumentingArtifactTransform +Transforming jsch.agentproxy.usocket-nc-0.0.7.jar (com.jcraft:jsch.agentproxy.usocket-nc:0.0.7) with InstrumentationAnalysisTransform +Transforming jsch.agentproxy.usocket-nc-0.0.7.jar (com.jcraft:jsch.agentproxy.usocket-nc:0.0.7) with MergeInstrumentationAnalysisTransform +Transforming jsch.agentproxy.sshagent-0.0.7.jar (com.jcraft:jsch.agentproxy.sshagent:0.0.7) with InstrumentationAnalysisTransform +Transforming jsch.agentproxy.sshagent-0.0.7.jar (com.jcraft:jsch.agentproxy.sshagent:0.0.7) with MergeInstrumentationAnalysisTransform +Transforming jsch.agentproxy.core-0.0.7.jar (com.jcraft:jsch.agentproxy.core:0.0.7) with InstrumentationAnalysisTransform +Transforming jsch.agentproxy.usocket-nc-0.0.7.jar (com.jcraft:jsch.agentproxy.usocket-nc:0.0.7) with ExternalDependencyInstrumentingArtifactTransform +Transforming jsch.agentproxy.core-0.0.7.jar (com.jcraft:jsch.agentproxy.core:0.0.7) with MergeInstrumentationAnalysisTransform +Transforming jsch.agentproxy.sshagent-0.0.7.jar (com.jcraft:jsch.agentproxy.sshagent:0.0.7) with ExternalDependencyInstrumentingArtifactTransform +Transforming jsch.agentproxy.core-0.0.7.jar (com.jcraft:jsch.agentproxy.core:0.0.7) with ExternalDependencyInstrumentingArtifactTransform +Transforming maven-api-xml-4.0.0-alpha-9.jar (org.apache.maven:maven-api-xml:4.0.0-alpha-9) with InstrumentationAnalysisTransform +Transforming maven-api-xml-4.0.0-alpha-9.jar (org.apache.maven:maven-api-xml:4.0.0-alpha-9) with MergeInstrumentationAnalysisTransform +Transforming woodstox-core-6.5.1.jar (com.fasterxml.woodstox:woodstox-core:6.5.1) with InstrumentationAnalysisTransform +Transforming woodstox-core-6.5.1.jar (com.fasterxml.woodstox:woodstox-core:6.5.1) with MergeInstrumentationAnalysisTransform +Transforming maven-api-xml-4.0.0-alpha-9.jar (org.apache.maven:maven-api-xml:4.0.0-alpha-9) with ExternalDependencyInstrumentingArtifactTransform +Transforming xz-1.9.jar (org.tukaani:xz:1.9) with InstrumentationAnalysisTransform +Transforming xz-1.9.jar (org.tukaani:xz:1.9) with MergeInstrumentationAnalysisTransform +Transforming woodstox-core-6.5.1.jar (com.fasterxml.woodstox:woodstox-core:6.5.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming xz-1.9.jar (org.tukaani:xz:1.9) with ExternalDependencyInstrumentingArtifactTransform +Transforming durian-swt.os-4.2.2.jar (com.diffplug.durian:durian-swt.os:4.2.2) with InstrumentationAnalysisTransform +Transforming durian-swt.os-4.2.2.jar (com.diffplug.durian:durian-swt.os:4.2.2) with MergeInstrumentationAnalysisTransform +Transforming kotlin-stdlib-common-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10) with InstrumentationAnalysisTransform +Transforming kotlin-stdlib-common-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10) with MergeInstrumentationAnalysisTransform +Transforming durian-swt.os-4.2.2.jar (com.diffplug.durian:durian-swt.os:4.2.2) with ExternalDependencyInstrumentingArtifactTransform +Transforming annotations-13.0.jar (org.jetbrains:annotations:13.0) with InstrumentationAnalysisTransform +Transforming annotations-13.0.jar (org.jetbrains:annotations:13.0) with MergeInstrumentationAnalysisTransform +Transforming platform-3.4.0.jar (net.java.dev.jna:platform:3.4.0) with InstrumentationAnalysisTransform +Transforming kotlin-stdlib-common-1.9.10.jar (org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10) with ExternalDependencyInstrumentingArtifactTransform +Transforming annotations-13.0.jar (org.jetbrains:annotations:13.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming platform-3.4.0.jar (net.java.dev.jna:platform:3.4.0) with MergeInstrumentationAnalysisTransform +Transforming maven-api-meta-4.0.0-alpha-9.jar (org.apache.maven:maven-api-meta:4.0.0-alpha-9) with InstrumentationAnalysisTransform +Transforming maven-api-meta-4.0.0-alpha-9.jar (org.apache.maven:maven-api-meta:4.0.0-alpha-9) with MergeInstrumentationAnalysisTransform +Transforming platform-3.4.0.jar (net.java.dev.jna:platform:3.4.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming stax2-api-4.2.1.jar (org.codehaus.woodstox:stax2-api:4.2.1) with InstrumentationAnalysisTransform +Transforming stax2-api-4.2.1.jar (org.codehaus.woodstox:stax2-api:4.2.1) with MergeInstrumentationAnalysisTransform +Transforming maven-api-meta-4.0.0-alpha-9.jar (org.apache.maven:maven-api-meta:4.0.0-alpha-9) with ExternalDependencyInstrumentingArtifactTransform +Transforming stax2-api-4.2.1.jar (org.codehaus.woodstox:stax2-api:4.2.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming build-tools-internal-9.0.0-SNAPSHOT.jar (project :build-tools-internal) with ProjectDependencyInstrumentingArtifactTransform +Transforming build-conventions.jar (project :build-conventions) with ProjectDependencyInstrumentingArtifactTransform +Transforming build-tools-9.0.0-SNAPSHOT.jar (project :build-tools) with ProjectDependencyInstrumentingArtifactTransform +Transforming reaper.jar (project :build-tools:reaper) with ProjectDependencyInstrumentingArtifactTransform +Settings evaluated using settings file '/Users/rene/dev/elastic/elasticsearch/settings.gradle'. +Projects loaded. Root project using build file '/Users/rene/dev/elastic/elasticsearch/build.gradle'. +Included projects: [root project 'elasticsearch', project ':benchmarks', project ':client', project ':distribution', project ':docs', project ':libs', project ':modules', project ':plugins', project ':qa', project ':rest-api-spec', project ':server', project ':test', project ':x-pack', project ':client:benchmark', project ':client:client-benchmark-noop-api-plugin', project ':client:rest', project ':client:sniffer', project ':client:test', project ':distribution:archives', project ':distribution:bwc', project ':distribution:docker', project ':distribution:packages', project ':distribution:tools', project ':libs:cli', project ':libs:core', project ':libs:dissect', project ':libs:entitlement', project ':libs:geo', project ':libs:grok', project ':libs:h3', project ':libs:log4j', project ':libs:logging', project ':libs:logstash-bridge', project ':libs:lz4', project ':libs:native', project ':libs:plugin-analysis-api', project ':libs:plugin-api', project ':libs:plugin-scanner', project ':libs:secure-sm', project ':libs:simdvec', project ':libs:ssl-config', project ':libs:tdigest', project ':libs:x-content', project ':modules:aggregations', project ':modules:analysis-common', project ':modules:apm', project ':modules:data-streams', project ':modules:dot-prefix-validation', project ':modules:health-shards-availability', project ':modules:ingest-attachment', project ':modules:ingest-common', project ':modules:ingest-geoip', project ':modules:ingest-user-agent', project ':modules:kibana', project ':modules:lang-expression', project ':modules:lang-mustache', project ':modules:lang-painless', project ':modules:legacy-geo', project ':modules:mapper-extras', project ':modules:parent-join', project ':modules:percolator', project ':modules:rank-eval', project ':modules:reindex', project ':modules:repository-azure', project ':modules:repository-gcs', project ':modules:repository-s3', project ':modules:repository-url', project ':modules:rest-root', project ':modules:runtime-fields-common', project ':modules:systemd', project ':modules:transport-netty4', project ':plugins:analysis-icu', project ':plugins:analysis-kuromoji', project ':plugins:analysis-nori', project ':plugins:analysis-phonetic', project ':plugins:analysis-smartcn', project ':plugins:analysis-stempel', project ':plugins:analysis-ukrainian', project ':plugins:discovery-azure-classic', project ':plugins:discovery-ec2', project ':plugins:discovery-gce', project ':plugins:mapper-annotated-text', project ':plugins:mapper-murmur3', project ':plugins:mapper-size', project ':plugins:repository-hdfs', project ':plugins:store-smb', project ':qa:ccs-common-rest', project ':qa:ccs-rolling-upgrade-remote-cluster', project ':qa:ccs-unavailable-clusters', project ':qa:custom-rest-controller', project ':qa:evil-tests', project ':qa:full-cluster-restart', project ':qa:logging-config', project ':qa:logging-spi', project ':qa:lucene-index-compatibility', project ':qa:mixed-cluster', project ':qa:multi-cluster-search', project ':qa:no-bootstrap-tests', project ':qa:packaging', project ':qa:remote-clusters', project ':qa:repository-multi-version', project ':qa:restricted-loggers', project ':qa:rolling-upgrade', project ':qa:rolling-upgrade-legacy', project ':qa:smoke-test-http', project ':qa:smoke-test-ingest-disabled', project ':qa:smoke-test-ingest-with-all-dependencies', project ':qa:smoke-test-multinode', project ':qa:smoke-test-plugins', project ':qa:stable-api', project ':qa:system-indices', project ':qa:unconfigured-node-name', project ':qa:verify-version-constants', project ':test:external-modules', project ':test:fixtures', project ':test:framework', project ':test:immutable-collections-patch', project ':test:logger-usage', project ':test:metadata-extractor', project ':test:test-clusters', project ':test:x-content', project ':test:yaml-rest-runner', project ':x-pack:libs', project ':x-pack:license-tools', project ':x-pack:plugin', project ':x-pack:qa', project ':x-pack:rest-resources-zip', project ':x-pack:test', project ':distribution:archives:darwin-aarch64-tar', project ':distribution:archives:darwin-tar', project ':distribution:archives:integ-test-zip', project ':distribution:archives:linux-aarch64-tar', project ':distribution:archives:linux-tar', project ':distribution:archives:windows-zip', project ':distribution:bwc:bugfix', project ':distribution:bwc:bugfix2', project ':distribution:bwc:maintenance', project ':distribution:bwc:minor', project ':distribution:bwc:staged', project ':distribution:docker:cloud-ess-docker-aarch64-export', project ':distribution:docker:cloud-ess-docker-export', project ':distribution:docker:docker-aarch64-export', project ':distribution:docker:docker-export', project ':distribution:docker:ironbank-docker-aarch64-export', project ':distribution:docker:ironbank-docker-export', project ':distribution:docker:wolfi-docker-aarch64-export', project ':distribution:docker:wolfi-docker-export', project ':distribution:packages:aarch64-deb', project ':distribution:packages:aarch64-rpm', project ':distribution:packages:deb', project ':distribution:packages:rpm', project ':distribution:tools:ansi-console', project ':distribution:tools:cli-launcher', project ':distribution:tools:geoip-cli', project ':distribution:tools:java-version-checker', project ':distribution:tools:keystore-cli', project ':distribution:tools:plugin-cli', project ':distribution:tools:server-cli', project ':distribution:tools:windows-service-cli', project ':libs:entitlement:agent', project ':libs:entitlement:asm-provider', project ':libs:entitlement:bridge', project ':libs:entitlement:qa', project ':libs:entitlement:tools', project ':libs:native:native-libraries', project ':libs:x-content:impl', project ':modules:ingest-geoip:qa', project ':modules:lang-painless:spi', project ':plugins:discovery-ec2:qa', project ':plugins:discovery-gce:qa', project ':plugins:repository-hdfs:hadoop-client-api', project ':qa:stable-api:logging', project ':qa:stable-api:plugin-analysis-api', project ':qa:stable-api:plugin-api', project ':test:external-modules:test-apm-integration', project ':test:external-modules:test-delayed-aggs', project ':test:external-modules:test-die-with-dignity', project ':test:external-modules:test-error-query', project ':test:external-modules:test-esql-heap-attack', project ':test:external-modules:test-jvm-crash', project ':test:external-modules:test-latency-simulating-directory', project ':test:fixtures:aws-sts-fixture', project ':test:fixtures:azure-fixture', project ':test:fixtures:ec2-imds-fixture', project ':test:fixtures:gcs-fixture', project ':test:fixtures:geoip-fixture', project ':test:fixtures:hdfs-fixture', project ':test:fixtures:krb5kdc-fixture', project ':test:fixtures:minio-fixture', project ':test:fixtures:old-elasticsearch', project ':test:fixtures:s3-fixture', project ':test:fixtures:testcontainer-utils', project ':test:fixtures:url-fixture', project ':x-pack:libs:es-opensaml-security-api', project ':x-pack:plugin:analytics', project ':x-pack:plugin:apm-data', project ':x-pack:plugin:async', project ':x-pack:plugin:async-search', project ':x-pack:plugin:autoscaling', project ':x-pack:plugin:blob-cache', project ':x-pack:plugin:ccr', project ':x-pack:plugin:core', project ':x-pack:plugin:deprecation', project ':x-pack:plugin:downsample', project ':x-pack:plugin:enrich', project ':x-pack:plugin:ent-search', project ':x-pack:plugin:eql', project ':x-pack:plugin:esql', project ':x-pack:plugin:esql-core', project ':x-pack:plugin:fleet', project ':x-pack:plugin:frozen-indices', project ':x-pack:plugin:geoip-enterprise-downloader', project ':x-pack:plugin:graph', project ':x-pack:plugin:identity-provider', project ':x-pack:plugin:ilm', project ':x-pack:plugin:inference', project ':x-pack:plugin:kql', project ':x-pack:plugin:logsdb', project ':x-pack:plugin:logstash', project ':x-pack:plugin:mapper-aggregate-metric', project ':x-pack:plugin:mapper-constant-keyword', project ':x-pack:plugin:mapper-counted-keyword', project ':x-pack:plugin:mapper-unsigned-long', project ':x-pack:plugin:mapper-version', project ':x-pack:plugin:migrate', project ':x-pack:plugin:ml', project ':x-pack:plugin:ml-package-loader', project ':x-pack:plugin:monitoring', project ':x-pack:plugin:old-lucene-versions', project ':x-pack:plugin:otel-data', project ':x-pack:plugin:profiling', project ':x-pack:plugin:ql', project ':x-pack:plugin:rank-rrf', project ':x-pack:plugin:redact', project ':x-pack:plugin:repositories-metering-api', project ':x-pack:plugin:rollup', project ':x-pack:plugin:search-business-rules', project ':x-pack:plugin:searchable-snapshots', project ':x-pack:plugin:security', project ':x-pack:plugin:shutdown', project ':x-pack:plugin:slm', project ':x-pack:plugin:snapshot-based-recoveries', project ':x-pack:plugin:snapshot-repo-test-kit', project ':x-pack:plugin:spatial', project ':x-pack:plugin:sql', project ':x-pack:plugin:stack', project ':x-pack:plugin:text-structure', project ':x-pack:plugin:transform', project ':x-pack:plugin:vector-tile', project ':x-pack:plugin:voting-only-node', project ':x-pack:plugin:watcher', project ':x-pack:plugin:wildcard', project ':x-pack:plugin:write-load-forecaster', project ':x-pack:qa:core-rest-tests-with-security', project ':x-pack:qa:evil-tests', project ':x-pack:qa:freeze-plugin', project ':x-pack:qa:full-cluster-restart', project ':x-pack:qa:kerberos-tests', project ':x-pack:qa:mixed-tier-cluster', project ':x-pack:qa:multi-cluster-search-security', project ':x-pack:qa:multi-node', project ':x-pack:qa:oidc-op-tests', project ':x-pack:qa:openldap-tests', project ':x-pack:qa:password-protected-keystore', project ':x-pack:qa:reindex-tests-with-security', project ':x-pack:qa:repository-old-versions', project ':x-pack:qa:rolling-upgrade', project ':x-pack:qa:rolling-upgrade-basic', project ':x-pack:qa:rolling-upgrade-multi-cluster', project ':x-pack:qa:runtime-fields', project ':x-pack:qa:saml-idp-tests', project ':x-pack:qa:security-example-spi-extension', project ':x-pack:qa:security-setup-password-tests', project ':x-pack:qa:security-tools-tests', project ':x-pack:qa:smoke-test-plugins', project ':x-pack:qa:smoke-test-plugins-ssl', project ':x-pack:qa:smoke-test-security-with-mustache', project ':x-pack:qa:third-party', project ':x-pack:test:idp-fixture', project ':x-pack:test:smb-fixture', project ':libs:entitlement:qa:common', project ':libs:entitlement:qa:entitlement-allowed', project ':libs:entitlement:qa:entitlement-allowed-nonmodular', project ':libs:entitlement:qa:entitlement-denied', project ':libs:entitlement:qa:entitlement-denied-nonmodular', project ':libs:entitlement:tools:common', project ':libs:entitlement:tools:public-callers-finder', project ':libs:entitlement:tools:securitymanager-scanner', project ':modules:ingest-geoip:qa:file-based-update', project ':modules:ingest-geoip:qa:full-cluster-restart', project ':plugins:discovery-ec2:qa:amazon-ec2', project ':plugins:discovery-gce:qa:gce', project ':x-pack:plugin:async-search:qa', project ':x-pack:plugin:autoscaling:qa', project ':x-pack:plugin:ccr:qa', project ':x-pack:plugin:core:template-resources', project ':x-pack:plugin:deprecation:qa', project ':x-pack:plugin:downsample:qa', project ':x-pack:plugin:enrich:qa', project ':x-pack:plugin:ent-search:qa', project ':x-pack:plugin:eql:qa', project ':x-pack:plugin:esql:arrow', project ':x-pack:plugin:esql:compute', project ':x-pack:plugin:esql:qa', project ':x-pack:plugin:esql-core:test-fixtures', project ':x-pack:plugin:fleet:qa', project ':x-pack:plugin:graph:qa', project ':x-pack:plugin:identity-provider:qa', project ':x-pack:plugin:ilm:qa', project ':x-pack:plugin:inference:qa', project ':x-pack:plugin:logsdb:qa', project ':x-pack:plugin:ml:qa', project ':x-pack:plugin:ql:test-fixtures', project ':x-pack:plugin:repositories-metering-api:qa', project ':x-pack:plugin:searchable-snapshots:qa', project ':x-pack:plugin:security:cli', project ':x-pack:plugin:security:lib', project ':x-pack:plugin:security:qa', project ':x-pack:plugin:shutdown:qa', project ':x-pack:plugin:slm:qa', project ':x-pack:plugin:snapshot-based-recoveries:qa', project ':x-pack:plugin:snapshot-repo-test-kit:qa', project ':x-pack:plugin:sql:jdbc', project ':x-pack:plugin:sql:qa', project ':x-pack:plugin:sql:sql-action', project ':x-pack:plugin:sql:sql-cli', project ':x-pack:plugin:sql:sql-client', project ':x-pack:plugin:sql:sql-proto', project ':x-pack:plugin:stack:qa', project ':x-pack:plugin:text-structure:qa', project ':x-pack:plugin:transform:qa', project ':x-pack:plugin:vector-tile:qa', project ':x-pack:plugin:watcher:qa', project ':x-pack:qa:multi-cluster-search-security:legacy-with-basic-license', project ':x-pack:qa:multi-cluster-search-security:legacy-with-full-license', project ':x-pack:qa:multi-cluster-search-security:legacy-with-restricted-trust', project ':x-pack:qa:runtime-fields:core-with-mapped', project ':x-pack:qa:runtime-fields:core-with-search', project ':x-pack:qa:runtime-fields:with-security', project ':x-pack:qa:third-party:active-directory', project ':x-pack:qa:third-party:jira', project ':x-pack:qa:third-party:pagerduty', project ':x-pack:qa:third-party:slack', project ':x-pack:plugin:async-search:qa:rest', project ':x-pack:plugin:async-search:qa:security', project ':x-pack:plugin:autoscaling:qa:rest', project ':x-pack:plugin:ccr:qa:downgrade-to-basic-license', project ':x-pack:plugin:ccr:qa:multi-cluster', project ':x-pack:plugin:ccr:qa:non-compliant-license', project ':x-pack:plugin:ccr:qa:rest', project ':x-pack:plugin:ccr:qa:restart', project ':x-pack:plugin:ccr:qa:security', project ':x-pack:plugin:deprecation:qa:common', project ':x-pack:plugin:deprecation:qa:early-deprecation-rest', project ':x-pack:plugin:deprecation:qa:rest', project ':x-pack:plugin:downsample:qa:mixed-cluster', project ':x-pack:plugin:downsample:qa:rest', project ':x-pack:plugin:downsample:qa:with-security', project ':x-pack:plugin:enrich:qa:common', project ':x-pack:plugin:enrich:qa:rest', project ':x-pack:plugin:enrich:qa:rest-with-advanced-security', project ':x-pack:plugin:enrich:qa:rest-with-security', project ':x-pack:plugin:ent-search:qa:full-cluster-restart', project ':x-pack:plugin:ent-search:qa:rest', project ':x-pack:plugin:eql:qa:ccs-rolling-upgrade', project ':x-pack:plugin:eql:qa:common', project ':x-pack:plugin:eql:qa:correctness', project ':x-pack:plugin:eql:qa:mixed-node', project ':x-pack:plugin:eql:qa:multi-cluster-with-security', project ':x-pack:plugin:eql:qa:rest', project ':x-pack:plugin:eql:qa:security', project ':x-pack:plugin:esql:compute:ann', project ':x-pack:plugin:esql:compute:gen', project ':x-pack:plugin:esql:qa:action', project ':x-pack:plugin:esql:qa:security', project ':x-pack:plugin:esql:qa:server', project ':x-pack:plugin:esql:qa:testFixtures', project ':x-pack:plugin:fleet:qa:rest', project ':x-pack:plugin:graph:qa:with-security', project ':x-pack:plugin:identity-provider:qa:idp-rest-tests', project ':x-pack:plugin:ilm:qa:multi-cluster', project ':x-pack:plugin:ilm:qa:multi-node', project ':x-pack:plugin:ilm:qa:rest', project ':x-pack:plugin:ilm:qa:with-security', project ':x-pack:plugin:inference:qa:inference-service-tests', project ':x-pack:plugin:inference:qa:mixed-cluster', project ':x-pack:plugin:inference:qa:rolling-upgrade', project ':x-pack:plugin:inference:qa:test-service-plugin', project ':x-pack:plugin:logsdb:qa:with-basic', project ':x-pack:plugin:logsdb:qa:with-custom-cutoff', project ':x-pack:plugin:ml:qa:basic-multi-node', project ':x-pack:plugin:ml:qa:disabled', project ':x-pack:plugin:ml:qa:ml-inference-service-tests', project ':x-pack:plugin:ml:qa:ml-with-security', project ':x-pack:plugin:ml:qa:multi-cluster-tests-with-security', project ':x-pack:plugin:ml:qa:native-multi-node-tests', project ':x-pack:plugin:ml:qa:no-bootstrap-tests', project ':x-pack:plugin:ml:qa:single-node-tests', project ':x-pack:plugin:repositories-metering-api:qa:azure', project ':x-pack:plugin:repositories-metering-api:qa:gcs', project ':x-pack:plugin:repositories-metering-api:qa:s3', project ':x-pack:plugin:searchable-snapshots:qa:azure', project ':x-pack:plugin:searchable-snapshots:qa:gcs', project ':x-pack:plugin:searchable-snapshots:qa:hdfs', project ':x-pack:plugin:searchable-snapshots:qa:minio', project ':x-pack:plugin:searchable-snapshots:qa:rest', project ':x-pack:plugin:searchable-snapshots:qa:s3', project ':x-pack:plugin:searchable-snapshots:qa:url', project ':x-pack:plugin:security:lib:nimbus-jose-jwt-modified', project ':x-pack:plugin:security:lib:nimbus-jose-jwt-modified-part1', project ':x-pack:plugin:security:lib:nimbus-jose-jwt-modified-part2', project ':x-pack:plugin:security:qa:audit', project ':x-pack:plugin:security:qa:basic-enable-security', project ':x-pack:plugin:security:qa:consistency-checks', project ':x-pack:plugin:security:qa:jwt-realm', project ':x-pack:plugin:security:qa:multi-cluster', project ':x-pack:plugin:security:qa:operator-privileges-tests', project ':x-pack:plugin:security:qa:profile', project ':x-pack:plugin:security:qa:saml-rest-tests', project ':x-pack:plugin:security:qa:secondary-auth-actions', project ':x-pack:plugin:security:qa:security-basic', project ':x-pack:plugin:security:qa:security-disabled', project ':x-pack:plugin:security:qa:security-trial', project ':x-pack:plugin:security:qa:service-account', project ':x-pack:plugin:security:qa:smoke-test-all-realms', project ':x-pack:plugin:security:qa:tls-basic', project ':x-pack:plugin:shutdown:qa:full-cluster-restart', project ':x-pack:plugin:shutdown:qa:multi-node', project ':x-pack:plugin:shutdown:qa:rolling-upgrade', project ':x-pack:plugin:slm:qa:multi-node', project ':x-pack:plugin:slm:qa:rest', project ':x-pack:plugin:slm:qa:with-security', project ':x-pack:plugin:snapshot-based-recoveries:qa:azure', project ':x-pack:plugin:snapshot-based-recoveries:qa:fs', project ':x-pack:plugin:snapshot-based-recoveries:qa:gcs', project ':x-pack:plugin:snapshot-based-recoveries:qa:license-enforcing', project ':x-pack:plugin:snapshot-based-recoveries:qa:s3', project ':x-pack:plugin:snapshot-repo-test-kit:qa:azure', project ':x-pack:plugin:snapshot-repo-test-kit:qa:gcs', project ':x-pack:plugin:snapshot-repo-test-kit:qa:hdfs', project ':x-pack:plugin:snapshot-repo-test-kit:qa:minio', project ':x-pack:plugin:snapshot-repo-test-kit:qa:rest', project ':x-pack:plugin:snapshot-repo-test-kit:qa:s3', project ':x-pack:plugin:sql:qa:jdbc', project ':x-pack:plugin:sql:qa:mixed-node', project ':x-pack:plugin:sql:qa:server', project ':x-pack:plugin:stack:qa:rest', project ':x-pack:plugin:text-structure:qa:text-structure-with-security', project ':x-pack:plugin:transform:qa:common', project ':x-pack:plugin:transform:qa:multi-cluster-tests-with-security', project ':x-pack:plugin:transform:qa:multi-node-tests', project ':x-pack:plugin:transform:qa:single-node-tests', project ':x-pack:plugin:vector-tile:qa:multi-cluster', project ':x-pack:plugin:watcher:qa:common', project ':x-pack:plugin:watcher:qa:rest', project ':x-pack:plugin:watcher:qa:with-security', project ':x-pack:plugin:esql:qa:server:mixed-cluster', project ':x-pack:plugin:esql:qa:server:multi-clusters', project ':x-pack:plugin:esql:qa:server:multi-node', project ':x-pack:plugin:esql:qa:server:single-node', project ':x-pack:plugin:sql:qa:jdbc:multi-node', project ':x-pack:plugin:sql:qa:jdbc:no-sql', project ':x-pack:plugin:sql:qa:jdbc:security', project ':x-pack:plugin:sql:qa:jdbc:single-node', project ':x-pack:plugin:sql:qa:server:multi-cluster-with-security', project ':x-pack:plugin:sql:qa:server:multi-node', project ':x-pack:plugin:sql:qa:server:security', project ':x-pack:plugin:sql:qa:server:single-node', project ':x-pack:plugin:sql:qa:jdbc:security:with-ssl', project ':x-pack:plugin:sql:qa:jdbc:security:without-ssl', project ':x-pack:plugin:sql:qa:server:security:with-ssl', project ':x-pack:plugin:sql:qa:server:security:without-ssl'] + +> Configure project : +Evaluating root project 'elasticsearch' using build file '/Users/rene/dev/elastic/elasticsearch/build.gradle'. + +> Configure project :benchmarks +Evaluating project ':benchmarks' using build file '/Users/rene/dev/elastic/elasticsearch/benchmarks/build.gradle'. + +> Configure project :client +Evaluating project ':client' using build file '/Users/rene/dev/elastic/elasticsearch/client/build.gradle'. + +> Configure project :distribution +Evaluating project ':distribution' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/build.gradle'. + +> Configure project :docs +Evaluating project ':docs' using build file '/Users/rene/dev/elastic/elasticsearch/docs/build.gradle'. + +> Configure project :libs +Evaluating project ':libs' using build file '/Users/rene/dev/elastic/elasticsearch/libs/build.gradle'. + +> Configure project :modules +Evaluating project ':modules' using build file '/Users/rene/dev/elastic/elasticsearch/modules/build.gradle'. + +> Configure project :plugins +Evaluating project ':plugins' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/build.gradle'. + +> Configure project :qa +Evaluating project ':qa' using build file '/Users/rene/dev/elastic/elasticsearch/qa/build.gradle'. + +> Configure project :rest-api-spec +Evaluating project ':rest-api-spec' using build file '/Users/rene/dev/elastic/elasticsearch/rest-api-spec/build.gradle'. + +> Configure project :server +Evaluating project ':server' using build file '/Users/rene/dev/elastic/elasticsearch/server/build.gradle'. + +> Configure project :test +Evaluating project ':test' using build file '/Users/rene/dev/elastic/elasticsearch/test/build.gradle'. + +> Configure project :x-pack +Evaluating project ':x-pack' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/build.gradle'. + +> Configure project :client:benchmark +Evaluating project ':client:benchmark' using build file '/Users/rene/dev/elastic/elasticsearch/client/benchmark/build.gradle'. + +> Configure project :client:client-benchmark-noop-api-plugin +Evaluating project ':client:client-benchmark-noop-api-plugin' using build file '/Users/rene/dev/elastic/elasticsearch/client/client-benchmark-noop-api-plugin/build.gradle'. + +> Configure project :client:rest +Evaluating project ':client:rest' using build file '/Users/rene/dev/elastic/elasticsearch/client/rest/build.gradle'. + +> Configure project :client:sniffer +Evaluating project ':client:sniffer' using build file '/Users/rene/dev/elastic/elasticsearch/client/sniffer/build.gradle'. + +> Configure project :client:test +Evaluating project ':client:test' using build file '/Users/rene/dev/elastic/elasticsearch/client/test/build.gradle'. + +> Configure project :distribution:archives +Evaluating project ':distribution:archives' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/archives/build.gradle'. + +> Configure project :distribution:bwc +Evaluating project ':distribution:bwc' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/bwc/build.gradle'. + +> Configure project :distribution:docker +Evaluating project ':distribution:docker' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/docker/build.gradle'. + +> Configure project :distribution:packages +Evaluating project ':distribution:packages' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/packages/build.gradle'. +Transforming gradle-ospackage-plugin-11.10.0.jar (com.netflix.nebula:gradle-ospackage-plugin:11.10.0) with InstrumentationAnalysisTransform +Transforming commons-lang3-3.9.jar (org.apache.commons:commons-lang3:3.9) with InstrumentationAnalysisTransform +Transforming gradle-docker-plugin-3.2.1.jar (com.bmuschko:gradle-docker-plugin:3.2.1) with InstrumentationAnalysisTransform +Transforming redline-1.2.10.jar (org.redline-rpm:redline:1.2.10) with InstrumentationAnalysisTransform +Transforming jdeb-1.10.jar (org.vafer:jdeb:1.10) with InstrumentationAnalysisTransform +Transforming ant-1.10.12.jar (org.apache.ant:ant:1.10.12) with InstrumentationAnalysisTransform +Transforming slf4j-api-1.7.5.jar (org.slf4j:slf4j-api:1.7.5) with InstrumentationAnalysisTransform +Transforming maven-archiver-3.5.1.jar (org.apache.maven:maven-archiver:3.5.1) with InstrumentationAnalysisTransform +Transforming plexus-archiver-4.2.3.jar (org.codehaus.plexus:plexus-archiver:4.2.3) with InstrumentationAnalysisTransform +Transforming commons-compress-1.21.jar (org.apache.commons:commons-compress:1.21) with InstrumentationAnalysisTransform +Transforming xz-1.8.jar (org.tukaani:xz:1.8) with InstrumentationAnalysisTransform +Transforming bcpg-jdk15on-1.69.jar (org.bouncycastle:bcpg-jdk15on:1.69) with InstrumentationAnalysisTransform +Transforming bcprov-jdk15on-1.69.jar (org.bouncycastle:bcprov-jdk15on:1.69) with InstrumentationAnalysisTransform +Transforming ant-launcher-1.10.12.jar (org.apache.ant:ant-launcher:1.10.12) with InstrumentationAnalysisTransform +Transforming maven-model-3.1.1.jar (org.apache.maven:maven-model:3.1.1) with InstrumentationAnalysisTransform +Transforming maven-shared-utils-3.3.3.jar (org.apache.maven.shared:maven-shared-utils:3.3.3) with InstrumentationAnalysisTransform +Transforming plexus-io-3.2.0.jar (org.codehaus.plexus:plexus-io:3.2.0) with InstrumentationAnalysisTransform +Transforming commons-io-2.6.jar (commons-io:commons-io:2.6) with InstrumentationAnalysisTransform +Transforming plexus-interpolation-1.26.jar (org.codehaus.plexus:plexus-interpolation:1.26) with InstrumentationAnalysisTransform +Transforming snappy-0.4.jar (org.iq80.snappy:snappy:0.4) with InstrumentationAnalysisTransform +Transforming gradle-ospackage-plugin-11.10.0.jar (com.netflix.nebula:gradle-ospackage-plugin:11.10.0) with InstrumentationAnalysisTransform +Transforming gradle-ospackage-plugin-11.10.0.jar (com.netflix.nebula:gradle-ospackage-plugin:11.10.0) with MergeInstrumentationAnalysisTransform +Transforming commons-lang3-3.9.jar (org.apache.commons:commons-lang3:3.9) with InstrumentationAnalysisTransform +Transforming commons-lang3-3.9.jar (org.apache.commons:commons-lang3:3.9) with MergeInstrumentationAnalysisTransform +Transforming gradle-ospackage-plugin-11.10.0.jar (com.netflix.nebula:gradle-ospackage-plugin:11.10.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming gradle-docker-plugin-3.2.1.jar (com.bmuschko:gradle-docker-plugin:3.2.1) with InstrumentationAnalysisTransform +Transforming commons-lang3-3.9.jar (org.apache.commons:commons-lang3:3.9) with ExternalDependencyInstrumentingArtifactTransform +Transforming gradle-docker-plugin-3.2.1.jar (com.bmuschko:gradle-docker-plugin:3.2.1) with MergeInstrumentationAnalysisTransform +Transforming redline-1.2.10.jar (org.redline-rpm:redline:1.2.10) with InstrumentationAnalysisTransform +Transforming redline-1.2.10.jar (org.redline-rpm:redline:1.2.10) with MergeInstrumentationAnalysisTransform +Transforming gradle-docker-plugin-3.2.1.jar (com.bmuschko:gradle-docker-plugin:3.2.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming jdeb-1.10.jar (org.vafer:jdeb:1.10) with InstrumentationAnalysisTransform +Transforming jdeb-1.10.jar (org.vafer:jdeb:1.10) with MergeInstrumentationAnalysisTransform +Transforming redline-1.2.10.jar (org.redline-rpm:redline:1.2.10) with ExternalDependencyInstrumentingArtifactTransform +Transforming ant-1.10.12.jar (org.apache.ant:ant:1.10.12) with InstrumentationAnalysisTransform +Transforming jdeb-1.10.jar (org.vafer:jdeb:1.10) with ExternalDependencyInstrumentingArtifactTransform +Transforming ant-1.10.12.jar (org.apache.ant:ant:1.10.12) with MergeInstrumentationAnalysisTransform +Transforming slf4j-api-1.7.5.jar (org.slf4j:slf4j-api:1.7.5) with InstrumentationAnalysisTransform +Transforming slf4j-api-1.7.5.jar (org.slf4j:slf4j-api:1.7.5) with MergeInstrumentationAnalysisTransform +Transforming ant-1.10.12.jar (org.apache.ant:ant:1.10.12) with ExternalDependencyInstrumentingArtifactTransform +Transforming maven-archiver-3.5.1.jar (org.apache.maven:maven-archiver:3.5.1) with InstrumentationAnalysisTransform +Transforming slf4j-api-1.7.5.jar (org.slf4j:slf4j-api:1.7.5) with ExternalDependencyInstrumentingArtifactTransform +Transforming maven-archiver-3.5.1.jar (org.apache.maven:maven-archiver:3.5.1) with MergeInstrumentationAnalysisTransform +Transforming plexus-archiver-4.2.3.jar (org.codehaus.plexus:plexus-archiver:4.2.3) with InstrumentationAnalysisTransform +Transforming plexus-archiver-4.2.3.jar (org.codehaus.plexus:plexus-archiver:4.2.3) with MergeInstrumentationAnalysisTransform +Transforming maven-archiver-3.5.1.jar (org.apache.maven:maven-archiver:3.5.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming commons-compress-1.21.jar (org.apache.commons:commons-compress:1.21) with InstrumentationAnalysisTransform +Transforming commons-compress-1.21.jar (org.apache.commons:commons-compress:1.21) with MergeInstrumentationAnalysisTransform +Transforming plexus-archiver-4.2.3.jar (org.codehaus.plexus:plexus-archiver:4.2.3) with ExternalDependencyInstrumentingArtifactTransform +Transforming xz-1.8.jar (org.tukaani:xz:1.8) with InstrumentationAnalysisTransform +Transforming xz-1.8.jar (org.tukaani:xz:1.8) with MergeInstrumentationAnalysisTransform +Transforming commons-compress-1.21.jar (org.apache.commons:commons-compress:1.21) with ExternalDependencyInstrumentingArtifactTransform +Transforming bcpg-jdk15on-1.69.jar (org.bouncycastle:bcpg-jdk15on:1.69) with InstrumentationAnalysisTransform +Transforming bcpg-jdk15on-1.69.jar (org.bouncycastle:bcpg-jdk15on:1.69) with MergeInstrumentationAnalysisTransform +Transforming xz-1.8.jar (org.tukaani:xz:1.8) with ExternalDependencyInstrumentingArtifactTransform +Transforming bcprov-jdk15on-1.69.jar (org.bouncycastle:bcprov-jdk15on:1.69) with InstrumentationAnalysisTransform +Transforming bcprov-jdk15on-1.69.jar (org.bouncycastle:bcprov-jdk15on:1.69) with MergeInstrumentationAnalysisTransform +Transforming ant-launcher-1.10.12.jar (org.apache.ant:ant-launcher:1.10.12) with InstrumentationAnalysisTransform +Transforming bcpg-jdk15on-1.69.jar (org.bouncycastle:bcpg-jdk15on:1.69) with ExternalDependencyInstrumentingArtifactTransform +Transforming bcprov-jdk15on-1.69.jar (org.bouncycastle:bcprov-jdk15on:1.69) with ExternalDependencyInstrumentingArtifactTransform +Transforming ant-launcher-1.10.12.jar (org.apache.ant:ant-launcher:1.10.12) with MergeInstrumentationAnalysisTransform +Transforming maven-model-3.1.1.jar (org.apache.maven:maven-model:3.1.1) with InstrumentationAnalysisTransform +Transforming maven-model-3.1.1.jar (org.apache.maven:maven-model:3.1.1) with MergeInstrumentationAnalysisTransform +Transforming maven-shared-utils-3.3.3.jar (org.apache.maven.shared:maven-shared-utils:3.3.3) with InstrumentationAnalysisTransform +Transforming ant-launcher-1.10.12.jar (org.apache.ant:ant-launcher:1.10.12) with ExternalDependencyInstrumentingArtifactTransform +Transforming maven-model-3.1.1.jar (org.apache.maven:maven-model:3.1.1) with ExternalDependencyInstrumentingArtifactTransform +Transforming maven-shared-utils-3.3.3.jar (org.apache.maven.shared:maven-shared-utils:3.3.3) with MergeInstrumentationAnalysisTransform +Transforming plexus-io-3.2.0.jar (org.codehaus.plexus:plexus-io:3.2.0) with InstrumentationAnalysisTransform +Transforming plexus-io-3.2.0.jar (org.codehaus.plexus:plexus-io:3.2.0) with MergeInstrumentationAnalysisTransform +Transforming commons-io-2.6.jar (commons-io:commons-io:2.6) with InstrumentationAnalysisTransform +Transforming maven-shared-utils-3.3.3.jar (org.apache.maven.shared:maven-shared-utils:3.3.3) with ExternalDependencyInstrumentingArtifactTransform +Transforming plexus-io-3.2.0.jar (org.codehaus.plexus:plexus-io:3.2.0) with ExternalDependencyInstrumentingArtifactTransform +Transforming commons-io-2.6.jar (commons-io:commons-io:2.6) with MergeInstrumentationAnalysisTransform +Transforming plexus-interpolation-1.26.jar (org.codehaus.plexus:plexus-interpolation:1.26) with InstrumentationAnalysisTransform +Transforming plexus-interpolation-1.26.jar (org.codehaus.plexus:plexus-interpolation:1.26) with MergeInstrumentationAnalysisTransform +Transforming snappy-0.4.jar (org.iq80.snappy:snappy:0.4) with InstrumentationAnalysisTransform +Transforming commons-io-2.6.jar (commons-io:commons-io:2.6) with ExternalDependencyInstrumentingArtifactTransform +Transforming plexus-interpolation-1.26.jar (org.codehaus.plexus:plexus-interpolation:1.26) with ExternalDependencyInstrumentingArtifactTransform +Transforming snappy-0.4.jar (org.iq80.snappy:snappy:0.4) with MergeInstrumentationAnalysisTransform +Transforming snappy-0.4.jar (org.iq80.snappy:snappy:0.4) with ExternalDependencyInstrumentingArtifactTransform + +> Configure project :distribution:tools +Evaluating project ':distribution:tools' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/tools/build.gradle'. + +> Configure project :libs:cli +Evaluating project ':libs:cli' using build file '/Users/rene/dev/elastic/elasticsearch/libs/cli/build.gradle'. + +> Configure project :libs:core +Evaluating project ':libs:core' using build file '/Users/rene/dev/elastic/elasticsearch/libs/core/build.gradle'. + +> Configure project :libs:dissect +Evaluating project ':libs:dissect' using build file '/Users/rene/dev/elastic/elasticsearch/libs/dissect/build.gradle'. + +> Configure project :libs:entitlement +Evaluating project ':libs:entitlement' using build file '/Users/rene/dev/elastic/elasticsearch/libs/entitlement/build.gradle'. + +> Configure project :libs:geo +Evaluating project ':libs:geo' using build file '/Users/rene/dev/elastic/elasticsearch/libs/geo/build.gradle'. + +> Configure project :libs:grok +Evaluating project ':libs:grok' using build file '/Users/rene/dev/elastic/elasticsearch/libs/grok/build.gradle'. + +> Configure project :libs:h3 +Evaluating project ':libs:h3' using build file '/Users/rene/dev/elastic/elasticsearch/libs/h3/build.gradle'. + +> Configure project :libs:log4j +Evaluating project ':libs:log4j' using build file '/Users/rene/dev/elastic/elasticsearch/libs/log4j/build.gradle'. + +> Configure project :libs:logging +Evaluating project ':libs:logging' using build file '/Users/rene/dev/elastic/elasticsearch/libs/logging/build.gradle'. + +> Configure project :libs:logstash-bridge +Evaluating project ':libs:logstash-bridge' using build file '/Users/rene/dev/elastic/elasticsearch/libs/logstash-bridge/build.gradle'. + +> Configure project :libs:lz4 +Evaluating project ':libs:lz4' using build file '/Users/rene/dev/elastic/elasticsearch/libs/lz4/build.gradle'. + +> Configure project :libs:native +Evaluating project ':libs:native' using build file '/Users/rene/dev/elastic/elasticsearch/libs/native/build.gradle'. + +> Configure project :libs:plugin-analysis-api +Evaluating project ':libs:plugin-analysis-api' using build file '/Users/rene/dev/elastic/elasticsearch/libs/plugin-analysis-api/build.gradle'. + +> Configure project :libs:plugin-api +Evaluating project ':libs:plugin-api' using build file '/Users/rene/dev/elastic/elasticsearch/libs/plugin-api/build.gradle'. + +> Configure project :libs:plugin-scanner +Evaluating project ':libs:plugin-scanner' using build file '/Users/rene/dev/elastic/elasticsearch/libs/plugin-scanner/build.gradle'. + +> Configure project :libs:secure-sm +Evaluating project ':libs:secure-sm' using build file '/Users/rene/dev/elastic/elasticsearch/libs/secure-sm/build.gradle'. + +> Configure project :libs:simdvec +Evaluating project ':libs:simdvec' using build file '/Users/rene/dev/elastic/elasticsearch/libs/simdvec/build.gradle'. + +> Configure project :libs:ssl-config +Evaluating project ':libs:ssl-config' using build file '/Users/rene/dev/elastic/elasticsearch/libs/ssl-config/build.gradle'. + +> Configure project :libs:tdigest +Evaluating project ':libs:tdigest' using build file '/Users/rene/dev/elastic/elasticsearch/libs/tdigest/build.gradle'. + +> Configure project :libs:x-content +Evaluating project ':libs:x-content' using build file '/Users/rene/dev/elastic/elasticsearch/libs/x-content/build.gradle'. + +> Configure project :modules:aggregations +Evaluating project ':modules:aggregations' using build file '/Users/rene/dev/elastic/elasticsearch/modules/aggregations/build.gradle'. + +> Configure project :modules:analysis-common +Evaluating project ':modules:analysis-common' using build file '/Users/rene/dev/elastic/elasticsearch/modules/analysis-common/build.gradle'. + +> Configure project :modules:apm +Evaluating project ':modules:apm' using build file '/Users/rene/dev/elastic/elasticsearch/modules/apm/build.gradle'. + +> Configure project :modules:data-streams +Evaluating project ':modules:data-streams' using build file '/Users/rene/dev/elastic/elasticsearch/modules/data-streams/build.gradle'. + +> Configure project :modules:dot-prefix-validation +Evaluating project ':modules:dot-prefix-validation' using build file '/Users/rene/dev/elastic/elasticsearch/modules/dot-prefix-validation/build.gradle'. + +> Configure project :modules:health-shards-availability +Evaluating project ':modules:health-shards-availability' using build file '/Users/rene/dev/elastic/elasticsearch/modules/health-shards-availability/build.gradle'. + +> Configure project :modules:ingest-attachment +Evaluating project ':modules:ingest-attachment' using build file '/Users/rene/dev/elastic/elasticsearch/modules/ingest-attachment/build.gradle'. + +> Configure project :modules:ingest-common +Evaluating project ':modules:ingest-common' using build file '/Users/rene/dev/elastic/elasticsearch/modules/ingest-common/build.gradle'. + +> Configure project :modules:ingest-geoip +Evaluating project ':modules:ingest-geoip' using build file '/Users/rene/dev/elastic/elasticsearch/modules/ingest-geoip/build.gradle'. + +> Configure project :modules:ingest-user-agent +Evaluating project ':modules:ingest-user-agent' using build file '/Users/rene/dev/elastic/elasticsearch/modules/ingest-user-agent/build.gradle'. + +> Configure project :modules:kibana +Evaluating project ':modules:kibana' using build file '/Users/rene/dev/elastic/elasticsearch/modules/kibana/build.gradle'. + +> Configure project :modules:lang-expression +Evaluating project ':modules:lang-expression' using build file '/Users/rene/dev/elastic/elasticsearch/modules/lang-expression/build.gradle'. + +> Configure project :modules:lang-mustache +Evaluating project ':modules:lang-mustache' using build file '/Users/rene/dev/elastic/elasticsearch/modules/lang-mustache/build.gradle'. + +> Configure project :modules:lang-painless +Evaluating project ':modules:lang-painless' using build file '/Users/rene/dev/elastic/elasticsearch/modules/lang-painless/build.gradle'. + +> Configure project :modules:legacy-geo +Evaluating project ':modules:legacy-geo' using build file '/Users/rene/dev/elastic/elasticsearch/modules/legacy-geo/build.gradle'. + +> Configure project :modules:mapper-extras +Evaluating project ':modules:mapper-extras' using build file '/Users/rene/dev/elastic/elasticsearch/modules/mapper-extras/build.gradle'. + +> Configure project :modules:parent-join +Evaluating project ':modules:parent-join' using build file '/Users/rene/dev/elastic/elasticsearch/modules/parent-join/build.gradle'. + +> Configure project :modules:percolator +Evaluating project ':modules:percolator' using build file '/Users/rene/dev/elastic/elasticsearch/modules/percolator/build.gradle'. + +> Configure project :modules:rank-eval +Evaluating project ':modules:rank-eval' using build file '/Users/rene/dev/elastic/elasticsearch/modules/rank-eval/build.gradle'. + +> Configure project :modules:reindex +Evaluating project ':modules:reindex' using build file '/Users/rene/dev/elastic/elasticsearch/modules/reindex/build.gradle'. + +> Configure project :modules:repository-azure +Evaluating project ':modules:repository-azure' using build file '/Users/rene/dev/elastic/elasticsearch/modules/repository-azure/build.gradle'. + +> Configure project :modules:repository-gcs +Evaluating project ':modules:repository-gcs' using build file '/Users/rene/dev/elastic/elasticsearch/modules/repository-gcs/build.gradle'. + +> Configure project :modules:repository-s3 +Evaluating project ':modules:repository-s3' using build file '/Users/rene/dev/elastic/elasticsearch/modules/repository-s3/build.gradle'. + +> Configure project :modules:repository-url +Evaluating project ':modules:repository-url' using build file '/Users/rene/dev/elastic/elasticsearch/modules/repository-url/build.gradle'. + +> Configure project :modules:rest-root +Evaluating project ':modules:rest-root' using build file '/Users/rene/dev/elastic/elasticsearch/modules/rest-root/build.gradle'. + +> Configure project :modules:runtime-fields-common +Evaluating project ':modules:runtime-fields-common' using build file '/Users/rene/dev/elastic/elasticsearch/modules/runtime-fields-common/build.gradle'. + +> Configure project :modules:systemd +Evaluating project ':modules:systemd' using build file '/Users/rene/dev/elastic/elasticsearch/modules/systemd/build.gradle'. + +> Configure project :modules:transport-netty4 +Evaluating project ':modules:transport-netty4' using build file '/Users/rene/dev/elastic/elasticsearch/modules/transport-netty4/build.gradle'. + +> Configure project :plugins:analysis-icu +Evaluating project ':plugins:analysis-icu' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/analysis-icu/build.gradle'. + +> Configure project :plugins:analysis-kuromoji +Evaluating project ':plugins:analysis-kuromoji' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/analysis-kuromoji/build.gradle'. + +> Configure project :plugins:analysis-nori +Evaluating project ':plugins:analysis-nori' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/analysis-nori/build.gradle'. + +> Configure project :plugins:analysis-phonetic +Evaluating project ':plugins:analysis-phonetic' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/analysis-phonetic/build.gradle'. + +> Configure project :plugins:analysis-smartcn +Evaluating project ':plugins:analysis-smartcn' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/analysis-smartcn/build.gradle'. + +> Configure project :plugins:analysis-stempel +Evaluating project ':plugins:analysis-stempel' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/analysis-stempel/build.gradle'. + +> Configure project :plugins:analysis-ukrainian +Evaluating project ':plugins:analysis-ukrainian' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/analysis-ukrainian/build.gradle'. + +> Configure project :plugins:discovery-azure-classic +Evaluating project ':plugins:discovery-azure-classic' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/discovery-azure-classic/build.gradle'. + +> Configure project :plugins:discovery-ec2 +Evaluating project ':plugins:discovery-ec2' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/discovery-ec2/build.gradle'. + +> Configure project :plugins:discovery-gce +Evaluating project ':plugins:discovery-gce' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/discovery-gce/build.gradle'. + +> Configure project :plugins:mapper-annotated-text +Evaluating project ':plugins:mapper-annotated-text' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/mapper-annotated-text/build.gradle'. + +> Configure project :plugins:mapper-murmur3 +Evaluating project ':plugins:mapper-murmur3' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/mapper-murmur3/build.gradle'. + +> Configure project :plugins:mapper-size +Evaluating project ':plugins:mapper-size' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/mapper-size/build.gradle'. + +> Configure project :plugins:repository-hdfs +Evaluating project ':plugins:repository-hdfs' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/repository-hdfs/build.gradle'. + +> Configure project :plugins:store-smb +Evaluating project ':plugins:store-smb' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/store-smb/build.gradle'. + +> Configure project :qa:ccs-common-rest +Evaluating project ':qa:ccs-common-rest' using build file '/Users/rene/dev/elastic/elasticsearch/qa/ccs-common-rest/build.gradle'. + +> Configure project :qa:ccs-rolling-upgrade-remote-cluster +Evaluating project ':qa:ccs-rolling-upgrade-remote-cluster' using build file '/Users/rene/dev/elastic/elasticsearch/qa/ccs-rolling-upgrade-remote-cluster/build.gradle'. + +> Configure project :qa:ccs-unavailable-clusters +Evaluating project ':qa:ccs-unavailable-clusters' using build file '/Users/rene/dev/elastic/elasticsearch/qa/ccs-unavailable-clusters/build.gradle'. + +> Configure project :qa:custom-rest-controller +Evaluating project ':qa:custom-rest-controller' using build file '/Users/rene/dev/elastic/elasticsearch/qa/custom-rest-controller/build.gradle'. + +> Configure project :qa:evil-tests +Evaluating project ':qa:evil-tests' using build file '/Users/rene/dev/elastic/elasticsearch/qa/evil-tests/build.gradle'. + +> Configure project :qa:full-cluster-restart +Evaluating project ':qa:full-cluster-restart' using build file '/Users/rene/dev/elastic/elasticsearch/qa/full-cluster-restart/build.gradle'. + +> Configure project :qa:logging-config +Evaluating project ':qa:logging-config' using build file '/Users/rene/dev/elastic/elasticsearch/qa/logging-config/build.gradle'. + +> Configure project :qa:logging-spi +Evaluating project ':qa:logging-spi' using build file '/Users/rene/dev/elastic/elasticsearch/qa/logging-spi/build.gradle'. + +> Configure project :qa:lucene-index-compatibility +Evaluating project ':qa:lucene-index-compatibility' using build file '/Users/rene/dev/elastic/elasticsearch/qa/lucene-index-compatibility/build.gradle'. + +> Configure project :qa:mixed-cluster +Evaluating project ':qa:mixed-cluster' using build file '/Users/rene/dev/elastic/elasticsearch/qa/mixed-cluster/build.gradle'. + +> Configure project :qa:multi-cluster-search +Evaluating project ':qa:multi-cluster-search' using build file '/Users/rene/dev/elastic/elasticsearch/qa/multi-cluster-search/build.gradle'. + +> Configure project :qa:no-bootstrap-tests +Evaluating project ':qa:no-bootstrap-tests' using build file '/Users/rene/dev/elastic/elasticsearch/qa/no-bootstrap-tests/build.gradle'. + +> Configure project :qa:packaging +Evaluating project ':qa:packaging' using build file '/Users/rene/dev/elastic/elasticsearch/qa/packaging/build.gradle'. + +> Configure project :qa:remote-clusters +Evaluating project ':qa:remote-clusters' using build file '/Users/rene/dev/elastic/elasticsearch/qa/remote-clusters/build.gradle'. + +> Configure project :qa:repository-multi-version +Evaluating project ':qa:repository-multi-version' using build file '/Users/rene/dev/elastic/elasticsearch/qa/repository-multi-version/build.gradle'. + +> Configure project :qa:restricted-loggers +Evaluating project ':qa:restricted-loggers' using build file '/Users/rene/dev/elastic/elasticsearch/qa/restricted-loggers/build.gradle'. + +> Configure project :qa:rolling-upgrade +Evaluating project ':qa:rolling-upgrade' using build file '/Users/rene/dev/elastic/elasticsearch/qa/rolling-upgrade/build.gradle'. + +> Configure project :qa:rolling-upgrade-legacy +Evaluating project ':qa:rolling-upgrade-legacy' using build file '/Users/rene/dev/elastic/elasticsearch/qa/rolling-upgrade-legacy/build.gradle'. + +> Configure project :qa:smoke-test-http +Evaluating project ':qa:smoke-test-http' using build file '/Users/rene/dev/elastic/elasticsearch/qa/smoke-test-http/build.gradle'. + +> Configure project :qa:smoke-test-ingest-disabled +Evaluating project ':qa:smoke-test-ingest-disabled' using build file '/Users/rene/dev/elastic/elasticsearch/qa/smoke-test-ingest-disabled/build.gradle'. + +> Configure project :qa:smoke-test-ingest-with-all-dependencies +Evaluating project ':qa:smoke-test-ingest-with-all-dependencies' using build file '/Users/rene/dev/elastic/elasticsearch/qa/smoke-test-ingest-with-all-dependencies/build.gradle'. + +> Configure project :qa:smoke-test-multinode +Evaluating project ':qa:smoke-test-multinode' using build file '/Users/rene/dev/elastic/elasticsearch/qa/smoke-test-multinode/build.gradle'. + +> Configure project :qa:smoke-test-plugins +Evaluating project ':qa:smoke-test-plugins' using build file '/Users/rene/dev/elastic/elasticsearch/qa/smoke-test-plugins/build.gradle'. + +> Configure project :qa:stable-api +Evaluating project ':qa:stable-api' using build file '/Users/rene/dev/elastic/elasticsearch/qa/stable-api/build.gradle'. + +> Configure project :qa:system-indices +Evaluating project ':qa:system-indices' using build file '/Users/rene/dev/elastic/elasticsearch/qa/system-indices/build.gradle'. + +> Configure project :qa:unconfigured-node-name +Evaluating project ':qa:unconfigured-node-name' using build file '/Users/rene/dev/elastic/elasticsearch/qa/unconfigured-node-name/build.gradle'. + +> Configure project :qa:verify-version-constants +Evaluating project ':qa:verify-version-constants' using build file '/Users/rene/dev/elastic/elasticsearch/qa/verify-version-constants/build.gradle'. + +> Configure project :test:external-modules +Evaluating project ':test:external-modules' using build file '/Users/rene/dev/elastic/elasticsearch/test/external-modules/build.gradle'. + +> Configure project :test:fixtures +Evaluating project ':test:fixtures' using build file '/Users/rene/dev/elastic/elasticsearch/test/fixtures/build.gradle'. + +> Configure project :test:framework +Evaluating project ':test:framework' using build file '/Users/rene/dev/elastic/elasticsearch/test/framework/build.gradle'. + +> Configure project :test:immutable-collections-patch +Evaluating project ':test:immutable-collections-patch' using build file '/Users/rene/dev/elastic/elasticsearch/test/immutable-collections-patch/build.gradle'. + +> Configure project :test:logger-usage +Evaluating project ':test:logger-usage' using build file '/Users/rene/dev/elastic/elasticsearch/test/logger-usage/build.gradle'. + +> Configure project :test:metadata-extractor +Evaluating project ':test:metadata-extractor' using build file '/Users/rene/dev/elastic/elasticsearch/test/metadata-extractor/build.gradle'. + +> Configure project :test:test-clusters +Evaluating project ':test:test-clusters' using build file '/Users/rene/dev/elastic/elasticsearch/test/test-clusters/build.gradle'. + +> Configure project :test:x-content +Evaluating project ':test:x-content' using build file '/Users/rene/dev/elastic/elasticsearch/test/x-content/build.gradle'. + +> Configure project :test:yaml-rest-runner +Evaluating project ':test:yaml-rest-runner' using build file '/Users/rene/dev/elastic/elasticsearch/test/yaml-rest-runner/build.gradle'. + +> Configure project :x-pack:libs +Evaluating project ':x-pack:libs' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/libs/build.gradle'. + +> Configure project :x-pack:license-tools +Evaluating project ':x-pack:license-tools' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/license-tools/build.gradle'. + +> Configure project :x-pack:plugin +Evaluating project ':x-pack:plugin' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/build.gradle'. + +> Configure project :x-pack:qa +Evaluating project ':x-pack:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/build.gradle'. + +> Configure project :x-pack:rest-resources-zip +Evaluating project ':x-pack:rest-resources-zip' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/rest-resources-zip/build.gradle'. + +> Configure project :x-pack:test +Evaluating project ':x-pack:test' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/test/build.gradle'. + +> Configure project :distribution:archives:darwin-aarch64-tar +Evaluating project ':distribution:archives:darwin-aarch64-tar' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/archives/darwin-aarch64-tar/build.gradle'. + +> Configure project :distribution:archives:darwin-tar +Evaluating project ':distribution:archives:darwin-tar' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/archives/darwin-tar/build.gradle'. + +> Configure project :distribution:archives:integ-test-zip +Evaluating project ':distribution:archives:integ-test-zip' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/archives/integ-test-zip/build.gradle'. + +> Configure project :distribution:archives:linux-aarch64-tar +Evaluating project ':distribution:archives:linux-aarch64-tar' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/archives/linux-aarch64-tar/build.gradle'. + +> Configure project :distribution:archives:linux-tar +Evaluating project ':distribution:archives:linux-tar' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/archives/linux-tar/build.gradle'. + +> Configure project :distribution:archives:windows-zip +Evaluating project ':distribution:archives:windows-zip' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/archives/windows-zip/build.gradle'. + +> Configure project :distribution:bwc:bugfix +Evaluating project ':distribution:bwc:bugfix' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/bwc/bugfix/build.gradle'. + +> Configure project :distribution:bwc:bugfix2 +Evaluating project ':distribution:bwc:bugfix2' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/bwc/bugfix2/build.gradle'. + +> Configure project :distribution:bwc:maintenance +Evaluating project ':distribution:bwc:maintenance' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/bwc/maintenance/build.gradle'. + +> Configure project :distribution:bwc:minor +Evaluating project ':distribution:bwc:minor' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/bwc/minor/build.gradle'. + +> Configure project :distribution:bwc:staged +Evaluating project ':distribution:bwc:staged' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/bwc/staged/build.gradle'. + +> Configure project :distribution:docker:cloud-ess-docker-aarch64-export +Evaluating project ':distribution:docker:cloud-ess-docker-aarch64-export' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/docker/cloud-ess-docker-aarch64-export/build.gradle'. + +> Configure project :distribution:docker:cloud-ess-docker-export +Evaluating project ':distribution:docker:cloud-ess-docker-export' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/docker/cloud-ess-docker-export/build.gradle'. + +> Configure project :distribution:docker:docker-aarch64-export +Evaluating project ':distribution:docker:docker-aarch64-export' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/docker/docker-aarch64-export/build.gradle'. + +> Configure project :distribution:docker:docker-export +Evaluating project ':distribution:docker:docker-export' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/docker/docker-export/build.gradle'. + +> Configure project :distribution:docker:ironbank-docker-aarch64-export +Evaluating project ':distribution:docker:ironbank-docker-aarch64-export' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/docker/ironbank-docker-aarch64-export/build.gradle'. + +> Configure project :distribution:docker:ironbank-docker-export +Evaluating project ':distribution:docker:ironbank-docker-export' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/docker/ironbank-docker-export/build.gradle'. + +> Configure project :distribution:docker:wolfi-docker-aarch64-export +Evaluating project ':distribution:docker:wolfi-docker-aarch64-export' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/docker/wolfi-docker-aarch64-export/build.gradle'. + +> Configure project :distribution:docker:wolfi-docker-export +Evaluating project ':distribution:docker:wolfi-docker-export' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/docker/wolfi-docker-export/build.gradle'. + +> Configure project :distribution:packages:aarch64-deb +Evaluating project ':distribution:packages:aarch64-deb' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/packages/aarch64-deb/build.gradle'. + +> Configure project :distribution:packages:aarch64-rpm +Evaluating project ':distribution:packages:aarch64-rpm' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/packages/aarch64-rpm/build.gradle'. + +> Configure project :distribution:packages:deb +Evaluating project ':distribution:packages:deb' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/packages/deb/build.gradle'. + +> Configure project :distribution:packages:rpm +Evaluating project ':distribution:packages:rpm' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/packages/rpm/build.gradle'. + +> Configure project :distribution:tools:ansi-console +Evaluating project ':distribution:tools:ansi-console' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/tools/ansi-console/build.gradle'. + +> Configure project :distribution:tools:cli-launcher +Evaluating project ':distribution:tools:cli-launcher' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/tools/cli-launcher/build.gradle'. + +> Configure project :distribution:tools:geoip-cli +Evaluating project ':distribution:tools:geoip-cli' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/tools/geoip-cli/build.gradle'. + +> Configure project :distribution:tools:java-version-checker +Evaluating project ':distribution:tools:java-version-checker' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/tools/java-version-checker/build.gradle'. + +> Configure project :distribution:tools:keystore-cli +Evaluating project ':distribution:tools:keystore-cli' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/tools/keystore-cli/build.gradle'. + +> Configure project :distribution:tools:plugin-cli +Evaluating project ':distribution:tools:plugin-cli' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/tools/plugin-cli/build.gradle'. + +> Configure project :distribution:tools:server-cli +Evaluating project ':distribution:tools:server-cli' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/tools/server-cli/build.gradle'. + +> Configure project :distribution:tools:windows-service-cli +Evaluating project ':distribution:tools:windows-service-cli' using build file '/Users/rene/dev/elastic/elasticsearch/distribution/tools/windows-service-cli/build.gradle'. + +> Configure project :libs:entitlement:agent +Evaluating project ':libs:entitlement:agent' using build file '/Users/rene/dev/elastic/elasticsearch/libs/entitlement/agent/build.gradle'. + +> Configure project :libs:entitlement:asm-provider +Evaluating project ':libs:entitlement:asm-provider' using build file '/Users/rene/dev/elastic/elasticsearch/libs/entitlement/asm-provider/build.gradle'. + +> Configure project :libs:entitlement:bridge +Evaluating project ':libs:entitlement:bridge' using build file '/Users/rene/dev/elastic/elasticsearch/libs/entitlement/bridge/build.gradle'. + +> Configure project :libs:entitlement:qa +Evaluating project ':libs:entitlement:qa' using build file '/Users/rene/dev/elastic/elasticsearch/libs/entitlement/qa/build.gradle'. + +> Configure project :libs:entitlement:tools +Evaluating project ':libs:entitlement:tools' using build file '/Users/rene/dev/elastic/elasticsearch/libs/entitlement/tools/build.gradle'. + +> Configure project :libs:native:native-libraries +Evaluating project ':libs:native:native-libraries' using build file '/Users/rene/dev/elastic/elasticsearch/libs/native/libraries/build.gradle'. + +> Configure project :libs:x-content:impl +Evaluating project ':libs:x-content:impl' using build file '/Users/rene/dev/elastic/elasticsearch/libs/x-content/impl/build.gradle'. + +> Configure project :modules:ingest-geoip:qa +Evaluating project ':modules:ingest-geoip:qa' using build file '/Users/rene/dev/elastic/elasticsearch/modules/ingest-geoip/qa/build.gradle'. + +> Configure project :modules:lang-painless:spi +Evaluating project ':modules:lang-painless:spi' using build file '/Users/rene/dev/elastic/elasticsearch/modules/lang-painless/spi/build.gradle'. + +> Configure project :plugins:discovery-ec2:qa +Evaluating project ':plugins:discovery-ec2:qa' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/discovery-ec2/qa/build.gradle'. + +> Configure project :plugins:discovery-gce:qa +Evaluating project ':plugins:discovery-gce:qa' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/discovery-gce/qa/build.gradle'. + +> Configure project :plugins:repository-hdfs:hadoop-client-api +Evaluating project ':plugins:repository-hdfs:hadoop-client-api' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/repository-hdfs/hadoop-client-api/build.gradle'. + +> Configure project :qa:stable-api:logging +Evaluating project ':qa:stable-api:logging' using build file '/Users/rene/dev/elastic/elasticsearch/qa/stable-api/logging/build.gradle'. + +> Configure project :qa:stable-api:plugin-analysis-api +Evaluating project ':qa:stable-api:plugin-analysis-api' using build file '/Users/rene/dev/elastic/elasticsearch/qa/stable-api/plugin-analysis-api/build.gradle'. + +> Configure project :qa:stable-api:plugin-api +Evaluating project ':qa:stable-api:plugin-api' using build file '/Users/rene/dev/elastic/elasticsearch/qa/stable-api/plugin-api/build.gradle'. + +> Configure project :test:external-modules:test-apm-integration +Evaluating project ':test:external-modules:test-apm-integration' using build file '/Users/rene/dev/elastic/elasticsearch/test/external-modules/apm-integration/build.gradle'. + +> Configure project :test:external-modules:test-delayed-aggs +Evaluating project ':test:external-modules:test-delayed-aggs' using build file '/Users/rene/dev/elastic/elasticsearch/test/external-modules/delayed-aggs/build.gradle'. + +> Configure project :test:external-modules:test-die-with-dignity +Evaluating project ':test:external-modules:test-die-with-dignity' using build file '/Users/rene/dev/elastic/elasticsearch/test/external-modules/die-with-dignity/build.gradle'. + +> Configure project :test:external-modules:test-error-query +Evaluating project ':test:external-modules:test-error-query' using build file '/Users/rene/dev/elastic/elasticsearch/test/external-modules/error-query/build.gradle'. + +> Configure project :test:external-modules:test-esql-heap-attack +Evaluating project ':test:external-modules:test-esql-heap-attack' using build file '/Users/rene/dev/elastic/elasticsearch/test/external-modules/esql-heap-attack/build.gradle'. + +> Configure project :test:external-modules:test-jvm-crash +Evaluating project ':test:external-modules:test-jvm-crash' using build file '/Users/rene/dev/elastic/elasticsearch/test/external-modules/jvm-crash/build.gradle'. + +> Configure project :test:external-modules:test-latency-simulating-directory +Evaluating project ':test:external-modules:test-latency-simulating-directory' using build file '/Users/rene/dev/elastic/elasticsearch/test/external-modules/latency-simulating-directory/build.gradle'. + +> Configure project :test:fixtures:aws-sts-fixture +Evaluating project ':test:fixtures:aws-sts-fixture' using build file '/Users/rene/dev/elastic/elasticsearch/test/fixtures/aws-sts-fixture/build.gradle'. + +> Configure project :test:fixtures:azure-fixture +Evaluating project ':test:fixtures:azure-fixture' using build file '/Users/rene/dev/elastic/elasticsearch/test/fixtures/azure-fixture/build.gradle'. + +> Configure project :test:fixtures:ec2-imds-fixture +Evaluating project ':test:fixtures:ec2-imds-fixture' using build file '/Users/rene/dev/elastic/elasticsearch/test/fixtures/ec2-imds-fixture/build.gradle'. + +> Configure project :test:fixtures:gcs-fixture +Evaluating project ':test:fixtures:gcs-fixture' using build file '/Users/rene/dev/elastic/elasticsearch/test/fixtures/gcs-fixture/build.gradle'. + +> Configure project :test:fixtures:geoip-fixture +Evaluating project ':test:fixtures:geoip-fixture' using build file '/Users/rene/dev/elastic/elasticsearch/test/fixtures/geoip-fixture/build.gradle'. + +> Configure project :test:fixtures:hdfs-fixture +Evaluating project ':test:fixtures:hdfs-fixture' using build file '/Users/rene/dev/elastic/elasticsearch/test/fixtures/hdfs-fixture/build.gradle'. + +> Configure project :test:fixtures:krb5kdc-fixture +Evaluating project ':test:fixtures:krb5kdc-fixture' using build file '/Users/rene/dev/elastic/elasticsearch/test/fixtures/krb5kdc-fixture/build.gradle'. + +> Configure project :test:fixtures:minio-fixture +Evaluating project ':test:fixtures:minio-fixture' using build file '/Users/rene/dev/elastic/elasticsearch/test/fixtures/minio-fixture/build.gradle'. + +> Configure project :test:fixtures:old-elasticsearch +Evaluating project ':test:fixtures:old-elasticsearch' using build file '/Users/rene/dev/elastic/elasticsearch/test/fixtures/old-elasticsearch/build.gradle'. + +> Configure project :test:fixtures:s3-fixture +Evaluating project ':test:fixtures:s3-fixture' using build file '/Users/rene/dev/elastic/elasticsearch/test/fixtures/s3-fixture/build.gradle'. + +> Configure project :test:fixtures:testcontainer-utils +Evaluating project ':test:fixtures:testcontainer-utils' using build file '/Users/rene/dev/elastic/elasticsearch/test/fixtures/testcontainer-utils/build.gradle'. + +> Configure project :test:fixtures:url-fixture +Evaluating project ':test:fixtures:url-fixture' using build file '/Users/rene/dev/elastic/elasticsearch/test/fixtures/url-fixture/build.gradle'. + +> Configure project :x-pack:libs:es-opensaml-security-api +Evaluating project ':x-pack:libs:es-opensaml-security-api' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/libs/es-opensaml-security-api/build.gradle'. + +> Configure project :x-pack:plugin:analytics +Evaluating project ':x-pack:plugin:analytics' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/analytics/build.gradle'. + +> Configure project :x-pack:plugin:apm-data +Evaluating project ':x-pack:plugin:apm-data' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/apm-data/build.gradle'. + +> Configure project :x-pack:plugin:async +Evaluating project ':x-pack:plugin:async' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/async/build.gradle'. + +> Configure project :x-pack:plugin:async-search +Evaluating project ':x-pack:plugin:async-search' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/async-search/build.gradle'. + +> Configure project :x-pack:plugin:autoscaling +Evaluating project ':x-pack:plugin:autoscaling' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/autoscaling/build.gradle'. + +> Configure project :x-pack:plugin:blob-cache +Evaluating project ':x-pack:plugin:blob-cache' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/blob-cache/build.gradle'. + +> Configure project :x-pack:plugin:ccr +Evaluating project ':x-pack:plugin:ccr' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ccr/build.gradle'. + +> Configure project :x-pack:plugin:core +Evaluating project ':x-pack:plugin:core' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/core/build.gradle'. + +> Configure project :x-pack:plugin:deprecation +Evaluating project ':x-pack:plugin:deprecation' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/deprecation/build.gradle'. + +> Configure project :x-pack:plugin:downsample +Evaluating project ':x-pack:plugin:downsample' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/downsample/build.gradle'. + +> Configure project :x-pack:plugin:enrich +Evaluating project ':x-pack:plugin:enrich' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/enrich/build.gradle'. + +> Configure project :x-pack:plugin:ent-search +Evaluating project ':x-pack:plugin:ent-search' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ent-search/build.gradle'. + +> Configure project :x-pack:plugin:eql +Evaluating project ':x-pack:plugin:eql' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/eql/build.gradle'. + +> Configure project :x-pack:plugin:esql +Evaluating project ':x-pack:plugin:esql' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql/build.gradle'. + +> Configure project :x-pack:plugin:esql-core +Evaluating project ':x-pack:plugin:esql-core' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql-core/build.gradle'. + +> Configure project :x-pack:plugin:fleet +Evaluating project ':x-pack:plugin:fleet' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/fleet/build.gradle'. + +> Configure project :x-pack:plugin:frozen-indices +Evaluating project ':x-pack:plugin:frozen-indices' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/frozen-indices/build.gradle'. + +> Configure project :x-pack:plugin:geoip-enterprise-downloader +Evaluating project ':x-pack:plugin:geoip-enterprise-downloader' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/geoip-enterprise-downloader/build.gradle'. + +> Configure project :x-pack:plugin:graph +Evaluating project ':x-pack:plugin:graph' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/graph/build.gradle'. + +> Configure project :x-pack:plugin:identity-provider +Evaluating project ':x-pack:plugin:identity-provider' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/identity-provider/build.gradle'. + +> Configure project :x-pack:plugin:ilm +Evaluating project ':x-pack:plugin:ilm' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ilm/build.gradle'. + +> Configure project :x-pack:plugin:inference +Evaluating project ':x-pack:plugin:inference' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/inference/build.gradle'. + +> Configure project :x-pack:plugin:kql +Evaluating project ':x-pack:plugin:kql' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/kql/build.gradle'. + +> Configure project :x-pack:plugin:logsdb +Evaluating project ':x-pack:plugin:logsdb' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/logsdb/build.gradle'. + +> Configure project :x-pack:plugin:logstash +Evaluating project ':x-pack:plugin:logstash' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/logstash/build.gradle'. + +> Configure project :x-pack:plugin:mapper-aggregate-metric +Evaluating project ':x-pack:plugin:mapper-aggregate-metric' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/mapper-aggregate-metric/build.gradle'. + +> Configure project :x-pack:plugin:mapper-constant-keyword +Evaluating project ':x-pack:plugin:mapper-constant-keyword' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/mapper-constant-keyword/build.gradle'. + +> Configure project :x-pack:plugin:mapper-counted-keyword +Evaluating project ':x-pack:plugin:mapper-counted-keyword' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/mapper-counted-keyword/build.gradle'. + +> Configure project :x-pack:plugin:mapper-unsigned-long +Evaluating project ':x-pack:plugin:mapper-unsigned-long' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/mapper-unsigned-long/build.gradle'. + +> Configure project :x-pack:plugin:mapper-version +Evaluating project ':x-pack:plugin:mapper-version' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/mapper-version/build.gradle'. + +> Configure project :x-pack:plugin:migrate +Evaluating project ':x-pack:plugin:migrate' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/migrate/build.gradle'. + +> Configure project :x-pack:plugin:ml +Evaluating project ':x-pack:plugin:ml' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ml/build.gradle'. + +> Configure project :x-pack:plugin:ml-package-loader +Evaluating project ':x-pack:plugin:ml-package-loader' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ml-package-loader/build.gradle'. + +> Configure project :x-pack:plugin:monitoring +Evaluating project ':x-pack:plugin:monitoring' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/monitoring/build.gradle'. + +> Configure project :x-pack:plugin:old-lucene-versions +Evaluating project ':x-pack:plugin:old-lucene-versions' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/old-lucene-versions/build.gradle'. + +> Configure project :x-pack:plugin:otel-data +Evaluating project ':x-pack:plugin:otel-data' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/otel-data/build.gradle'. + +> Configure project :x-pack:plugin:profiling +Evaluating project ':x-pack:plugin:profiling' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/profiling/build.gradle'. + +> Configure project :x-pack:plugin:ql +Evaluating project ':x-pack:plugin:ql' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ql/build.gradle'. + +> Configure project :x-pack:plugin:rank-rrf +Evaluating project ':x-pack:plugin:rank-rrf' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/rank-rrf/build.gradle'. + +> Configure project :x-pack:plugin:redact +Evaluating project ':x-pack:plugin:redact' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/redact/build.gradle'. + +> Configure project :x-pack:plugin:repositories-metering-api +Evaluating project ':x-pack:plugin:repositories-metering-api' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/repositories-metering-api/build.gradle'. + +> Configure project :x-pack:plugin:rollup +Evaluating project ':x-pack:plugin:rollup' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/rollup/build.gradle'. + +> Configure project :x-pack:plugin:search-business-rules +Evaluating project ':x-pack:plugin:search-business-rules' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/search-business-rules/build.gradle'. + +> Configure project :x-pack:plugin:searchable-snapshots +Evaluating project ':x-pack:plugin:searchable-snapshots' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/searchable-snapshots/build.gradle'. + +> Configure project :x-pack:plugin:security +Evaluating project ':x-pack:plugin:security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/build.gradle'. + +> Configure project :x-pack:plugin:shutdown +Evaluating project ':x-pack:plugin:shutdown' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/shutdown/build.gradle'. + +> Configure project :x-pack:plugin:slm +Evaluating project ':x-pack:plugin:slm' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/slm/build.gradle'. + +> Configure project :x-pack:plugin:snapshot-based-recoveries +Evaluating project ':x-pack:plugin:snapshot-based-recoveries' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-based-recoveries/build.gradle'. + +> Configure project :x-pack:plugin:snapshot-repo-test-kit +Evaluating project ':x-pack:plugin:snapshot-repo-test-kit' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-repo-test-kit/build.gradle'. + +> Configure project :x-pack:plugin:spatial +Evaluating project ':x-pack:plugin:spatial' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/spatial/build.gradle'. + +> Configure project :x-pack:plugin:sql +Evaluating project ':x-pack:plugin:sql' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/build.gradle'. + +> Configure project :x-pack:plugin:stack +Evaluating project ':x-pack:plugin:stack' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/stack/build.gradle'. + +> Configure project :x-pack:plugin:text-structure +Evaluating project ':x-pack:plugin:text-structure' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/text-structure/build.gradle'. + +> Configure project :x-pack:plugin:transform +Evaluating project ':x-pack:plugin:transform' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/transform/build.gradle'. + +> Configure project :x-pack:plugin:vector-tile +Evaluating project ':x-pack:plugin:vector-tile' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/vector-tile/build.gradle'. + +> Configure project :x-pack:plugin:voting-only-node +Evaluating project ':x-pack:plugin:voting-only-node' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/voting-only-node/build.gradle'. + +> Configure project :x-pack:plugin:watcher +Evaluating project ':x-pack:plugin:watcher' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/watcher/build.gradle'. + +> Configure project :x-pack:plugin:wildcard +Evaluating project ':x-pack:plugin:wildcard' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/wildcard/build.gradle'. + +> Configure project :x-pack:plugin:write-load-forecaster +Evaluating project ':x-pack:plugin:write-load-forecaster' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/write-load-forecaster/build.gradle'. + +> Configure project :x-pack:qa:core-rest-tests-with-security +Evaluating project ':x-pack:qa:core-rest-tests-with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/core-rest-tests-with-security/build.gradle'. + +> Configure project :x-pack:qa:evil-tests +Evaluating project ':x-pack:qa:evil-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/evil-tests/build.gradle'. + +> Configure project :x-pack:qa:freeze-plugin +Evaluating project ':x-pack:qa:freeze-plugin' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/freeze-plugin/build.gradle'. + +> Configure project :x-pack:qa:full-cluster-restart +Evaluating project ':x-pack:qa:full-cluster-restart' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/full-cluster-restart/build.gradle'. + +> Configure project :x-pack:qa:kerberos-tests +Evaluating project ':x-pack:qa:kerberos-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/kerberos-tests/build.gradle'. + +> Configure project :x-pack:qa:mixed-tier-cluster +Evaluating project ':x-pack:qa:mixed-tier-cluster' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/mixed-tier-cluster/build.gradle'. + +> Configure project :x-pack:qa:multi-cluster-search-security +Evaluating project ':x-pack:qa:multi-cluster-search-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/multi-cluster-search-security/build.gradle'. + +> Configure project :x-pack:qa:multi-node +Evaluating project ':x-pack:qa:multi-node' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/multi-node/build.gradle'. + +> Configure project :x-pack:qa:oidc-op-tests +Evaluating project ':x-pack:qa:oidc-op-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/oidc-op-tests/build.gradle'. + +> Configure project :x-pack:qa:openldap-tests +Evaluating project ':x-pack:qa:openldap-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/openldap-tests/build.gradle'. + +> Configure project :x-pack:qa:password-protected-keystore +Evaluating project ':x-pack:qa:password-protected-keystore' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/password-protected-keystore/build.gradle'. + +> Configure project :x-pack:qa:reindex-tests-with-security +Evaluating project ':x-pack:qa:reindex-tests-with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/reindex-tests-with-security/build.gradle'. + +> Configure project :x-pack:qa:repository-old-versions +Evaluating project ':x-pack:qa:repository-old-versions' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/repository-old-versions/build.gradle'. + +> Configure project :x-pack:qa:rolling-upgrade +Evaluating project ':x-pack:qa:rolling-upgrade' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/rolling-upgrade/build.gradle'. + +> Configure project :x-pack:qa:rolling-upgrade-basic +Evaluating project ':x-pack:qa:rolling-upgrade-basic' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/rolling-upgrade-basic/build.gradle'. + +> Configure project :x-pack:qa:rolling-upgrade-multi-cluster +Evaluating project ':x-pack:qa:rolling-upgrade-multi-cluster' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle'. + +> Configure project :x-pack:qa:runtime-fields +Evaluating project ':x-pack:qa:runtime-fields' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/runtime-fields/build.gradle'. + +> Configure project :x-pack:qa:saml-idp-tests +Evaluating project ':x-pack:qa:saml-idp-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/saml-idp-tests/build.gradle'. + +> Configure project :x-pack:qa:security-example-spi-extension +Evaluating project ':x-pack:qa:security-example-spi-extension' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/security-example-spi-extension/build.gradle'. + +> Configure project :x-pack:qa:security-setup-password-tests +Evaluating project ':x-pack:qa:security-setup-password-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/security-setup-password-tests/build.gradle'. + +> Configure project :x-pack:qa:security-tools-tests +Evaluating project ':x-pack:qa:security-tools-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/security-tools-tests/build.gradle'. + +> Configure project :x-pack:qa:smoke-test-plugins +Evaluating project ':x-pack:qa:smoke-test-plugins' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/smoke-test-plugins/build.gradle'. + +> Configure project :x-pack:qa:smoke-test-plugins-ssl +Evaluating project ':x-pack:qa:smoke-test-plugins-ssl' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/smoke-test-plugins-ssl/build.gradle'. + +> Configure project :x-pack:qa:smoke-test-security-with-mustache +Evaluating project ':x-pack:qa:smoke-test-security-with-mustache' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/smoke-test-security-with-mustache/build.gradle'. + +> Configure project :x-pack:qa:third-party +Evaluating project ':x-pack:qa:third-party' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/third-party/build.gradle'. + +> Configure project :x-pack:test:idp-fixture +Evaluating project ':x-pack:test:idp-fixture' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/test/idp-fixture/build.gradle'. + +> Configure project :x-pack:test:smb-fixture +Evaluating project ':x-pack:test:smb-fixture' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/test/smb-fixture/build.gradle'. + +> Configure project :libs:entitlement:qa:common +Evaluating project ':libs:entitlement:qa:common' using build file '/Users/rene/dev/elastic/elasticsearch/libs/entitlement/qa/common/build.gradle'. + +> Configure project :libs:entitlement:qa:entitlement-allowed +Evaluating project ':libs:entitlement:qa:entitlement-allowed' using build file '/Users/rene/dev/elastic/elasticsearch/libs/entitlement/qa/entitlement-allowed/build.gradle'. + +> Configure project :libs:entitlement:qa:entitlement-allowed-nonmodular +Evaluating project ':libs:entitlement:qa:entitlement-allowed-nonmodular' using build file '/Users/rene/dev/elastic/elasticsearch/libs/entitlement/qa/entitlement-allowed-nonmodular/build.gradle'. + +> Configure project :libs:entitlement:qa:entitlement-denied +Evaluating project ':libs:entitlement:qa:entitlement-denied' using build file '/Users/rene/dev/elastic/elasticsearch/libs/entitlement/qa/entitlement-denied/build.gradle'. + +> Configure project :libs:entitlement:qa:entitlement-denied-nonmodular +Evaluating project ':libs:entitlement:qa:entitlement-denied-nonmodular' using build file '/Users/rene/dev/elastic/elasticsearch/libs/entitlement/qa/entitlement-denied-nonmodular/build.gradle'. + +> Configure project :libs:entitlement:tools:common +Evaluating project ':libs:entitlement:tools:common' using build file '/Users/rene/dev/elastic/elasticsearch/libs/entitlement/tools/common/build.gradle'. + +> Configure project :libs:entitlement:tools:public-callers-finder +Evaluating project ':libs:entitlement:tools:public-callers-finder' using build file '/Users/rene/dev/elastic/elasticsearch/libs/entitlement/tools/public-callers-finder/build.gradle'. + +> Configure project :libs:entitlement:tools:securitymanager-scanner +Evaluating project ':libs:entitlement:tools:securitymanager-scanner' using build file '/Users/rene/dev/elastic/elasticsearch/libs/entitlement/tools/securitymanager-scanner/build.gradle'. + +> Configure project :modules:ingest-geoip:qa:file-based-update +Evaluating project ':modules:ingest-geoip:qa:file-based-update' using build file '/Users/rene/dev/elastic/elasticsearch/modules/ingest-geoip/qa/file-based-update/build.gradle'. + +> Configure project :modules:ingest-geoip:qa:full-cluster-restart +Evaluating project ':modules:ingest-geoip:qa:full-cluster-restart' using build file '/Users/rene/dev/elastic/elasticsearch/modules/ingest-geoip/qa/full-cluster-restart/build.gradle'. + +> Configure project :plugins:discovery-ec2:qa:amazon-ec2 +Evaluating project ':plugins:discovery-ec2:qa:amazon-ec2' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/discovery-ec2/qa/amazon-ec2/build.gradle'. + +> Configure project :plugins:discovery-gce:qa:gce +Evaluating project ':plugins:discovery-gce:qa:gce' using build file '/Users/rene/dev/elastic/elasticsearch/plugins/discovery-gce/qa/gce/build.gradle'. + +> Configure project :x-pack:plugin:async-search:qa +Evaluating project ':x-pack:plugin:async-search:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/async-search/qa/build.gradle'. + +> Configure project :x-pack:plugin:autoscaling:qa +Evaluating project ':x-pack:plugin:autoscaling:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/autoscaling/qa/build.gradle'. + +> Configure project :x-pack:plugin:ccr:qa +Evaluating project ':x-pack:plugin:ccr:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ccr/qa/build.gradle'. + +> Configure project :x-pack:plugin:core:template-resources +Evaluating project ':x-pack:plugin:core:template-resources' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/core/template-resources/build.gradle'. + +> Configure project :x-pack:plugin:deprecation:qa +Evaluating project ':x-pack:plugin:deprecation:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/deprecation/qa/build.gradle'. + +> Configure project :x-pack:plugin:downsample:qa +Evaluating project ':x-pack:plugin:downsample:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/downsample/qa/build.gradle'. + +> Configure project :x-pack:plugin:enrich:qa +Evaluating project ':x-pack:plugin:enrich:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/enrich/qa/build.gradle'. + +> Configure project :x-pack:plugin:ent-search:qa +Evaluating project ':x-pack:plugin:ent-search:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ent-search/qa/build.gradle'. + +> Configure project :x-pack:plugin:eql:qa +Evaluating project ':x-pack:plugin:eql:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/eql/qa/build.gradle'. + +> Configure project :x-pack:plugin:esql:arrow +Evaluating project ':x-pack:plugin:esql:arrow' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql/arrow/build.gradle'. + +> Configure project :x-pack:plugin:esql:compute +Evaluating project ':x-pack:plugin:esql:compute' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql/compute/build.gradle'. + +> Configure project :x-pack:plugin:esql:qa +Evaluating project ':x-pack:plugin:esql:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql/qa/build.gradle'. + +> Configure project :x-pack:plugin:esql-core:test-fixtures +Evaluating project ':x-pack:plugin:esql-core:test-fixtures' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql-core/test-fixtures/build.gradle'. + +> Configure project :x-pack:plugin:fleet:qa +Evaluating project ':x-pack:plugin:fleet:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/fleet/qa/build.gradle'. + +> Configure project :x-pack:plugin:graph:qa +Evaluating project ':x-pack:plugin:graph:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/graph/qa/build.gradle'. + +> Configure project :x-pack:plugin:identity-provider:qa +Evaluating project ':x-pack:plugin:identity-provider:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/identity-provider/qa/build.gradle'. + +> Configure project :x-pack:plugin:ilm:qa +Evaluating project ':x-pack:plugin:ilm:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ilm/qa/build.gradle'. + +> Configure project :x-pack:plugin:inference:qa +Evaluating project ':x-pack:plugin:inference:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/inference/qa/build.gradle'. + +> Configure project :x-pack:plugin:logsdb:qa +Evaluating project ':x-pack:plugin:logsdb:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/logsdb/qa/build.gradle'. + +> Configure project :x-pack:plugin:ml:qa +Evaluating project ':x-pack:plugin:ml:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ml/qa/build.gradle'. + +> Configure project :x-pack:plugin:ql:test-fixtures +Evaluating project ':x-pack:plugin:ql:test-fixtures' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ql/test-fixtures/build.gradle'. + +> Configure project :x-pack:plugin:repositories-metering-api:qa +Evaluating project ':x-pack:plugin:repositories-metering-api:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/repositories-metering-api/qa/build.gradle'. + +> Configure project :x-pack:plugin:searchable-snapshots:qa +Evaluating project ':x-pack:plugin:searchable-snapshots:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/searchable-snapshots/qa/build.gradle'. + +> Configure project :x-pack:plugin:security:cli +Evaluating project ':x-pack:plugin:security:cli' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/cli/build.gradle'. + +> Configure project :x-pack:plugin:security:lib +Evaluating project ':x-pack:plugin:security:lib' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/lib/build.gradle'. + +> Configure project :x-pack:plugin:security:qa +Evaluating project ':x-pack:plugin:security:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/build.gradle'. + +> Configure project :x-pack:plugin:shutdown:qa +Evaluating project ':x-pack:plugin:shutdown:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/shutdown/qa/build.gradle'. + +> Configure project :x-pack:plugin:slm:qa +Evaluating project ':x-pack:plugin:slm:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/slm/qa/build.gradle'. + +> Configure project :x-pack:plugin:snapshot-based-recoveries:qa +Evaluating project ':x-pack:plugin:snapshot-based-recoveries:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-based-recoveries/qa/build.gradle'. + +> Configure project :x-pack:plugin:snapshot-repo-test-kit:qa +Evaluating project ':x-pack:plugin:snapshot-repo-test-kit:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-repo-test-kit/qa/build.gradle'. + +> Configure project :x-pack:plugin:sql:jdbc +Evaluating project ':x-pack:plugin:sql:jdbc' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/jdbc/build.gradle'. + +> Configure project :x-pack:plugin:sql:qa +Evaluating project ':x-pack:plugin:sql:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/build.gradle'. + +> Configure project :x-pack:plugin:sql:sql-action +Evaluating project ':x-pack:plugin:sql:sql-action' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/sql-action/build.gradle'. + +> Configure project :x-pack:plugin:sql:sql-cli +Evaluating project ':x-pack:plugin:sql:sql-cli' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/sql-cli/build.gradle'. + +> Configure project :x-pack:plugin:sql:sql-client +Evaluating project ':x-pack:plugin:sql:sql-client' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/sql-client/build.gradle'. + +> Configure project :x-pack:plugin:sql:sql-proto +Evaluating project ':x-pack:plugin:sql:sql-proto' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/sql-proto/build.gradle'. + +> Configure project :x-pack:plugin:stack:qa +Evaluating project ':x-pack:plugin:stack:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/stack/qa/build.gradle'. + +> Configure project :x-pack:plugin:text-structure:qa +Evaluating project ':x-pack:plugin:text-structure:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/text-structure/qa/build.gradle'. + +> Configure project :x-pack:plugin:transform:qa +Evaluating project ':x-pack:plugin:transform:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/transform/qa/build.gradle'. + +> Configure project :x-pack:plugin:vector-tile:qa +Evaluating project ':x-pack:plugin:vector-tile:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/vector-tile/qa/build.gradle'. + +> Configure project :x-pack:plugin:watcher:qa +Evaluating project ':x-pack:plugin:watcher:qa' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/watcher/qa/build.gradle'. + +> Configure project :x-pack:qa:multi-cluster-search-security:legacy-with-basic-license +Evaluating project ':x-pack:qa:multi-cluster-search-security:legacy-with-basic-license' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/build.gradle'. + +> Configure project :x-pack:qa:multi-cluster-search-security:legacy-with-full-license +Evaluating project ':x-pack:qa:multi-cluster-search-security:legacy-with-full-license' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/multi-cluster-search-security/legacy-with-full-license/build.gradle'. + +> Configure project :x-pack:qa:multi-cluster-search-security:legacy-with-restricted-trust +Evaluating project ':x-pack:qa:multi-cluster-search-security:legacy-with-restricted-trust' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/multi-cluster-search-security/legacy-with-restricted-trust/build.gradle'. + +> Configure project :x-pack:qa:runtime-fields:core-with-mapped +Evaluating project ':x-pack:qa:runtime-fields:core-with-mapped' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/runtime-fields/core-with-mapped/build.gradle'. + +> Configure project :x-pack:qa:runtime-fields:core-with-search +Evaluating project ':x-pack:qa:runtime-fields:core-with-search' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/runtime-fields/core-with-search/build.gradle'. + +> Configure project :x-pack:qa:runtime-fields:with-security +Evaluating project ':x-pack:qa:runtime-fields:with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/runtime-fields/with-security/build.gradle'. + +> Configure project :x-pack:qa:third-party:active-directory +Evaluating project ':x-pack:qa:third-party:active-directory' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/third-party/active-directory/build.gradle'. + +> Configure project :x-pack:qa:third-party:jira +Evaluating project ':x-pack:qa:third-party:jira' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/third-party/jira/build.gradle'. + +> Configure project :x-pack:qa:third-party:pagerduty +Evaluating project ':x-pack:qa:third-party:pagerduty' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/third-party/pagerduty/build.gradle'. + +> Configure project :x-pack:qa:third-party:slack +Evaluating project ':x-pack:qa:third-party:slack' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/qa/third-party/slack/build.gradle'. + +> Configure project :x-pack:plugin:async-search:qa:rest +Evaluating project ':x-pack:plugin:async-search:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/async-search/qa/rest/build.gradle'. + +> Configure project :x-pack:plugin:async-search:qa:security +Evaluating project ':x-pack:plugin:async-search:qa:security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/async-search/qa/security/build.gradle'. + +> Configure project :x-pack:plugin:autoscaling:qa:rest +Evaluating project ':x-pack:plugin:autoscaling:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/autoscaling/qa/rest/build.gradle'. + +> Configure project :x-pack:plugin:ccr:qa:downgrade-to-basic-license +Evaluating project ':x-pack:plugin:ccr:qa:downgrade-to-basic-license' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ccr/qa/downgrade-to-basic-license/build.gradle'. + +> Configure project :x-pack:plugin:ccr:qa:multi-cluster +Evaluating project ':x-pack:plugin:ccr:qa:multi-cluster' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ccr/qa/multi-cluster/build.gradle'. + +> Configure project :x-pack:plugin:ccr:qa:non-compliant-license +Evaluating project ':x-pack:plugin:ccr:qa:non-compliant-license' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ccr/qa/non-compliant-license/build.gradle'. + +> Configure project :x-pack:plugin:ccr:qa:rest +Evaluating project ':x-pack:plugin:ccr:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ccr/qa/rest/build.gradle'. + +> Configure project :x-pack:plugin:ccr:qa:restart +Evaluating project ':x-pack:plugin:ccr:qa:restart' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ccr/qa/restart/build.gradle'. + +> Configure project :x-pack:plugin:ccr:qa:security +Evaluating project ':x-pack:plugin:ccr:qa:security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ccr/qa/security/build.gradle'. + +> Configure project :x-pack:plugin:deprecation:qa:common +Evaluating project ':x-pack:plugin:deprecation:qa:common' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/deprecation/qa/common/build.gradle'. + +> Configure project :x-pack:plugin:deprecation:qa:early-deprecation-rest +Evaluating project ':x-pack:plugin:deprecation:qa:early-deprecation-rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/deprecation/qa/early-deprecation-rest/build.gradle'. + +> Configure project :x-pack:plugin:deprecation:qa:rest +Evaluating project ':x-pack:plugin:deprecation:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/deprecation/qa/rest/build.gradle'. + +> Configure project :x-pack:plugin:downsample:qa:mixed-cluster +Evaluating project ':x-pack:plugin:downsample:qa:mixed-cluster' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/downsample/qa/mixed-cluster/build.gradle'. + +> Configure project :x-pack:plugin:downsample:qa:rest +Evaluating project ':x-pack:plugin:downsample:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/downsample/qa/rest/build.gradle'. + +> Configure project :x-pack:plugin:downsample:qa:with-security +Evaluating project ':x-pack:plugin:downsample:qa:with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/downsample/qa/with-security/build.gradle'. + +> Configure project :x-pack:plugin:enrich:qa:common +Evaluating project ':x-pack:plugin:enrich:qa:common' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/enrich/qa/common/build.gradle'. + +> Configure project :x-pack:plugin:enrich:qa:rest +Evaluating project ':x-pack:plugin:enrich:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/enrich/qa/rest/build.gradle'. + +> Configure project :x-pack:plugin:enrich:qa:rest-with-advanced-security +Evaluating project ':x-pack:plugin:enrich:qa:rest-with-advanced-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/enrich/qa/rest-with-advanced-security/build.gradle'. + +> Configure project :x-pack:plugin:enrich:qa:rest-with-security +Evaluating project ':x-pack:plugin:enrich:qa:rest-with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/enrich/qa/rest-with-security/build.gradle'. + +> Configure project :x-pack:plugin:ent-search:qa:full-cluster-restart +Evaluating project ':x-pack:plugin:ent-search:qa:full-cluster-restart' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ent-search/qa/full-cluster-restart/build.gradle'. + +> Configure project :x-pack:plugin:ent-search:qa:rest +Evaluating project ':x-pack:plugin:ent-search:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ent-search/qa/rest/build.gradle'. + +> Configure project :x-pack:plugin:eql:qa:ccs-rolling-upgrade +Evaluating project ':x-pack:plugin:eql:qa:ccs-rolling-upgrade' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/eql/qa/ccs-rolling-upgrade/build.gradle'. + +> Configure project :x-pack:plugin:eql:qa:common +Evaluating project ':x-pack:plugin:eql:qa:common' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/eql/qa/common/build.gradle'. + +> Configure project :x-pack:plugin:eql:qa:correctness +Evaluating project ':x-pack:plugin:eql:qa:correctness' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/eql/qa/correctness/build.gradle'. + +> Configure project :x-pack:plugin:eql:qa:mixed-node +Evaluating project ':x-pack:plugin:eql:qa:mixed-node' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/eql/qa/mixed-node/build.gradle'. + +> Configure project :x-pack:plugin:eql:qa:multi-cluster-with-security +Evaluating project ':x-pack:plugin:eql:qa:multi-cluster-with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/eql/qa/multi-cluster-with-security/build.gradle'. + +> Configure project :x-pack:plugin:eql:qa:rest +Evaluating project ':x-pack:plugin:eql:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/eql/qa/rest/build.gradle'. + +> Configure project :x-pack:plugin:eql:qa:security +Evaluating project ':x-pack:plugin:eql:qa:security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/eql/qa/security/build.gradle'. + +> Configure project :x-pack:plugin:esql:compute:ann +Evaluating project ':x-pack:plugin:esql:compute:ann' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql/compute/ann/build.gradle'. + +> Configure project :x-pack:plugin:esql:compute:gen +Evaluating project ':x-pack:plugin:esql:compute:gen' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql/compute/gen/build.gradle'. + +> Configure project :x-pack:plugin:esql:qa:action +Evaluating project ':x-pack:plugin:esql:qa:action' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql/qa/action/build.gradle'. + +> Configure project :x-pack:plugin:esql:qa:security +Evaluating project ':x-pack:plugin:esql:qa:security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql/qa/security/build.gradle'. + +> Configure project :x-pack:plugin:esql:qa:server +Evaluating project ':x-pack:plugin:esql:qa:server' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql/qa/server/build.gradle'. + +> Configure project :x-pack:plugin:esql:qa:testFixtures +Evaluating project ':x-pack:plugin:esql:qa:testFixtures' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql/qa/testFixtures/build.gradle'. + +> Configure project :x-pack:plugin:fleet:qa:rest +Evaluating project ':x-pack:plugin:fleet:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/fleet/qa/rest/build.gradle'. + +> Configure project :x-pack:plugin:graph:qa:with-security +Evaluating project ':x-pack:plugin:graph:qa:with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/graph/qa/with-security/build.gradle'. + +> Configure project :x-pack:plugin:identity-provider:qa:idp-rest-tests +Evaluating project ':x-pack:plugin:identity-provider:qa:idp-rest-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/identity-provider/qa/idp-rest-tests/build.gradle'. + +> Configure project :x-pack:plugin:ilm:qa:multi-cluster +Evaluating project ':x-pack:plugin:ilm:qa:multi-cluster' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ilm/qa/multi-cluster/build.gradle'. + +> Configure project :x-pack:plugin:ilm:qa:multi-node +Evaluating project ':x-pack:plugin:ilm:qa:multi-node' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ilm/qa/multi-node/build.gradle'. + +> Configure project :x-pack:plugin:ilm:qa:rest +Evaluating project ':x-pack:plugin:ilm:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ilm/qa/rest/build.gradle'. + +> Configure project :x-pack:plugin:ilm:qa:with-security +Evaluating project ':x-pack:plugin:ilm:qa:with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ilm/qa/with-security/build.gradle'. + +> Configure project :x-pack:plugin:inference:qa:inference-service-tests +Evaluating project ':x-pack:plugin:inference:qa:inference-service-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/inference/qa/inference-service-tests/build.gradle'. + +> Configure project :x-pack:plugin:inference:qa:mixed-cluster +Evaluating project ':x-pack:plugin:inference:qa:mixed-cluster' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/inference/qa/mixed-cluster/build.gradle'. + +> Configure project :x-pack:plugin:inference:qa:rolling-upgrade +Evaluating project ':x-pack:plugin:inference:qa:rolling-upgrade' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/inference/qa/rolling-upgrade/build.gradle'. + +> Configure project :x-pack:plugin:inference:qa:test-service-plugin +Evaluating project ':x-pack:plugin:inference:qa:test-service-plugin' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/inference/qa/test-service-plugin/build.gradle'. + +> Configure project :x-pack:plugin:logsdb:qa:with-basic +Evaluating project ':x-pack:plugin:logsdb:qa:with-basic' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/logsdb/qa/with-basic/build.gradle'. + +> Configure project :x-pack:plugin:logsdb:qa:with-custom-cutoff +Evaluating project ':x-pack:plugin:logsdb:qa:with-custom-cutoff' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/logsdb/qa/with-custom-cutoff/build.gradle'. + +> Configure project :x-pack:plugin:ml:qa:basic-multi-node +Evaluating project ':x-pack:plugin:ml:qa:basic-multi-node' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ml/qa/basic-multi-node/build.gradle'. + +> Configure project :x-pack:plugin:ml:qa:disabled +Evaluating project ':x-pack:plugin:ml:qa:disabled' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ml/qa/disabled/build.gradle'. + +> Configure project :x-pack:plugin:ml:qa:ml-inference-service-tests +Evaluating project ':x-pack:plugin:ml:qa:ml-inference-service-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ml/qa/ml-inference-service-tests/build.gradle'. + +> Configure project :x-pack:plugin:ml:qa:ml-with-security +Evaluating project ':x-pack:plugin:ml:qa:ml-with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ml/qa/ml-with-security/build.gradle'. + +> Configure project :x-pack:plugin:ml:qa:multi-cluster-tests-with-security +Evaluating project ':x-pack:plugin:ml:qa:multi-cluster-tests-with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/build.gradle'. + +> Configure project :x-pack:plugin:ml:qa:native-multi-node-tests +Evaluating project ':x-pack:plugin:ml:qa:native-multi-node-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle'. + +> Configure project :x-pack:plugin:ml:qa:no-bootstrap-tests +Evaluating project ':x-pack:plugin:ml:qa:no-bootstrap-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ml/qa/no-bootstrap-tests/build.gradle'. + +> Configure project :x-pack:plugin:ml:qa:single-node-tests +Evaluating project ':x-pack:plugin:ml:qa:single-node-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/ml/qa/single-node-tests/build.gradle'. + +> Configure project :x-pack:plugin:repositories-metering-api:qa:azure +Evaluating project ':x-pack:plugin:repositories-metering-api:qa:azure' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/repositories-metering-api/qa/azure/build.gradle'. + +> Configure project :x-pack:plugin:repositories-metering-api:qa:gcs +Evaluating project ':x-pack:plugin:repositories-metering-api:qa:gcs' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/repositories-metering-api/qa/gcs/build.gradle'. + +> Configure project :x-pack:plugin:repositories-metering-api:qa:s3 +Evaluating project ':x-pack:plugin:repositories-metering-api:qa:s3' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/repositories-metering-api/qa/s3/build.gradle'. + +> Configure project :x-pack:plugin:searchable-snapshots:qa:azure +Evaluating project ':x-pack:plugin:searchable-snapshots:qa:azure' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/searchable-snapshots/qa/azure/build.gradle'. + +> Configure project :x-pack:plugin:searchable-snapshots:qa:gcs +Evaluating project ':x-pack:plugin:searchable-snapshots:qa:gcs' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle'. + +> Configure project :x-pack:plugin:searchable-snapshots:qa:hdfs +Evaluating project ':x-pack:plugin:searchable-snapshots:qa:hdfs' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle'. + +> Configure project :x-pack:plugin:searchable-snapshots:qa:minio +Evaluating project ':x-pack:plugin:searchable-snapshots:qa:minio' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/searchable-snapshots/qa/minio/build.gradle'. + +> Configure project :x-pack:plugin:searchable-snapshots:qa:rest +Evaluating project ':x-pack:plugin:searchable-snapshots:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle'. + +> Configure project :x-pack:plugin:searchable-snapshots:qa:s3 +Evaluating project ':x-pack:plugin:searchable-snapshots:qa:s3' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle'. + +> Configure project :x-pack:plugin:searchable-snapshots:qa:url +Evaluating project ':x-pack:plugin:searchable-snapshots:qa:url' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/searchable-snapshots/qa/url/build.gradle'. + +> Configure project :x-pack:plugin:security:lib:nimbus-jose-jwt-modified +Evaluating project ':x-pack:plugin:security:lib:nimbus-jose-jwt-modified' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/build.gradle'. + +> Configure project :x-pack:plugin:security:lib:nimbus-jose-jwt-modified-part1 +Evaluating project ':x-pack:plugin:security:lib:nimbus-jose-jwt-modified-part1' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/lib/nimbus-jose-jwt-modified-part1/build.gradle'. + +> Configure project :x-pack:plugin:security:lib:nimbus-jose-jwt-modified-part2 +Evaluating project ':x-pack:plugin:security:lib:nimbus-jose-jwt-modified-part2' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/lib/nimbus-jose-jwt-modified-part2/build.gradle'. + +> Configure project :x-pack:plugin:security:qa:audit +Evaluating project ':x-pack:plugin:security:qa:audit' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/audit/build.gradle'. + +> Configure project :x-pack:plugin:security:qa:basic-enable-security +Evaluating project ':x-pack:plugin:security:qa:basic-enable-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/basic-enable-security/build.gradle'. + +> Configure project :x-pack:plugin:security:qa:consistency-checks +Evaluating project ':x-pack:plugin:security:qa:consistency-checks' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/consistency-checks/build.gradle'. + +> Configure project :x-pack:plugin:security:qa:jwt-realm +Evaluating project ':x-pack:plugin:security:qa:jwt-realm' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/jwt-realm/build.gradle'. + +> Configure project :x-pack:plugin:security:qa:multi-cluster +Evaluating project ':x-pack:plugin:security:qa:multi-cluster' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/multi-cluster/build.gradle'. + +> Configure project :x-pack:plugin:security:qa:operator-privileges-tests +Evaluating project ':x-pack:plugin:security:qa:operator-privileges-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/operator-privileges-tests/build.gradle'. + +> Configure project :x-pack:plugin:security:qa:profile +Evaluating project ':x-pack:plugin:security:qa:profile' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/profile/build.gradle'. + +> Configure project :x-pack:plugin:security:qa:saml-rest-tests +Evaluating project ':x-pack:plugin:security:qa:saml-rest-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/saml-rest-tests/build.gradle'. + +> Configure project :x-pack:plugin:security:qa:secondary-auth-actions +Evaluating project ':x-pack:plugin:security:qa:secondary-auth-actions' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/secondary-auth-actions/build.gradle'. + +> Configure project :x-pack:plugin:security:qa:security-basic +Evaluating project ':x-pack:plugin:security:qa:security-basic' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/security-basic/build.gradle'. + +> Configure project :x-pack:plugin:security:qa:security-disabled +Evaluating project ':x-pack:plugin:security:qa:security-disabled' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/security-disabled/build.gradle'. + +> Configure project :x-pack:plugin:security:qa:security-trial +Evaluating project ':x-pack:plugin:security:qa:security-trial' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/security-trial/build.gradle'. + +> Configure project :x-pack:plugin:security:qa:service-account +Evaluating project ':x-pack:plugin:security:qa:service-account' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/service-account/build.gradle'. + +> Configure project :x-pack:plugin:security:qa:smoke-test-all-realms +Evaluating project ':x-pack:plugin:security:qa:smoke-test-all-realms' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/smoke-test-all-realms/build.gradle'. + +> Configure project :x-pack:plugin:security:qa:tls-basic +Evaluating project ':x-pack:plugin:security:qa:tls-basic' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/security/qa/tls-basic/build.gradle'. + +> Configure project :x-pack:plugin:shutdown:qa:full-cluster-restart +Evaluating project ':x-pack:plugin:shutdown:qa:full-cluster-restart' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/shutdown/qa/full-cluster-restart/build.gradle'. + +> Configure project :x-pack:plugin:shutdown:qa:multi-node +Evaluating project ':x-pack:plugin:shutdown:qa:multi-node' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/shutdown/qa/multi-node/build.gradle'. + +> Configure project :x-pack:plugin:shutdown:qa:rolling-upgrade +Evaluating project ':x-pack:plugin:shutdown:qa:rolling-upgrade' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/shutdown/qa/rolling-upgrade/build.gradle'. + +> Configure project :x-pack:plugin:slm:qa:multi-node +Evaluating project ':x-pack:plugin:slm:qa:multi-node' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/slm/qa/multi-node/build.gradle'. + +> Configure project :x-pack:plugin:slm:qa:rest +Evaluating project ':x-pack:plugin:slm:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/slm/qa/rest/build.gradle'. + +> Configure project :x-pack:plugin:slm:qa:with-security +Evaluating project ':x-pack:plugin:slm:qa:with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/slm/qa/with-security/build.gradle'. + +> Configure project :x-pack:plugin:snapshot-based-recoveries:qa:azure +Evaluating project ':x-pack:plugin:snapshot-based-recoveries:qa:azure' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-based-recoveries/qa/azure/build.gradle'. + +> Configure project :x-pack:plugin:snapshot-based-recoveries:qa:fs +Evaluating project ':x-pack:plugin:snapshot-based-recoveries:qa:fs' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-based-recoveries/qa/fs/build.gradle'. + +> Configure project :x-pack:plugin:snapshot-based-recoveries:qa:gcs +Evaluating project ':x-pack:plugin:snapshot-based-recoveries:qa:gcs' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-based-recoveries/qa/gcs/build.gradle'. + +> Configure project :x-pack:plugin:snapshot-based-recoveries:qa:license-enforcing +Evaluating project ':x-pack:plugin:snapshot-based-recoveries:qa:license-enforcing' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-based-recoveries/qa/license-enforcing/build.gradle'. + +> Configure project :x-pack:plugin:snapshot-based-recoveries:qa:s3 +Evaluating project ':x-pack:plugin:snapshot-based-recoveries:qa:s3' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-based-recoveries/qa/s3/build.gradle'. + +> Configure project :x-pack:plugin:snapshot-repo-test-kit:qa:azure +Evaluating project ':x-pack:plugin:snapshot-repo-test-kit:qa:azure' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-repo-test-kit/qa/azure/build.gradle'. + +> Configure project :x-pack:plugin:snapshot-repo-test-kit:qa:gcs +Evaluating project ':x-pack:plugin:snapshot-repo-test-kit:qa:gcs' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-repo-test-kit/qa/gcs/build.gradle'. + +> Configure project :x-pack:plugin:snapshot-repo-test-kit:qa:hdfs +Evaluating project ':x-pack:plugin:snapshot-repo-test-kit:qa:hdfs' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/build.gradle'. + +> Configure project :x-pack:plugin:snapshot-repo-test-kit:qa:minio +Evaluating project ':x-pack:plugin:snapshot-repo-test-kit:qa:minio' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-repo-test-kit/qa/minio/build.gradle'. + +> Configure project :x-pack:plugin:snapshot-repo-test-kit:qa:rest +Evaluating project ':x-pack:plugin:snapshot-repo-test-kit:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-repo-test-kit/qa/rest/build.gradle'. + +> Configure project :x-pack:plugin:snapshot-repo-test-kit:qa:s3 +Evaluating project ':x-pack:plugin:snapshot-repo-test-kit:qa:s3' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/snapshot-repo-test-kit/qa/s3/build.gradle'. + +> Configure project :x-pack:plugin:sql:qa:jdbc +Evaluating project ':x-pack:plugin:sql:qa:jdbc' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/jdbc/build.gradle'. + +> Configure project :x-pack:plugin:sql:qa:mixed-node +Evaluating project ':x-pack:plugin:sql:qa:mixed-node' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/mixed-node/build.gradle'. + +> Configure project :x-pack:plugin:sql:qa:server +Evaluating project ':x-pack:plugin:sql:qa:server' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/server/build.gradle'. + +> Configure project :x-pack:plugin:stack:qa:rest +Evaluating project ':x-pack:plugin:stack:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/stack/qa/rest/build.gradle'. + +> Configure project :x-pack:plugin:text-structure:qa:text-structure-with-security +Evaluating project ':x-pack:plugin:text-structure:qa:text-structure-with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/text-structure/qa/text-structure-with-security/build.gradle'. + +> Configure project :x-pack:plugin:transform:qa:common +Evaluating project ':x-pack:plugin:transform:qa:common' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/transform/qa/common/build.gradle'. + +> Configure project :x-pack:plugin:transform:qa:multi-cluster-tests-with-security +Evaluating project ':x-pack:plugin:transform:qa:multi-cluster-tests-with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/build.gradle'. + +> Configure project :x-pack:plugin:transform:qa:multi-node-tests +Evaluating project ':x-pack:plugin:transform:qa:multi-node-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/transform/qa/multi-node-tests/build.gradle'. + +> Configure project :x-pack:plugin:transform:qa:single-node-tests +Evaluating project ':x-pack:plugin:transform:qa:single-node-tests' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/transform/qa/single-node-tests/build.gradle'. + +> Configure project :x-pack:plugin:vector-tile:qa:multi-cluster +Evaluating project ':x-pack:plugin:vector-tile:qa:multi-cluster' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/vector-tile/qa/multi-cluster/build.gradle'. + +> Configure project :x-pack:plugin:watcher:qa:common +Evaluating project ':x-pack:plugin:watcher:qa:common' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/watcher/qa/common/build.gradle'. + +> Configure project :x-pack:plugin:watcher:qa:rest +Evaluating project ':x-pack:plugin:watcher:qa:rest' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/watcher/qa/rest/build.gradle'. + +> Configure project :x-pack:plugin:watcher:qa:with-security +Evaluating project ':x-pack:plugin:watcher:qa:with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/watcher/qa/with-security/build.gradle'. + +> Configure project :x-pack:plugin:esql:qa:server:mixed-cluster +Evaluating project ':x-pack:plugin:esql:qa:server:mixed-cluster' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle'. + +> Configure project :x-pack:plugin:esql:qa:server:multi-clusters +Evaluating project ':x-pack:plugin:esql:qa:server:multi-clusters' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql/qa/server/multi-clusters/build.gradle'. + +> Configure project :x-pack:plugin:esql:qa:server:multi-node +Evaluating project ':x-pack:plugin:esql:qa:server:multi-node' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql/qa/server/multi-node/build.gradle'. + +> Configure project :x-pack:plugin:esql:qa:server:single-node +Evaluating project ':x-pack:plugin:esql:qa:server:single-node' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/esql/qa/server/single-node/build.gradle'. + +> Configure project :x-pack:plugin:sql:qa:jdbc:multi-node +Evaluating project ':x-pack:plugin:sql:qa:jdbc:multi-node' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/jdbc/multi-node/build.gradle'. + +> Configure project :x-pack:plugin:sql:qa:jdbc:no-sql +Evaluating project ':x-pack:plugin:sql:qa:jdbc:no-sql' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/jdbc/no-sql/build.gradle'. + +> Configure project :x-pack:plugin:sql:qa:jdbc:security +Evaluating project ':x-pack:plugin:sql:qa:jdbc:security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/jdbc/security/build.gradle'. + +> Configure project :x-pack:plugin:sql:qa:jdbc:single-node +Evaluating project ':x-pack:plugin:sql:qa:jdbc:single-node' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/jdbc/single-node/build.gradle'. + +> Configure project :x-pack:plugin:sql:qa:server:multi-cluster-with-security +Evaluating project ':x-pack:plugin:sql:qa:server:multi-cluster-with-security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/server/multi-cluster-with-security/build.gradle'. + +> Configure project :x-pack:plugin:sql:qa:server:multi-node +Evaluating project ':x-pack:plugin:sql:qa:server:multi-node' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/server/multi-node/build.gradle'. + +> Configure project :x-pack:plugin:sql:qa:server:security +Evaluating project ':x-pack:plugin:sql:qa:server:security' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/server/security/build.gradle'. + +> Configure project :x-pack:plugin:sql:qa:server:single-node +Evaluating project ':x-pack:plugin:sql:qa:server:single-node' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/server/single-node/build.gradle'. + +> Configure project :x-pack:plugin:sql:qa:jdbc:security:with-ssl +Evaluating project ':x-pack:plugin:sql:qa:jdbc:security:with-ssl' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/jdbc/security/with-ssl/build.gradle'. + +> Configure project :x-pack:plugin:sql:qa:jdbc:security:without-ssl +Evaluating project ':x-pack:plugin:sql:qa:jdbc:security:without-ssl' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/jdbc/security/without-ssl/build.gradle'. + +> Configure project :x-pack:plugin:sql:qa:server:security:with-ssl +Evaluating project ':x-pack:plugin:sql:qa:server:security:with-ssl' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/server/security/with-ssl/build.gradle'. + +> Configure project :x-pack:plugin:sql:qa:server:security:without-ssl +Evaluating project ':x-pack:plugin:sql:qa:server:security:without-ssl' using build file '/Users/rene/dev/elastic/elasticsearch/x-pack/plugin/sql/qa/server/security/without-ssl/build.gradle'. +All projects evaluated. +test#with21 +Task name matched 'cleanTest' +Task name matched 'test' +Selected primary task 'cleanTest' from project :libs:simdvec +Selected primary task 'test' from project :libs:simdvec +Tasks to be executed: [task ':libs:simdvec:cleanTest', task ':libs:logging:compileJava', task ':libs:core:compileJava', task ':libs:native:compileJava', task ':libs:simdvec:compileJava', task ':libs:simdvec:processResources', task ':libs:simdvec:classes', task ':libs:simdvec:compileMain21Java', task ':libs:simdvec:processMain21Resources', task ':libs:simdvec:main21Classes', task ':libs:simdvec:compileMain22Java', task ':libs:simdvec:processMain22Resources', task ':libs:simdvec:main22Classes', task ':libs:simdvec:jar', task ':libs:cli:compileJava', task ':libs:x-content:compileJava', task ':libs:entitlement:bridge:compileJava', task ':libs:entitlement:compileJava', task ':libs:geo:compileJava', task ':libs:grok:compileJava', task ':libs:lz4:compileJava', task ':libs:plugin-api:compileJava', task ':libs:plugin-analysis-api:compileJava', task ':libs:secure-sm:compileJava', task ':libs:tdigest:compileJava', task ':server:compileJava', task ':server:processResources', task ':server:classes', task ':server:generateModulesList', task ':server:generatePluginsList', task ':server:jar', task ':client:rest:compileJava', task ':client:rest:processResources', task ':client:rest:classes', task ':client:rest:jar', task ':libs:cli:processResources', task ':libs:cli:classes', task ':libs:cli:jar', task ':libs:core:processResources', task ':libs:core:classes', task ':libs:core:jar', task ':libs:entitlement:asm-provider:compileJava', task ':libs:entitlement:asm-provider:processResources', task ':libs:entitlement:asm-provider:classes', task ':libs:entitlement:asm-provider:jar', task ':libs:entitlement:generateAsm-providerProviderManifest', task ':libs:entitlement:generateAsm-providerProviderImpl', task ':libs:entitlement:processResources', task ':libs:entitlement:classes', task ':libs:entitlement:bridge:processResources', task ':libs:entitlement:bridge:classes', task ':libs:entitlement:bridge:compileMain23Java', task ':libs:entitlement:bridge:processMain23Resources', task ':libs:entitlement:bridge:main23Classes', task ':libs:entitlement:bridge:java23Jar', task ':libs:entitlement:compileMain23Java', task ':libs:entitlement:processMain23Resources', task ':libs:entitlement:main23Classes', task ':libs:entitlement:jar', task ':libs:geo:processResources', task ':libs:geo:classes', task ':libs:geo:jar', task ':libs:grok:processResources', task ':libs:grok:classes', task ':libs:grok:jar', task ':libs:logging:processResources', task ':libs:logging:classes', task ':libs:logging:jar', task ':libs:lz4:processResources', task ':libs:lz4:classes', task ':libs:lz4:jar', task ':libs:native:processResources', task ':libs:native:classes', task ':libs:native:compileMain22Java', task ':libs:native:processMain22Resources', task ':libs:native:main22Classes', task ':libs:native:jar', task ':libs:plugin-analysis-api:processResources', task ':libs:plugin-analysis-api:classes', task ':libs:plugin-analysis-api:jar', task ':libs:plugin-api:processResources', task ':libs:plugin-api:classes', task ':libs:plugin-api:jar', task ':libs:secure-sm:processResources', task ':libs:secure-sm:classes', task ':libs:secure-sm:jar', task ':libs:ssl-config:compileJava', task ':modules:transport-netty4:compileJava', task ':test:framework:compileJava', task ':libs:simdvec:compileTestJava', task ':libs:simdvec:processTestResources', task ':libs:simdvec:testClasses', task ':libs:ssl-config:processResources', task ':libs:ssl-config:classes', task ':libs:ssl-config:jar', task ':libs:tdigest:processResources', task ':libs:tdigest:classes', task ':libs:tdigest:jar', task ':libs:x-content:impl:compileJava', task ':libs:x-content:impl:processResources', task ':libs:x-content:impl:classes', task ':libs:x-content:impl:jar', task ':libs:x-content:generateImplProviderManifest', task ':libs:x-content:generateImplProviderImpl', task ':libs:x-content:processResources', task ':libs:x-content:classes', task ':libs:x-content:jar', task ':modules:transport-netty4:processResources', task ':modules:transport-netty4:classes', task ':modules:transport-netty4:jar', task ':test:framework:processResources', task ':test:framework:classes', task ':test:framework:jar', task ':test:immutable-collections-patch:compileJava', task ':test:immutable-collections-patch:processResources', task ':test:immutable-collections-patch:classes', task ':test:immutable-collections-patch:generatePatch', task ':libs:native:native-libraries:extractLibs', task ':libs:simdvec:test'] +Tasks that were excluded: [] +======================================= +Elasticsearch Build Hamster says Hello! + Gradle Version : 8.11.1 + OS Info : Mac OS X 15.2 (aarch64) + JDK Version : 21.0.5+9-LTS-239 (Oracle) + JAVA_HOME : /Users/rene/.sdkman/candidates/java/21.0.5-oracle + Random Testing Seed : 7B469FBE8B6D0C65 + In FIPS 140 mode : false +======================================= +Ignoring listeners of task graph ready event, as this build (:) has already executed work. +Resolve mutations for :libs:simdvec:cleanTest (Thread[#1322,Execution worker,5,main]) started. +work action Unknown value (Thread[#1392,included builds Thread 2,5,main]) started. +work action Unknown value (Thread[#1330,Execution worker Thread 9,5,main]) started. +:libs:simdvec:cleanTest (Thread[#1322,Execution worker,5,main]) started. +destroyer locations for task group 0 (Thread[#1332,Execution worker Thread 11,5,main]) started. +Resolve mutations for :libs:logging:compileJava (Thread[#1332,Execution worker Thread 11,5,main]) started. +Resolve mutations for :libs:simdvec:processResources (Thread[#1326,Execution worker Thread 5,5,main]) started. +Resolve mutations for :libs:simdvec:processMain22Resources (Thread[#1329,Execution worker Thread 8,5,main]) started. +Resolve mutations for :libs:simdvec:processMain21Resources (Thread[#1324,Execution worker Thread 3,5,main]) started. +Resolve mutations for :libs:grok:compileJava (Thread[#1327,Execution worker Thread 6,5,main]) started. +:libs:simdvec:processResources (Thread[#1326,Execution worker Thread 5,5,main]) started. +:libs:simdvec:processMain21Resources (Thread[#1324,Execution worker Thread 3,5,main]) started. +:libs:simdvec:processMain22Resources (Thread[#1329,Execution worker Thread 8,5,main]) started. +Resolve mutations for :libs:geo:compileJava (Thread[#1328,Execution worker Thread 7,5,main]) started. +Resolve mutations for :libs:secure-sm:compileJava (Thread[#1332,Execution worker Thread 11,5,main]) started. +Resolve mutations for :libs:entitlement:bridge:compileJava (Thread[#1325,Execution worker Thread 4,5,main]) started. +Resolve mutations for :server:generateModulesList (Thread[#1392,included builds Thread 2,5,main]) started. +:libs:secure-sm:compileJava (Thread[#1332,Execution worker Thread 11,5,main]) started. +Resolve mutations for :server:processResources (Thread[#1330,Execution worker Thread 9,5,main]) started. +:libs:logging:compileJava (Thread[#1323,Execution worker Thread 2,5,main]) started. +Resolve mutations for :libs:plugin-api:compileJava (Thread[#1331,Execution worker Thread 10,5,main]) started. +:server:processResources (Thread[#1330,Execution worker Thread 9,5,main]) started. +:server:generateModulesList (Thread[#1392,included builds Thread 2,5,main]) started. +:libs:entitlement:bridge:compileJava (Thread[#1325,Execution worker Thread 4,5,main]) started. +:libs:geo:compileJava (Thread[#1328,Execution worker Thread 7,5,main]) started. +:libs:plugin-api:compileJava (Thread[#1331,Execution worker Thread 10,5,main]) started. +:libs:grok:compileJava (Thread[#1327,Execution worker Thread 6,5,main]) started. + +> Task :libs:simdvec:processMain22Resources NO-SOURCE +Skipping task ':libs:simdvec:processMain22Resources' as it has no source files and no previous output files. + +> Task :libs:simdvec:processMain21Resources NO-SOURCE +Skipping task ':libs:simdvec:processMain21Resources' as it has no source files and no previous output files. + +> Task :server:generateModulesList UP-TO-DATE +Caching disabled for task ':server:generateModulesList' because: + Build cache is disabled + Gradle would require more information to cache this task +Skipping task ':server:generateModulesList' as it is up-to-date. + +> Task :libs:simdvec:processResources NO-SOURCE +Skipping task ':libs:simdvec:processResources' as it has no source files and no previous output files. +Resolve mutations for :client:rest:processResources (Thread[#1392,included builds Thread 2,5,main]) started. + +> Task :libs:grok:compileJava UP-TO-DATE +Caching disabled for task ':libs:grok:compileJava' because: + Build cache is disabled +Skipping task ':libs:grok:compileJava' as it is up-to-date. +:client:rest:processResources (Thread[#1392,included builds Thread 2,5,main]) started. +Resolve mutations for :libs:cli:processResources (Thread[#1326,Execution worker Thread 5,5,main]) started. +Resolve mutations for :client:rest:compileJava (Thread[#1324,Execution worker Thread 3,5,main]) started. + +> Task :libs:plugin-api:compileJava UP-TO-DATE +Caching disabled for task ':libs:plugin-api:compileJava' because: + Build cache is disabled +Skipping task ':libs:plugin-api:compileJava' as it is up-to-date. +No compile result for :libs:plugin-api:compileJava +No compile result for :libs:plugin-api:compileJava +Resolve mutations for :server:generatePluginsList (Thread[#1329,Execution worker Thread 8,5,main]) started. + +> Task :libs:secure-sm:compileJava UP-TO-DATE +Caching disabled for task ':libs:secure-sm:compileJava' because: + Build cache is disabled +Skipping task ':libs:secure-sm:compileJava' as it is up-to-date. +No compile result for :libs:secure-sm:compileJava +No compile result for :libs:secure-sm:compileJava +No compile result for :libs:plugin-api:compileJava +No compile result for :libs:plugin-api:compileJava +:libs:cli:processResources (Thread[#1326,Execution worker Thread 5,5,main]) started. + +> Task :libs:geo:compileJava UP-TO-DATE +Caching disabled for task ':libs:geo:compileJava' because: + Build cache is disabled +Skipping task ':libs:geo:compileJava' as it is up-to-date. + +> Task :libs:entitlement:bridge:compileJava UP-TO-DATE +Caching disabled for task ':libs:entitlement:bridge:compileJava' because: + Build cache is disabled +Skipping task ':libs:entitlement:bridge:compileJava' as it is up-to-date. +No compile result for :libs:geo:compileJava +No compile result for :libs:geo:compileJava +No compile result for :libs:grok:compileJava +No compile result for :libs:grok:compileJava + +> Task :server:processResources UP-TO-DATE +Caching disabled for task ':server:processResources' because: + Build cache is disabled + Not worth caching +Skipping task ':server:processResources' as it is up-to-date. +No compile result for :libs:grok:compileJava +work action resolve main (project :libs:plugin-api) (Thread[#1330,Execution worker Thread 9,5,main]) started. +No compile result for :libs:geo:compileJava +No compile result for :libs:geo:compileJava +No compile result for :libs:entitlement:bridge:compileJava +No compile result for :libs:entitlement:bridge:compileJava +No compile result for :libs:entitlement:bridge:compileJava +No compile result for :libs:entitlement:bridge:compileJava + +> Task :libs:logging:compileJava UP-TO-DATE +Caching disabled for task ':libs:logging:compileJava' because: + Build cache is disabled +Skipping task ':libs:logging:compileJava' as it is up-to-date. +work action resolve main (project :libs:entitlement:bridge) (Thread[#1325,Execution worker Thread 4,5,main]) started. +work action resolve main (project :libs:plugin-api) (Thread[#1331,Execution worker Thread 10,5,main]) started. +No compile result for :libs:secure-sm:compileJava +:server:generatePluginsList (Thread[#1329,Execution worker Thread 8,5,main]) started. +:client:rest:compileJava (Thread[#1324,Execution worker Thread 3,5,main]) started. +No compile result for :libs:secure-sm:compileJava +Resolve mutations for :libs:plugin-analysis-api:compileJava (Thread[#1331,Execution worker Thread 10,5,main]) started. +work action resolve main (project :libs:entitlement:bridge) (Thread[#1325,Execution worker Thread 4,5,main]) started. +No compile result for :libs:logging:compileJava +work action resolve main (project :libs:geo) (Thread[#1328,Execution worker Thread 7,5,main]) started. +Resolve mutations for :libs:core:processResources (Thread[#1330,Execution worker Thread 9,5,main]) started. +No compile result for :libs:grok:compileJava +work action resolve main (project :libs:grok) (Thread[#1330,Execution worker Thread 9,5,main]) started. +work action resolve main (project :libs:grok) (Thread[#1327,Execution worker Thread 6,5,main]) started. +:libs:plugin-analysis-api:compileJava (Thread[#1331,Execution worker Thread 10,5,main]) started. +Resolve mutations for :libs:entitlement:asm-provider:processResources (Thread[#1327,Execution worker Thread 6,5,main]) started. +work action resolve main (project :libs:secure-sm) (Thread[#1328,Execution worker Thread 7,5,main]) started. +:libs:entitlement:asm-provider:processResources (Thread[#1327,Execution worker Thread 6,5,main]) started. +No compile result for :libs:logging:compileJava +work action resolve main (project :libs:geo) (Thread[#1325,Execution worker Thread 4,5,main]) started. +work action resolve main (project :libs:secure-sm) (Thread[#1332,Execution worker Thread 11,5,main]) started. +Resolve mutations for :libs:entitlement:bridge:processResources (Thread[#1325,Execution worker Thread 4,5,main]) started. +No compile result for :libs:logging:compileJava +No compile result for :libs:logging:compileJava +:libs:entitlement:bridge:processResources (Thread[#1325,Execution worker Thread 4,5,main]) started. +work action resolve main (project :libs:logging) (Thread[#1323,Execution worker Thread 2,5,main]) started. +Resolve mutations for :libs:entitlement:processResources (Thread[#1328,Execution worker Thread 7,5,main]) started. +:libs:entitlement:processResources (Thread[#1328,Execution worker Thread 7,5,main]) started. + +> Task :libs:cli:processResources NO-SOURCE +Skipping task ':libs:cli:processResources' as it has no source files and no previous output files. +:libs:core:processResources (Thread[#1330,Execution worker Thread 9,5,main]) started. +Resolve mutations for :libs:entitlement:processMain23Resources (Thread[#1326,Execution worker Thread 5,5,main]) started. + +> Task :server:generatePluginsList UP-TO-DATE +Caching disabled for task ':server:generatePluginsList' because: + Build cache is disabled + Gradle would require more information to cache this task +Skipping task ':server:generatePluginsList' as it is up-to-date. +Resolve mutations for :libs:geo:processResources (Thread[#1329,Execution worker Thread 8,5,main]) started. +:libs:entitlement:processMain23Resources (Thread[#1326,Execution worker Thread 5,5,main]) started. +work action resolve main (project :libs:logging) (Thread[#1323,Execution worker Thread 2,5,main]) started. +Resolve mutations for :libs:core:compileJava (Thread[#1323,Execution worker Thread 2,5,main]) started. +Resolve mutations for :libs:entitlement:bridge:processMain23Resources (Thread[#1332,Execution worker Thread 11,5,main]) started. + +> Task :libs:entitlement:processResources NO-SOURCE +Skipping task ':libs:entitlement:processResources' as it has no source files and no previous output files. +Resolve mutations for :libs:grok:processResources (Thread[#1328,Execution worker Thread 7,5,main]) started. + +> Task :libs:entitlement:bridge:processResources NO-SOURCE +Skipping task ':libs:entitlement:bridge:processResources' as it has no source files and no previous output files. + +> Task :libs:plugin-analysis-api:compileJava UP-TO-DATE +Caching disabled for task ':libs:plugin-analysis-api:compileJava' because: + Build cache is disabled +Skipping task ':libs:plugin-analysis-api:compileJava' as it is up-to-date. +:libs:grok:processResources (Thread[#1328,Execution worker Thread 7,5,main]) started. + +> Task :libs:entitlement:processMain23Resources NO-SOURCE +Skipping task ':libs:entitlement:processMain23Resources' as it has no source files and no previous output files. +:libs:geo:processResources (Thread[#1329,Execution worker Thread 8,5,main]) started. + +> Task :client:rest:processResources UP-TO-DATE +Caching disabled for task ':client:rest:processResources' because: + Build cache is disabled + Not worth caching +Skipping task ':client:rest:processResources' as it is up-to-date. +:libs:core:compileJava (Thread[#1323,Execution worker Thread 2,5,main]) started. + +> Task :client:rest:compileJava UP-TO-DATE +Caching disabled for task ':client:rest:compileJava' because: + Build cache is disabled +Skipping task ':client:rest:compileJava' as it is up-to-date. +No compile result for :client:rest:compileJava +No compile result for :libs:plugin-analysis-api:compileJava +Resolve mutations for :libs:entitlement:bridge:classes (Thread[#1325,Execution worker Thread 4,5,main]) started. +:libs:entitlement:bridge:processMain23Resources (Thread[#1332,Execution worker Thread 11,5,main]) started. +No compile result for :libs:plugin-analysis-api:compileJava +Resolve mutations for :libs:lz4:processResources (Thread[#1326,Execution worker Thread 5,5,main]) started. +Resolve mutations for :libs:logging:processResources (Thread[#1392,included builds Thread 2,5,main]) started. +No compile result for :client:rest:compileJava +:libs:logging:processResources (Thread[#1392,included builds Thread 2,5,main]) started. + +> Task :libs:geo:processResources NO-SOURCE +Skipping task ':libs:geo:processResources' as it has no source files and no previous output files. + +> Task :libs:core:processResources NO-SOURCE +Skipping task ':libs:core:processResources' as it has no source files and no previous output files. +No compile result for :client:rest:compileJava +:libs:lz4:processResources (Thread[#1326,Execution worker Thread 5,5,main]) started. +No compile result for :libs:plugin-analysis-api:compileJava + +> Task :libs:entitlement:asm-provider:processResources UP-TO-DATE +Caching disabled for task ':libs:entitlement:asm-provider:processResources' because: + Build cache is disabled + Not worth caching +Skipping task ':libs:entitlement:asm-provider:processResources' as it is up-to-date. + +> Task :libs:grok:processResources UP-TO-DATE +Caching disabled for task ':libs:grok:processResources' because: + Build cache is disabled + Not worth caching +Skipping task ':libs:grok:processResources' as it is up-to-date. +:libs:entitlement:bridge:classes (Thread[#1325,Execution worker Thread 4,5,main]) started. +Resolve mutations for :libs:native:processMain22Resources (Thread[#1327,Execution worker Thread 6,5,main]) started. +Resolve mutations for :libs:grok:classes (Thread[#1328,Execution worker Thread 7,5,main]) started. +No compile result for :libs:plugin-analysis-api:compileJava + +> Task :libs:entitlement:bridge:processMain23Resources NO-SOURCE +Skipping task ':libs:entitlement:bridge:processMain23Resources' as it has no source files and no previous output files. +:libs:grok:classes (Thread[#1328,Execution worker Thread 7,5,main]) started. +work action resolve main (project :libs:plugin-analysis-api) (Thread[#1331,Execution worker Thread 10,5,main]) started. +work action resolve main (project :libs:plugin-analysis-api) (Thread[#1332,Execution worker Thread 11,5,main]) started. +Resolve mutations for :libs:plugin-analysis-api:processResources (Thread[#1331,Execution worker Thread 10,5,main]) started. +Resolve mutations for :libs:plugin-api:processResources (Thread[#1332,Execution worker Thread 11,5,main]) started. + +> Task :libs:logging:processResources NO-SOURCE +Skipping task ':libs:logging:processResources' as it has no source files and no previous output files. +No compile result for :client:rest:compileJava +:libs:plugin-analysis-api:processResources (Thread[#1331,Execution worker Thread 10,5,main]) started. +Resolve mutations for :client:rest:classes (Thread[#1324,Execution worker Thread 3,5,main]) started. +Resolve mutations for :libs:logging:classes (Thread[#1392,included builds Thread 2,5,main]) started. +:client:rest:classes (Thread[#1324,Execution worker Thread 3,5,main]) started. +Resolve mutations for :libs:native:processResources (Thread[#1330,Execution worker Thread 9,5,main]) started. +:libs:logging:classes (Thread[#1392,included builds Thread 2,5,main]) started. +Resolve mutations for :libs:geo:classes (Thread[#1329,Execution worker Thread 8,5,main]) started. +:libs:native:processResources (Thread[#1330,Execution worker Thread 9,5,main]) started. +:libs:plugin-api:processResources (Thread[#1332,Execution worker Thread 11,5,main]) started. + +> Task :libs:plugin-analysis-api:processResources NO-SOURCE +Skipping task ':libs:plugin-analysis-api:processResources' as it has no source files and no previous output files. +:libs:native:processMain22Resources (Thread[#1327,Execution worker Thread 6,5,main]) started. +Resolve mutations for :libs:plugin-analysis-api:classes (Thread[#1331,Execution worker Thread 10,5,main]) started. +:libs:plugin-analysis-api:classes (Thread[#1331,Execution worker Thread 10,5,main]) started. + +> Task :libs:grok:classes UP-TO-DATE +Skipping task ':libs:grok:classes' as it has no actions. + +> Task :libs:core:compileJava UP-TO-DATE +Caching disabled for task ':libs:core:compileJava' because: + Build cache is disabled +Skipping task ':libs:core:compileJava' as it is up-to-date. +No compile result for :libs:core:compileJava +:libs:geo:classes (Thread[#1329,Execution worker Thread 8,5,main]) started. + +> Task :libs:logging:classes UP-TO-DATE +Skipping task ':libs:logging:classes' as it has no actions. + +> Task :client:rest:classes UP-TO-DATE +Skipping task ':client:rest:classes' as it has no actions. + +> Task :libs:geo:classes UP-TO-DATE +Skipping task ':libs:geo:classes' as it has no actions. +Resolve mutations for :client:rest:jar (Thread[#1324,Execution worker Thread 3,5,main]) started. +Resolve mutations for :libs:logging:jar (Thread[#1392,included builds Thread 2,5,main]) started. +No compile result for :libs:core:compileJava + +> Task :libs:native:processMain22Resources NO-SOURCE +Skipping task ':libs:native:processMain22Resources' as it has no source files and no previous output files. +No compile result for :libs:core:compileJava +:client:rest:jar (Thread[#1324,Execution worker Thread 3,5,main]) started. +Resolve mutations for :libs:grok:jar (Thread[#1328,Execution worker Thread 7,5,main]) started. + +> Task :libs:native:processResources NO-SOURCE +Skipping task ':libs:native:processResources' as it has no source files and no previous output files. +work action resolve main (project :client:rest) (Thread[#1330,Execution worker Thread 9,5,main]) started. +Resolve mutations for :libs:simdvec:processTestResources (Thread[#1330,Execution worker Thread 9,5,main]) started. + +> Task :libs:entitlement:bridge:classes UP-TO-DATE +Skipping task ':libs:entitlement:bridge:classes' as it has no actions. + +> Task :libs:plugin-analysis-api:classes UP-TO-DATE +Skipping task ':libs:plugin-analysis-api:classes' as it has no actions. + +> Task :libs:lz4:processResources NO-SOURCE +Skipping task ':libs:lz4:processResources' as it has no source files and no previous output files. +:libs:simdvec:processTestResources (Thread[#1330,Execution worker Thread 9,5,main]) started. +Resolve mutations for :libs:entitlement:bridge:compileMain23Java (Thread[#1331,Execution worker Thread 10,5,main]) started. +:libs:grok:jar (Thread[#1328,Execution worker Thread 7,5,main]) started. +Resolve mutations for :libs:ssl-config:processResources (Thread[#1326,Execution worker Thread 5,5,main]) started. +Resolve mutations for :libs:secure-sm:processResources (Thread[#1327,Execution worker Thread 6,5,main]) started. +:libs:logging:jar (Thread[#1392,included builds Thread 2,5,main]) started. + +> Task :libs:simdvec:processTestResources NO-SOURCE +Skipping task ':libs:simdvec:processTestResources' as it has no source files and no previous output files. +No compile result for :libs:core:compileJava + +> Task :libs:plugin-api:processResources NO-SOURCE +Skipping task ':libs:plugin-api:processResources' as it has no source files and no previous output files. +Resolve mutations for :libs:geo:jar (Thread[#1329,Execution worker Thread 8,5,main]) started. +work action resolve main (project :libs:core) (Thread[#1332,Execution worker Thread 11,5,main]) started. +work action resolve main (project :libs:core) (Thread[#1323,Execution worker Thread 2,5,main]) started. +Resolve mutations for :libs:tdigest:processResources (Thread[#1330,Execution worker Thread 9,5,main]) started. +:libs:secure-sm:processResources (Thread[#1327,Execution worker Thread 6,5,main]) started. +:libs:ssl-config:processResources (Thread[#1326,Execution worker Thread 5,5,main]) started. +:libs:entitlement:bridge:compileMain23Java (Thread[#1331,Execution worker Thread 10,5,main]) started. +Resolve mutations for :libs:plugin-analysis-api:jar (Thread[#1325,Execution worker Thread 4,5,main]) started. +Resolve mutations for :libs:lz4:compileJava (Thread[#1325,Execution worker Thread 4,5,main]) started. +Resolve mutations for :libs:x-content:compileJava (Thread[#1330,Execution worker Thread 9,5,main]) started. +Resolve mutations for :libs:cli:compileJava (Thread[#1323,Execution worker Thread 2,5,main]) started. +Resolve mutations for :libs:native:compileJava (Thread[#1329,Execution worker Thread 8,5,main]) started. +Resolve mutations for :libs:core:classes (Thread[#1332,Execution worker Thread 11,5,main]) started. +Resolve mutations for :libs:tdigest:compileJava (Thread[#1332,Execution worker Thread 11,5,main]) started. +:libs:lz4:compileJava (Thread[#1325,Execution worker Thread 4,5,main]) started. + +> Task :libs:simdvec:cleanTest +Caching disabled for task ':libs:simdvec:cleanTest' because: + Build cache is disabled +Task ':libs:simdvec:cleanTest' is not up-to-date because: + Task has not declared any outputs despite executing actions. +:libs:cli:compileJava (Thread[#1323,Execution worker Thread 2,5,main]) started. + +> Task :libs:ssl-config:processResources NO-SOURCE +Skipping task ':libs:ssl-config:processResources' as it has no source files and no previous output files. +:libs:tdigest:compileJava (Thread[#1332,Execution worker Thread 11,5,main]) started. +:libs:native:compileJava (Thread[#1329,Execution worker Thread 8,5,main]) started. +:libs:x-content:compileJava (Thread[#1330,Execution worker Thread 9,5,main]) started. + +> Task :libs:secure-sm:processResources NO-SOURCE +Skipping task ':libs:secure-sm:processResources' as it has no source files and no previous output files. +:libs:geo:jar (Thread[#1326,Execution worker Thread 5,5,main]) started. +:libs:core:classes (Thread[#1322,Execution worker,5,main]) started. +:libs:plugin-analysis-api:jar (Thread[#1327,Execution worker Thread 6,5,main]) started. + +> Task :libs:entitlement:bridge:compileMain23Java UP-TO-DATE +Caching disabled for task ':libs:entitlement:bridge:compileMain23Java' because: + Build cache is disabled +Skipping task ':libs:entitlement:bridge:compileMain23Java' as it is up-to-date. + +> Task :libs:lz4:compileJava UP-TO-DATE +Caching disabled for task ':libs:lz4:compileJava' because: + Build cache is disabled +Skipping task ':libs:lz4:compileJava' as it is up-to-date. +No compile result for :libs:lz4:compileJava +No compile result for :libs:lz4:compileJava +No compile result for :libs:lz4:compileJava +No compile result for :libs:entitlement:bridge:compileMain23Java + +> Task :libs:cli:compileJava UP-TO-DATE +Caching disabled for task ':libs:cli:compileJava' because: + Build cache is disabled +Skipping task ':libs:cli:compileJava' as it is up-to-date. +No compile result for :libs:cli:compileJava +No compile result for :libs:cli:compileJava +No compile result for :libs:cli:compileJava +No compile result for :libs:cli:compileJava +work action resolve main (project :libs:cli) (Thread[#1323,Execution worker Thread 2,5,main]) started. +No compile result for :libs:entitlement:bridge:compileMain23Java +No compile result for :libs:entitlement:bridge:compileMain23Java +No compile result for :libs:lz4:compileJava +work action resolve main (project :libs:lz4) (Thread[#1325,Execution worker Thread 4,5,main]) started. +No compile result for :libs:entitlement:bridge:compileMain23Java + +> Task :libs:native:compileJava UP-TO-DATE +Caching disabled for task ':libs:native:compileJava' because: + Build cache is disabled +Skipping task ':libs:native:compileJava' as it is up-to-date. +No compile result for :libs:native:compileJava +Resolve mutations for :libs:cli:classes (Thread[#1331,Execution worker Thread 10,5,main]) started. +work action resolve main (project :libs:cli) (Thread[#1323,Execution worker Thread 2,5,main]) started. +No compile result for :libs:native:compileJava +work action resolve main (project :libs:lz4) (Thread[#1325,Execution worker Thread 4,5,main]) started. + +> Task :libs:core:classes UP-TO-DATE +Skipping task ':libs:core:classes' as it has no actions. +No compile result for :libs:native:compileJava +:libs:cli:classes (Thread[#1331,Execution worker Thread 10,5,main]) started. + +> Task :libs:tdigest:compileJava UP-TO-DATE +Caching disabled for task ':libs:tdigest:compileJava' because: + Build cache is disabled +Skipping task ':libs:tdigest:compileJava' as it is up-to-date. +Resolve mutations for :libs:entitlement:bridge:main23Classes (Thread[#1323,Execution worker Thread 2,5,main]) started. +No compile result for :libs:tdigest:compileJava +:libs:entitlement:bridge:main23Classes (Thread[#1323,Execution worker Thread 2,5,main]) started. + +> Task :libs:x-content:compileJava UP-TO-DATE +Caching disabled for task ':libs:x-content:compileJava' because: + Build cache is disabled +Skipping task ':libs:x-content:compileJava' as it is up-to-date. +No compile result for :libs:native:compileJava + +> Task :libs:entitlement:bridge:main23Classes UP-TO-DATE +Skipping task ':libs:entitlement:bridge:main23Classes' as it has no actions. +Resolve mutations for :libs:core:jar (Thread[#1322,Execution worker,5,main]) started. +work action resolve main (project :libs:native) (Thread[#1323,Execution worker Thread 2,5,main]) started. +Resolve mutations for :libs:lz4:classes (Thread[#1325,Execution worker Thread 4,5,main]) started. +Resolve mutations for :libs:entitlement:bridge:java23Jar (Thread[#1323,Execution worker Thread 2,5,main]) started. +:libs:lz4:classes (Thread[#1325,Execution worker Thread 4,5,main]) started. +work action resolve main (project :libs:native) (Thread[#1329,Execution worker Thread 8,5,main]) started. +No compile result for :libs:x-content:compileJava +:libs:entitlement:bridge:java23Jar (Thread[#1323,Execution worker Thread 2,5,main]) started. +No compile result for :libs:tdigest:compileJava +No compile result for :libs:tdigest:compileJava +No compile result for :libs:tdigest:compileJava +No compile result for :libs:x-content:compileJava +Resolve mutations for :libs:simdvec:compileJava (Thread[#1329,Execution worker Thread 8,5,main]) started. +:libs:core:jar (Thread[#1322,Execution worker,5,main]) started. + +> Task :libs:lz4:classes UP-TO-DATE +Skipping task ':libs:lz4:classes' as it has no actions. +No compile result for :libs:x-content:compileJava +work action resolve main (project :libs:tdigest) (Thread[#1332,Execution worker Thread 11,5,main]) started. + +> Task :libs:cli:classes UP-TO-DATE +Skipping task ':libs:cli:classes' as it has no actions. +Resolve mutations for :libs:lz4:jar (Thread[#1332,Execution worker Thread 11,5,main]) started. +No compile result for :libs:x-content:compileJava +work action resolve main (project :libs:x-content) (Thread[#1330,Execution worker Thread 9,5,main]) started. +work action resolve main (project :libs:tdigest) (Thread[#1325,Execution worker Thread 4,5,main]) started. +:libs:lz4:jar (Thread[#1332,Execution worker Thread 11,5,main]) started. +Resolve mutations for :libs:native:classes (Thread[#1325,Execution worker Thread 4,5,main]) started. +:libs:simdvec:compileJava (Thread[#1329,Execution worker Thread 8,5,main]) started. +:libs:native:classes (Thread[#1325,Execution worker Thread 4,5,main]) started. +work action resolve main (project :libs:x-content) (Thread[#1330,Execution worker Thread 9,5,main]) started. +Resolve mutations for :libs:cli:jar (Thread[#1331,Execution worker Thread 10,5,main]) started. +Resolve mutations for :libs:entitlement:compileJava (Thread[#1330,Execution worker Thread 9,5,main]) started. +:libs:cli:jar (Thread[#1331,Execution worker Thread 10,5,main]) started. + +> Task :libs:native:classes UP-TO-DATE +Skipping task ':libs:native:classes' as it has no actions. +Resolve mutations for :libs:native:compileMain22Java (Thread[#1325,Execution worker Thread 4,5,main]) started. +:libs:entitlement:compileJava (Thread[#1330,Execution worker Thread 9,5,main]) started. + +> Task :libs:entitlement:bridge:java23Jar UP-TO-DATE +Caching disabled for task ':libs:entitlement:bridge:java23Jar' because: + Build cache is disabled + Not worth caching +Skipping task ':libs:entitlement:bridge:java23Jar' as it is up-to-date. +work action resolve elasticsearch-entitlement-bridge-9.0.0-SNAPSHOT.jar (project :libs:entitlement:bridge) (Thread[#1323,Execution worker Thread 2,5,main]) started. +:libs:native:compileMain22Java (Thread[#1325,Execution worker Thread 4,5,main]) started. +Resolve mutations for :libs:plugin-api:classes (Thread[#1323,Execution worker Thread 2,5,main]) started. +:libs:plugin-api:classes (Thread[#1323,Execution worker Thread 2,5,main]) started. + +> Task :libs:plugin-api:classes UP-TO-DATE +Skipping task ':libs:plugin-api:classes' as it has no actions. +Resolve mutations for :libs:plugin-api:jar (Thread[#1323,Execution worker Thread 2,5,main]) started. +:libs:plugin-api:jar (Thread[#1323,Execution worker Thread 2,5,main]) started. + +> Task :libs:simdvec:compileJava UP-TO-DATE +Caching disabled for task ':libs:simdvec:compileJava' because: + Build cache is disabled +Skipping task ':libs:simdvec:compileJava' as it is up-to-date. +No compile result for :libs:simdvec:compileJava +No compile result for :libs:simdvec:compileJava +No compile result for :libs:simdvec:compileJava +No compile result for :libs:simdvec:compileJava +Resolve mutations for :libs:simdvec:classes (Thread[#1329,Execution worker Thread 8,5,main]) started. +:libs:simdvec:classes (Thread[#1329,Execution worker Thread 8,5,main]) started. + +> Task :libs:entitlement:compileJava UP-TO-DATE +Caching disabled for task ':libs:entitlement:compileJava' because: + Build cache is disabled +Skipping task ':libs:entitlement:compileJava' as it is up-to-date. +No compile result for :libs:entitlement:compileJava + +> Task :libs:native:compileMain22Java UP-TO-DATE +Caching disabled for task ':libs:native:compileMain22Java' because: + Build cache is disabled +Skipping task ':libs:native:compileMain22Java' as it is up-to-date. +No compile result for :libs:entitlement:compileJava +No compile result for :libs:native:compileMain22Java +No compile result for :libs:native:compileMain22Java +No compile result for :libs:entitlement:compileJava +No compile result for :libs:native:compileMain22Java +No compile result for :libs:native:compileMain22Java +No compile result for :libs:entitlement:compileJava +work action resolve main (project :libs:simdvec) (Thread[#1325,Execution worker Thread 4,5,main]) started. +work action resolve main (project :libs:entitlement) (Thread[#1330,Execution worker Thread 9,5,main]) started. + +> Task :libs:simdvec:classes UP-TO-DATE +Skipping task ':libs:simdvec:classes' as it has no actions. +work action resolve main (project :libs:simdvec) (Thread[#1330,Execution worker Thread 9,5,main]) started. +work action resolve main (project :libs:entitlement) (Thread[#1325,Execution worker Thread 4,5,main]) started. +Resolve mutations for :libs:simdvec:compileMain21Java (Thread[#1329,Execution worker Thread 8,5,main]) started. +Resolve mutations for :libs:native:main22Classes (Thread[#1330,Execution worker Thread 9,5,main]) started. +Resolve mutations for :server:compileJava (Thread[#1325,Execution worker Thread 4,5,main]) started. +Resolve mutations for :libs:entitlement:asm-provider:compileJava (Thread[#1330,Execution worker Thread 9,5,main]) started. +:libs:simdvec:compileMain21Java (Thread[#1329,Execution worker Thread 8,5,main]) started. +:server:compileJava (Thread[#1325,Execution worker Thread 4,5,main]) started. +:libs:entitlement:asm-provider:compileJava (Thread[#1330,Execution worker Thread 9,5,main]) started. + +> Task :client:rest:jar UP-TO-DATE +Caching disabled for task ':client:rest:jar' because: + Build cache is disabled + Not worth caching +Skipping task ':client:rest:jar' as it is up-to-date. +:libs:native:main22Classes (Thread[#1324,Execution worker Thread 3,5,main]) started. + +> Task :libs:native:main22Classes UP-TO-DATE +Skipping task ':libs:native:main22Classes' as it has no actions. +Resolve mutations for :libs:native:jar (Thread[#1324,Execution worker Thread 3,5,main]) started. +:libs:native:jar (Thread[#1324,Execution worker Thread 3,5,main]) started. + +> Task :libs:simdvec:compileMain21Java UP-TO-DATE +Caching disabled for task ':libs:simdvec:compileMain21Java' because: + Build cache is disabled +Skipping task ':libs:simdvec:compileMain21Java' as it is up-to-date. +No compile result for :libs:simdvec:compileMain21Java +No compile result for :libs:simdvec:compileMain21Java + +> Task :libs:entitlement:asm-provider:compileJava UP-TO-DATE +Caching disabled for task ':libs:entitlement:asm-provider:compileJava' because: + Build cache is disabled +Skipping task ':libs:entitlement:asm-provider:compileJava' as it is up-to-date. +No compile result for :libs:simdvec:compileMain21Java +No compile result for :libs:entitlement:asm-provider:compileJava +No compile result for :libs:simdvec:compileMain21Java +No compile result for :libs:entitlement:asm-provider:compileJava +Resolve mutations for :libs:simdvec:main21Classes (Thread[#1329,Execution worker Thread 8,5,main]) started. +No compile result for :libs:entitlement:asm-provider:compileJava +No compile result for :libs:entitlement:asm-provider:compileJava +:libs:simdvec:main21Classes (Thread[#1329,Execution worker Thread 8,5,main]) started. +Resolve mutations for :libs:entitlement:asm-provider:classes (Thread[#1330,Execution worker Thread 9,5,main]) started. +:libs:entitlement:asm-provider:classes (Thread[#1330,Execution worker Thread 9,5,main]) started. + +> Task :libs:simdvec:main21Classes UP-TO-DATE +Skipping task ':libs:simdvec:main21Classes' as it has no actions. + +> Task :libs:entitlement:asm-provider:classes UP-TO-DATE +Skipping task ':libs:entitlement:asm-provider:classes' as it has no actions. +Resolve mutations for :libs:simdvec:compileMain22Java (Thread[#1329,Execution worker Thread 8,5,main]) started. +Resolve mutations for :libs:entitlement:asm-provider:jar (Thread[#1330,Execution worker Thread 9,5,main]) started. +:libs:entitlement:asm-provider:jar (Thread[#1330,Execution worker Thread 9,5,main]) started. +:libs:simdvec:compileMain22Java (Thread[#1329,Execution worker Thread 8,5,main]) started. + +> Task :libs:simdvec:compileMain22Java UP-TO-DATE +Caching disabled for task ':libs:simdvec:compileMain22Java' because: + Build cache is disabled +Skipping task ':libs:simdvec:compileMain22Java' as it is up-to-date. +No compile result for :libs:simdvec:compileMain22Java +No compile result for :libs:simdvec:compileMain22Java +No compile result for :libs:simdvec:compileMain22Java +No compile result for :libs:simdvec:compileMain22Java +Resolve mutations for :libs:simdvec:main22Classes (Thread[#1329,Execution worker Thread 8,5,main]) started. +:libs:simdvec:main22Classes (Thread[#1329,Execution worker Thread 8,5,main]) started. + +> Task :libs:simdvec:main22Classes UP-TO-DATE +Skipping task ':libs:simdvec:main22Classes' as it has no actions. +Resolve mutations for :libs:simdvec:jar (Thread[#1329,Execution worker Thread 8,5,main]) started. +:libs:simdvec:jar (Thread[#1329,Execution worker Thread 8,5,main]) started. + +> Task :libs:logging:jar UP-TO-DATE +Caching disabled for task ':libs:logging:jar' because: + Build cache is disabled + Not worth caching +Skipping task ':libs:logging:jar' as it is up-to-date. +Resolve mutations for :libs:secure-sm:classes (Thread[#1392,included builds Thread 2,5,main]) started. +:libs:secure-sm:classes (Thread[#1392,included builds Thread 2,5,main]) started. + +> Task :libs:secure-sm:classes UP-TO-DATE +Skipping task ':libs:secure-sm:classes' as it has no actions. +Resolve mutations for :libs:secure-sm:jar (Thread[#1392,included builds Thread 2,5,main]) started. +:libs:secure-sm:jar (Thread[#1392,included builds Thread 2,5,main]) started. + +> Task :server:compileJava UP-TO-DATE +Caching disabled for task ':server:compileJava' because: + Build cache is disabled +Skipping task ':server:compileJava' as it is up-to-date. +No compile result for :server:compileJava +No compile result for :server:compileJava +No compile result for :server:compileJava +No compile result for :server:compileJava +Resolve mutations for :server:classes (Thread[#1325,Execution worker Thread 4,5,main]) started. +:server:classes (Thread[#1325,Execution worker Thread 4,5,main]) started. + +> Task :server:classes UP-TO-DATE +Skipping task ':server:classes' as it has no actions. + +> Task :libs:grok:jar UP-TO-DATE +Caching disabled for task ':libs:grok:jar' because: + Build cache is disabled + Not worth caching +Skipping task ':libs:grok:jar' as it is up-to-date. +Resolve mutations for :server:jar (Thread[#1325,Execution worker Thread 4,5,main]) started. +Resolve mutations for :libs:ssl-config:compileJava (Thread[#1328,Execution worker Thread 7,5,main]) started. +:server:jar (Thread[#1325,Execution worker Thread 4,5,main]) started. +:libs:ssl-config:compileJava (Thread[#1328,Execution worker Thread 7,5,main]) started. + +> Task :libs:ssl-config:compileJava UP-TO-DATE +Caching disabled for task ':libs:ssl-config:compileJava' because: + Build cache is disabled +Skipping task ':libs:ssl-config:compileJava' as it is up-to-date. +No compile result for :libs:ssl-config:compileJava +No compile result for :libs:ssl-config:compileJava +No compile result for :libs:ssl-config:compileJava +No compile result for :libs:ssl-config:compileJava +work action resolve main (project :server) (Thread[#1328,Execution worker Thread 7,5,main]) started. +work action resolve main (project :libs:ssl-config) (Thread[#1328,Execution worker Thread 7,5,main]) started. +work action resolve main (project :server) (Thread[#1328,Execution worker Thread 7,5,main]) started. +work action resolve main (project :libs:ssl-config) (Thread[#1328,Execution worker Thread 7,5,main]) started. +Resolve mutations for :modules:transport-netty4:compileJava (Thread[#1328,Execution worker Thread 7,5,main]) started. +:modules:transport-netty4:compileJava (Thread[#1328,Execution worker Thread 7,5,main]) started. + +> Task :libs:geo:jar UP-TO-DATE +Caching disabled for task ':libs:geo:jar' because: + Build cache is disabled + Not worth caching +Skipping task ':libs:geo:jar' as it is up-to-date. +Resolve mutations for :libs:ssl-config:classes (Thread[#1326,Execution worker Thread 5,5,main]) started. +:libs:ssl-config:classes (Thread[#1326,Execution worker Thread 5,5,main]) started. + +> Task :libs:ssl-config:classes UP-TO-DATE +Skipping task ':libs:ssl-config:classes' as it has no actions. + +> Task :modules:transport-netty4:compileJava UP-TO-DATE +Caching disabled for task ':modules:transport-netty4:compileJava' because: + Build cache is disabled +Skipping task ':modules:transport-netty4:compileJava' as it is up-to-date. +No compile result for :modules:transport-netty4:compileJava +Resolve mutations for :libs:ssl-config:jar (Thread[#1326,Execution worker Thread 5,5,main]) started. +No compile result for :modules:transport-netty4:compileJava +No compile result for :modules:transport-netty4:compileJava +No compile result for :modules:transport-netty4:compileJava +:libs:ssl-config:jar (Thread[#1326,Execution worker Thread 5,5,main]) started. +work action resolve main (project :modules:transport-netty4) (Thread[#1328,Execution worker Thread 7,5,main]) started. +Resolve mutations for :test:framework:compileJava (Thread[#1328,Execution worker Thread 7,5,main]) started. +:test:framework:compileJava (Thread[#1328,Execution worker Thread 7,5,main]) started. + +> Task :test:framework:compileJava UP-TO-DATE +Caching disabled for task ':test:framework:compileJava' because: + Build cache is disabled +Skipping task ':test:framework:compileJava' as it is up-to-date. +No compile result for :test:framework:compileJava +No compile result for :test:framework:compileJava +No compile result for :test:framework:compileJava +No compile result for :test:framework:compileJava +work action resolve main (project :test:framework) (Thread[#1328,Execution worker Thread 7,5,main]) started. +Resolve mutations for :libs:simdvec:compileTestJava (Thread[#1328,Execution worker Thread 7,5,main]) started. +:libs:simdvec:compileTestJava (Thread[#1328,Execution worker Thread 7,5,main]) started. + +> Task :libs:plugin-analysis-api:jar UP-TO-DATE +Caching disabled for task ':libs:plugin-analysis-api:jar' because: + Build cache is disabled + Not worth caching +Skipping task ':libs:plugin-analysis-api:jar' as it is up-to-date. +:libs:tdigest:processResources (Thread[#1327,Execution worker Thread 6,5,main]) started. + +> Task :libs:tdigest:processResources NO-SOURCE +Skipping task ':libs:tdigest:processResources' as it has no source files and no previous output files. +Resolve mutations for :libs:tdigest:classes (Thread[#1327,Execution worker Thread 6,5,main]) started. +:libs:tdigest:classes (Thread[#1327,Execution worker Thread 6,5,main]) started. + +> Task :libs:tdigest:classes UP-TO-DATE +Skipping task ':libs:tdigest:classes' as it has no actions. +Resolve mutations for :libs:tdigest:jar (Thread[#1327,Execution worker Thread 6,5,main]) started. +:libs:tdigest:jar (Thread[#1327,Execution worker Thread 6,5,main]) started. + +> Task :libs:simdvec:compileTestJava UP-TO-DATE +Caching disabled for task ':libs:simdvec:compileTestJava' because: + Build cache is disabled +Skipping task ':libs:simdvec:compileTestJava' as it is up-to-date. +No compile result for :libs:simdvec:compileTestJava +No compile result for :libs:simdvec:compileTestJava +No compile result for :libs:simdvec:compileTestJava +No compile result for :libs:simdvec:compileTestJava +Resolve mutations for :libs:simdvec:testClasses (Thread[#1328,Execution worker Thread 7,5,main]) started. +:libs:simdvec:testClasses (Thread[#1328,Execution worker Thread 7,5,main]) started. + +> Task :libs:simdvec:testClasses UP-TO-DATE +Skipping task ':libs:simdvec:testClasses' as it has no actions. +Resolve mutations for :libs:x-content:impl:compileJava (Thread[#1328,Execution worker Thread 7,5,main]) started. +:libs:x-content:impl:compileJava (Thread[#1328,Execution worker Thread 7,5,main]) started. + +> Task :libs:x-content:impl:compileJava UP-TO-DATE +Caching disabled for task ':libs:x-content:impl:compileJava' because: + Build cache is disabled +Skipping task ':libs:x-content:impl:compileJava' as it is up-to-date. +No compile result for :libs:x-content:impl:compileJava +No compile result for :libs:x-content:impl:compileJava +No compile result for :libs:x-content:impl:compileJava +No compile result for :libs:x-content:impl:compileJava +Resolve mutations for :libs:x-content:impl:processResources (Thread[#1328,Execution worker Thread 7,5,main]) started. +:libs:x-content:impl:processResources (Thread[#1328,Execution worker Thread 7,5,main]) started. + +> Task :libs:x-content:impl:processResources UP-TO-DATE +Caching disabled for task ':libs:x-content:impl:processResources' because: + Build cache is disabled + Not worth caching +Skipping task ':libs:x-content:impl:processResources' as it is up-to-date. +Resolve mutations for :libs:x-content:impl:classes (Thread[#1328,Execution worker Thread 7,5,main]) started. +:libs:x-content:impl:classes (Thread[#1328,Execution worker Thread 7,5,main]) started. + +> Task :libs:x-content:impl:classes UP-TO-DATE +Skipping task ':libs:x-content:impl:classes' as it has no actions. +Resolve mutations for :libs:x-content:impl:jar (Thread[#1328,Execution worker Thread 7,5,main]) started. +:libs:x-content:impl:jar (Thread[#1328,Execution worker Thread 7,5,main]) started. + +> Task :libs:lz4:jar UP-TO-DATE +Caching disabled for task ':libs:lz4:jar' because: + Build cache is disabled + Not worth caching +Skipping task ':libs:lz4:jar' as it is up-to-date. +Resolve mutations for :libs:x-content:processResources (Thread[#1332,Execution worker Thread 11,5,main]) started. +:libs:x-content:processResources (Thread[#1332,Execution worker Thread 11,5,main]) started. + +> Task :libs:x-content:processResources NO-SOURCE +Skipping task ':libs:x-content:processResources' as it has no source files and no previous output files. +Resolve mutations for :modules:transport-netty4:processResources (Thread[#1332,Execution worker Thread 11,5,main]) started. +:modules:transport-netty4:processResources (Thread[#1332,Execution worker Thread 11,5,main]) started. + +> Task :modules:transport-netty4:processResources NO-SOURCE +Skipping task ':modules:transport-netty4:processResources' as it has no source files and no previous output files. +Resolve mutations for :modules:transport-netty4:classes (Thread[#1332,Execution worker Thread 11,5,main]) started. +:modules:transport-netty4:classes (Thread[#1332,Execution worker Thread 11,5,main]) started. + +> Task :modules:transport-netty4:classes UP-TO-DATE +Skipping task ':modules:transport-netty4:classes' as it has no actions. +Resolve mutations for :modules:transport-netty4:jar (Thread[#1332,Execution worker Thread 11,5,main]) started. +:modules:transport-netty4:jar (Thread[#1332,Execution worker Thread 11,5,main]) started. + +> Task :libs:core:jar UP-TO-DATE +Caching disabled for task ':libs:core:jar' because: + Build cache is disabled + Not worth caching +Skipping task ':libs:core:jar' as it is up-to-date. +Resolve mutations for :test:framework:processResources (Thread[#1322,Execution worker,5,main]) started. +:test:framework:processResources (Thread[#1322,Execution worker,5,main]) started. + +> Task :test:framework:processResources UP-TO-DATE +Caching disabled for task ':test:framework:processResources' because: + Build cache is disabled + Not worth caching +Skipping task ':test:framework:processResources' as it is up-to-date. +Resolve mutations for :test:framework:classes (Thread[#1322,Execution worker,5,main]) started. +:test:framework:classes (Thread[#1322,Execution worker,5,main]) started. + +> Task :test:framework:classes UP-TO-DATE +Skipping task ':test:framework:classes' as it has no actions. +Resolve mutations for :test:framework:jar (Thread[#1322,Execution worker,5,main]) started. +:test:framework:jar (Thread[#1322,Execution worker,5,main]) started. + +> Task :libs:cli:jar UP-TO-DATE +Caching disabled for task ':libs:cli:jar' because: + Build cache is disabled + Not worth caching +Skipping task ':libs:cli:jar' as it is up-to-date. +Resolve mutations for :test:immutable-collections-patch:compileJava (Thread[#1331,Execution worker Thread 10,5,main]) started. +:test:immutable-collections-patch:compileJava (Thread[#1331,Execution worker Thread 10,5,main]) started. + +> Task :test:immutable-collections-patch:compileJava UP-TO-DATE +Caching disabled for task ':test:immutable-collections-patch:compileJava' because: + Build cache is disabled +Skipping task ':test:immutable-collections-patch:compileJava' as it is up-to-date. +No compile result for :test:immutable-collections-patch:compileJava +No compile result for :test:immutable-collections-patch:compileJava +No compile result for :test:immutable-collections-patch:compileJava +No compile result for :test:immutable-collections-patch:compileJava +Resolve mutations for :test:immutable-collections-patch:processResources (Thread[#1331,Execution worker Thread 10,5,main]) started. +:test:immutable-collections-patch:processResources (Thread[#1331,Execution worker Thread 10,5,main]) started. + +> Task :test:immutable-collections-patch:processResources NO-SOURCE +Skipping task ':test:immutable-collections-patch:processResources' as it has no source files and no previous output files. +Resolve mutations for :test:immutable-collections-patch:classes (Thread[#1331,Execution worker Thread 10,5,main]) started. +:test:immutable-collections-patch:classes (Thread[#1331,Execution worker Thread 10,5,main]) started. + +> Task :test:immutable-collections-patch:classes UP-TO-DATE +Skipping task ':test:immutable-collections-patch:classes' as it has no actions. +Resolve mutations for :test:immutable-collections-patch:generatePatch (Thread[#1331,Execution worker Thread 10,5,main]) started. +:test:immutable-collections-patch:generatePatch (Thread[#1331,Execution worker Thread 10,5,main]) started. + +> Task :test:immutable-collections-patch:generatePatch UP-TO-DATE +Caching disabled for task ':test:immutable-collections-patch:generatePatch' because: + Build cache is disabled + Gradle would require more information to cache this task +Skipping task ':test:immutable-collections-patch:generatePatch' as it is up-to-date. +Resolve mutations for :libs:native:native-libraries:extractLibs (Thread[#1331,Execution worker Thread 10,5,main]) started. +:libs:native:native-libraries:extractLibs (Thread[#1331,Execution worker Thread 10,5,main]) started. + +> Task :libs:plugin-api:jar UP-TO-DATE +Caching disabled for task ':libs:plugin-api:jar' because: + Build cache is disabled + Not worth caching +Skipping task ':libs:plugin-api:jar' as it is up-to-date. +work action resolve jdk-patches (project :test:immutable-collections-patch) (Thread[#1323,Execution worker Thread 2,5,main]) started. +work action resolve elasticsearch-grok-9.0.0-SNAPSHOT.jar (project :libs:grok) (Thread[#1323,Execution worker Thread 2,5,main]) started. +work action resolve elasticsearch-plugin-api-9.0.0-SNAPSHOT.jar (project :libs:plugin-api) (Thread[#1323,Execution worker Thread 2,5,main]) started. +work action resolve elasticsearch-geo-9.0.0-SNAPSHOT.jar (project :libs:geo) (Thread[#1323,Execution worker Thread 2,5,main]) started. +work action resolve elasticsearch-rest-client-9.0.0-SNAPSHOT.jar (project :client:rest) (Thread[#1323,Execution worker Thread 2,5,main]) started. +work action resolve elasticsearch-core-9.0.0-SNAPSHOT.jar (project :libs:core) (Thread[#1323,Execution worker Thread 2,5,main]) started. +work action resolve elasticsearch-lz4-9.0.0-SNAPSHOT.jar (project :libs:lz4) (Thread[#1323,Execution worker Thread 2,5,main]) started. +work action resolve elasticsearch-cli-9.0.0-SNAPSHOT.jar (project :libs:cli) (Thread[#1323,Execution worker Thread 2,5,main]) started. +work action resolve elasticsearch-plugin-analysis-api-9.0.0-SNAPSHOT.jar (project :libs:plugin-analysis-api) (Thread[#1323,Execution worker Thread 2,5,main]) started. +work action resolve elasticsearch-logging-9.0.0-SNAPSHOT.jar (project :libs:logging) (Thread[#1323,Execution worker Thread 2,5,main]) started. + +> Task :libs:native:native-libraries:extractLibs UP-TO-DATE +Transforming zstd-1.5.5-darwin-aarch64.jar (org.elasticsearch:zstd:1.5.5) with UnzipTransform +Transforming zstd-1.5.5-darwin-x86-64.jar (org.elasticsearch:zstd:1.5.5) with UnzipTransform +Transforming zstd-1.5.5-linux-aarch64.jar (org.elasticsearch:zstd:1.5.5) with UnzipTransform +Transforming zstd-1.5.5-linux-x86-64.jar (org.elasticsearch:zstd:1.5.5) with UnzipTransform +Transforming zstd-1.5.5-windows-x86-64.jar (org.elasticsearch:zstd:1.5.5) with UnzipTransform +Transforming vec-1.0.10.zip (org.elasticsearch:vec:1.0.10) with UnzipTransform +Caching disabled for task ':libs:native:native-libraries:extractLibs' because: + Build cache is disabled + Not worth caching +Skipping task ':libs:native:native-libraries:extractLibs' as it is up-to-date. +work action resolve platform (project :libs:native:native-libraries) (Thread[#1331,Execution worker Thread 10,5,main]) started. + +> Task :libs:native:jar UP-TO-DATE +Caching disabled for task ':libs:native:jar' because: + Build cache is disabled + Not worth caching +Skipping task ':libs:native:jar' as it is up-to-date. +work action resolve elasticsearch-native-9.0.0-SNAPSHOT.jar (project :libs:native) (Thread[#1324,Execution worker Thread 3,5,main]) started. + +> Task :libs:entitlement:asm-provider:jar UP-TO-DATE +Caching disabled for task ':libs:entitlement:asm-provider:jar' because: + Build cache is disabled + Not worth caching +Skipping task ':libs:entitlement:asm-provider:jar' as it is up-to-date. +work action resolve elasticsearch-entitlement-asm-provider-9.0.0-SNAPSHOT.jar (project :libs:entitlement:asm-provider) (Thread[#1330,Execution worker Thread 9,5,main]) started. +UnzipTransform (Thread[#1331,Execution worker Thread 10,5,main]) started. + +> Transform elasticsearch-entitlement-asm-provider-9.0.0-SNAPSHOT.jar (project :libs:entitlement:asm-provider) with UnzipTransform +Transforming elasticsearch-entitlement-asm-provider-9.0.0-SNAPSHOT.jar (project :libs:entitlement:asm-provider) with UnzipTransform +Resolve mutations for :libs:entitlement:generateAsm-providerProviderManifest (Thread[#1331,Execution worker Thread 10,5,main]) started. +:libs:entitlement:generateAsm-providerProviderManifest (Thread[#1331,Execution worker Thread 10,5,main]) started. + +> Task :libs:entitlement:generateAsm-providerProviderManifest UP-TO-DATE +Transforming asm-9.7.1.jar (org.ow2.asm:asm:9.7.1) with UnzipTransform +Caching disabled for task ':libs:entitlement:generateAsm-providerProviderManifest' because: + Build cache is disabled + Caching has not been enabled for the task +Skipping task ':libs:entitlement:generateAsm-providerProviderManifest' as it is up-to-date. +Resolve mutations for :libs:entitlement:generateAsm-providerProviderImpl (Thread[#1331,Execution worker Thread 10,5,main]) started. +:libs:entitlement:generateAsm-providerProviderImpl (Thread[#1331,Execution worker Thread 10,5,main]) started. + +> Task :libs:entitlement:generateAsm-providerProviderImpl UP-TO-DATE +Caching disabled for task ':libs:entitlement:generateAsm-providerProviderImpl' because: + Build cache is disabled + Not worth caching +Skipping task ':libs:entitlement:generateAsm-providerProviderImpl' as it is up-to-date. +Resolve mutations for :libs:entitlement:classes (Thread[#1331,Execution worker Thread 10,5,main]) started. +:libs:entitlement:classes (Thread[#1331,Execution worker Thread 10,5,main]) started. + +> Task :libs:entitlement:classes UP-TO-DATE +Skipping task ':libs:entitlement:classes' as it has no actions. +Resolve mutations for :libs:entitlement:compileMain23Java (Thread[#1331,Execution worker Thread 10,5,main]) started. +:libs:entitlement:compileMain23Java (Thread[#1331,Execution worker Thread 10,5,main]) started. + +> Task :libs:entitlement:compileMain23Java UP-TO-DATE +Caching disabled for task ':libs:entitlement:compileMain23Java' because: + Build cache is disabled +Skipping task ':libs:entitlement:compileMain23Java' as it is up-to-date. +No compile result for :libs:entitlement:compileMain23Java +No compile result for :libs:entitlement:compileMain23Java +No compile result for :libs:entitlement:compileMain23Java + +> Task :libs:simdvec:jar UP-TO-DATE +Caching disabled for task ':libs:simdvec:jar' because: + Build cache is disabled + Not worth caching +Skipping task ':libs:simdvec:jar' as it is up-to-date. +No compile result for :libs:entitlement:compileMain23Java +Resolve mutations for :libs:entitlement:main23Classes (Thread[#1331,Execution worker Thread 10,5,main]) started. +work action resolve elasticsearch-simdvec-9.0.0-SNAPSHOT.jar (project :libs:simdvec) (Thread[#1329,Execution worker Thread 8,5,main]) started. +:libs:entitlement:main23Classes (Thread[#1331,Execution worker Thread 10,5,main]) started. + +> Task :libs:entitlement:main23Classes UP-TO-DATE +Skipping task ':libs:entitlement:main23Classes' as it has no actions. +Resolve mutations for :libs:entitlement:jar (Thread[#1331,Execution worker Thread 10,5,main]) started. +:libs:entitlement:jar (Thread[#1331,Execution worker Thread 10,5,main]) started. + +> Task :libs:secure-sm:jar UP-TO-DATE +Caching disabled for task ':libs:secure-sm:jar' because: + Build cache is disabled + Not worth caching +Skipping task ':libs:secure-sm:jar' as it is up-to-date. +work action resolve elasticsearch-secure-sm-9.0.0-SNAPSHOT.jar (project :libs:secure-sm) (Thread[#1392,included builds Thread 2,5,main]) started. + +> Task :server:jar UP-TO-DATE +Caching disabled for task ':server:jar' because: + Build cache is disabled + Not worth caching +Skipping task ':server:jar' as it is up-to-date. +work action resolve elasticsearch-9.0.0-SNAPSHOT.jar (project :server) (Thread[#1325,Execution worker Thread 4,5,main]) started. + +> Task :libs:ssl-config:jar UP-TO-DATE +Caching disabled for task ':libs:ssl-config:jar' because: + Build cache is disabled + Not worth caching +Skipping task ':libs:ssl-config:jar' as it is up-to-date. +work action resolve elasticsearch-ssl-config-9.0.0-SNAPSHOT.jar (project :libs:ssl-config) (Thread[#1326,Execution worker Thread 5,5,main]) started. + +> Task :libs:tdigest:jar UP-TO-DATE +Caching disabled for task ':libs:tdigest:jar' because: + Build cache is disabled + Not worth caching +Skipping task ':libs:tdigest:jar' as it is up-to-date. +work action resolve elasticsearch-tdigest-9.0.0-SNAPSHOT.jar (project :libs:tdigest) (Thread[#1327,Execution worker Thread 6,5,main]) started. + +> Task :libs:x-content:impl:jar UP-TO-DATE +Caching disabled for task ':libs:x-content:impl:jar' because: + Build cache is disabled + Not worth caching +Skipping task ':libs:x-content:impl:jar' as it is up-to-date. +work action resolve x-content-impl-9.0.0-SNAPSHOT.jar (project :libs:x-content:impl) (Thread[#1328,Execution worker Thread 7,5,main]) started. +UnzipTransform (Thread[#1328,Execution worker Thread 7,5,main]) started. + +> Transform x-content-impl-9.0.0-SNAPSHOT.jar (project :libs:x-content:impl) with UnzipTransform +Transforming x-content-impl-9.0.0-SNAPSHOT.jar (project :libs:x-content:impl) with UnzipTransform +Resolve mutations for :libs:x-content:generateImplProviderManifest (Thread[#1328,Execution worker Thread 7,5,main]) started. +:libs:x-content:generateImplProviderManifest (Thread[#1328,Execution worker Thread 7,5,main]) started. + +> Task :libs:x-content:generateImplProviderManifest UP-TO-DATE +Transforming jackson-core-2.17.2.jar (com.fasterxml.jackson.core:jackson-core:2.17.2) with UnzipTransform +Transforming jackson-dataformat-smile-2.17.2.jar (com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.17.2) with UnzipTransform +Transforming jackson-dataformat-yaml-2.17.2.jar (com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.17.2) with UnzipTransform +Transforming jackson-dataformat-cbor-2.17.2.jar (com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.17.2) with UnzipTransform +Transforming snakeyaml-2.0.jar (org.yaml:snakeyaml:2.0) with UnzipTransform +Caching disabled for task ':libs:x-content:generateImplProviderManifest' because: + Build cache is disabled + Caching has not been enabled for the task +Skipping task ':libs:x-content:generateImplProviderManifest' as it is up-to-date. +Resolve mutations for :libs:x-content:generateImplProviderImpl (Thread[#1328,Execution worker Thread 7,5,main]) started. +:libs:x-content:generateImplProviderImpl (Thread[#1328,Execution worker Thread 7,5,main]) started. + +> Task :libs:x-content:generateImplProviderImpl UP-TO-DATE +Caching disabled for task ':libs:x-content:generateImplProviderImpl' because: + Build cache is disabled + Not worth caching +Skipping task ':libs:x-content:generateImplProviderImpl' as it is up-to-date. +Resolve mutations for :libs:x-content:classes (Thread[#1328,Execution worker Thread 7,5,main]) started. +:libs:x-content:classes (Thread[#1328,Execution worker Thread 7,5,main]) started. + +> Task :libs:x-content:classes UP-TO-DATE +Skipping task ':libs:x-content:classes' as it has no actions. +Resolve mutations for :libs:x-content:jar (Thread[#1328,Execution worker Thread 7,5,main]) started. +:libs:x-content:jar (Thread[#1328,Execution worker Thread 7,5,main]) started. + +> Task :modules:transport-netty4:jar UP-TO-DATE +Caching disabled for task ':modules:transport-netty4:jar' because: + Build cache is disabled + Not worth caching +Skipping task ':modules:transport-netty4:jar' as it is up-to-date. +work action resolve transport-netty4-9.0.0-SNAPSHOT.jar (project :modules:transport-netty4) (Thread[#1332,Execution worker Thread 11,5,main]) started. + +> Task :test:framework:jar UP-TO-DATE +Caching disabled for task ':test:framework:jar' because: + Build cache is disabled + Not worth caching +Skipping task ':test:framework:jar' as it is up-to-date. +work action resolve framework-9.0.0-SNAPSHOT.jar (project :test:framework) (Thread[#1322,Execution worker,5,main]) started. + +> Task :libs:entitlement:jar UP-TO-DATE +Caching disabled for task ':libs:entitlement:jar' because: + Build cache is disabled + Not worth caching +Skipping task ':libs:entitlement:jar' as it is up-to-date. +work action resolve elasticsearch-entitlement-9.0.0-SNAPSHOT.jar (project :libs:entitlement) (Thread[#1331,Execution worker Thread 10,5,main]) started. + +> Task :libs:x-content:jar UP-TO-DATE +Caching disabled for task ':libs:x-content:jar' because: + Build cache is disabled + Not worth caching +Skipping task ':libs:x-content:jar' as it is up-to-date. +work action resolve elasticsearch-x-content-9.0.0-SNAPSHOT.jar (project :libs:x-content) (Thread[#1328,Execution worker Thread 7,5,main]) started. +Resolve mutations for :libs:simdvec:test (Thread[#1328,Execution worker Thread 7,5,main]) started. +:libs:simdvec:test (Thread[#1328,Execution worker Thread 7,5,main]) started. +Gradle Test Executor 18 started executing tests. +Gradle Test Executor 17 started executing tests. +Gradle Test Executor 19 started executing tests. +Gradle Test Executor 19 finished executing tests. +WARNING: A terminally deprecated method in java.lang.System has been called +WARNING: System::setSecurityManager has been called by org.gradle.api.internal.tasks.testing.worker.TestWorker (file:/Users/rene/.gradle/wrapper/dists/gradle-8.11.1-all/2qik7nd48slq1ooc2496ixf4i/gradle-8.11.1/lib/plugins/gradle-testing-base-infrastructure-8.11.1.jar) +WARNING: Please consider reporting this to the maintainers of org.gradle.api.internal.tasks.testing.worker.TestWorker +WARNING: System::setSecurityManager will be removed in a future release +Gradle Test Executor 18 finished executing tests. +WARNING: A terminally deprecated method in java.lang.System has been called +WARNING: System::setSecurityManager has been called by org.gradle.api.internal.tasks.testing.worker.TestWorker (file:/Users/rene/.gradle/wrapper/dists/gradle-8.11.1-all/2qik7nd48slq1ooc2496ixf4i/gradle-8.11.1/lib/plugins/gradle-testing-base-infrastructure-8.11.1.jar) +WARNING: Please consider reporting this to the maintainers of org.gradle.api.internal.tasks.testing.worker.TestWorker +WARNING: System::setSecurityManager will be removed in a future release +Gradle Test Executor 17 finished executing tests. + +> Task :libs:simdvec:test +Caching disabled for task ':libs:simdvec:test' because: + Build cache is disabled +Task ':libs:simdvec:test' is not up-to-date because: + Output property 'binaryResultsDirectory' file /Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/test-results/test/binary has been removed. + Output property 'binaryResultsDirectory' file /Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/test-results/test/binary/output.bin has been removed. + Output property 'binaryResultsDirectory' file /Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/test-results/test/binary/output.bin.idx has been removed. + and more... + and more... +Starting process 'Gradle Test Executor 17'. Working directory: /Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/testrun/test Command: /Users/rene/.gradle/jdks/oracle_corporation-23-aarch64-os_x.2/jdk-23.jdk/Contents/Home/bin/java -Des.nativelibs.path=/Users/rene/dev/elastic/elasticsearch/libs/native/libraries/build/platform/darwin-aarch64 -Des.scripting.update.ctx_in_params=false -Des.search.rewrite_sort=true -Des.transport.cname_in_publish_address=true -Dgradle.dist.lib=/Users/rene/.gradle/wrapper/dists/gradle-8.11.1-all/2qik7nd48slq1ooc2496ixf4i/gradle-8.11.1/lib -Dgradle.user.home=/Users/rene/.gradle -Dgradle.worker.jar=/Users/rene/.gradle/caches/8.11.1/workerMain/gradle-worker.jar -Dio.netty.noKeySetOptimization=true -Dio.netty.noUnsafe=true -Dio.netty.recycler.maxCapacityPerThread=0 -Djava.awt.headless=true -Djava.locale.providers=CLDR -Djava.security.manager=allow -Djna.nosys=true -Dorg.gradle.internal.worker.tmpdir=/Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/tmp/test/work -Dtests.artifact=simdvec -Dtests.gradle=true -Dtests.hackImmutableCollections=true -Dtests.logger.level=WARN -Dtests.security.manager=true -Dtests.seed=7B469FBE8B6D0C65 -Dtests.task=:libs:simdvec:test -Dtests.testfeatures.enabled=true -Dworkspace.dir=/Users/rene/dev/elastic/elasticsearch --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.security.cert=ALL-UNNAMED --add-opens=java.base/java.nio.channels=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/javax.net.ssl=ALL-UNNAMED --add-opens=java.base/java.nio.file=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.management/java.lang.management=ALL-UNNAMED -XX:+HeapDumpOnOutOfMemoryError -esa --add-modules=jdk.incubator.vector -XX:HeapDumpPath=/Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/heapdump --patch-module=java.base=/Users/rene/dev/elastic/elasticsearch/test/immutable-collections-patch/build/jdk-patches/java.base --add-opens=java.base/java.util=ALL-UNNAMED @/Users/rene/.gradle/.tmp/gradle-worker-classpath15834082604113316666txt -Xms512m -Xmx512m -Dfile.encoding=UTF-8 -Djava.io.tmpdir=/Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/testrun/test/temp -Duser.country=DE -Duser.language=en -Duser.variant -ea worker.org.gradle.process.internal.worker.GradleWorkerMain 'Gradle Test Executor 17' +Starting process 'Gradle Test Executor 19'. Working directory: /Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/testrun/test Command: /Users/rene/.gradle/jdks/oracle_corporation-23-aarch64-os_x.2/jdk-23.jdk/Contents/Home/bin/java -Des.nativelibs.path=/Users/rene/dev/elastic/elasticsearch/libs/native/libraries/build/platform/darwin-aarch64 -Des.scripting.update.ctx_in_params=false -Des.search.rewrite_sort=true -Des.transport.cname_in_publish_address=true -Dgradle.dist.lib=/Users/rene/.gradle/wrapper/dists/gradle-8.11.1-all/2qik7nd48slq1ooc2496ixf4i/gradle-8.11.1/lib -Dgradle.user.home=/Users/rene/.gradle -Dgradle.worker.jar=/Users/rene/.gradle/caches/8.11.1/workerMain/gradle-worker.jar -Dio.netty.noKeySetOptimization=true -Dio.netty.noUnsafe=true -Dio.netty.recycler.maxCapacityPerThread=0 -Djava.awt.headless=true -Djava.locale.providers=CLDR -Djava.security.manager=allow -Djna.nosys=true -Dorg.gradle.internal.worker.tmpdir=/Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/tmp/test/work -Dtests.artifact=simdvec -Dtests.gradle=true -Dtests.hackImmutableCollections=true -Dtests.logger.level=WARN -Dtests.security.manager=true -Dtests.seed=7B469FBE8B6D0C65 -Dtests.task=:libs:simdvec:test -Dtests.testfeatures.enabled=true -Dworkspace.dir=/Users/rene/dev/elastic/elasticsearch --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.security.cert=ALL-UNNAMED --add-opens=java.base/java.nio.channels=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/javax.net.ssl=ALL-UNNAMED --add-opens=java.base/java.nio.file=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.management/java.lang.management=ALL-UNNAMED -XX:+HeapDumpOnOutOfMemoryError -esa --add-modules=jdk.incubator.vector -XX:HeapDumpPath=/Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/heapdump --patch-module=java.base=/Users/rene/dev/elastic/elasticsearch/test/immutable-collections-patch/build/jdk-patches/java.base --add-opens=java.base/java.util=ALL-UNNAMED @/Users/rene/.gradle/.tmp/gradle-worker-classpath8760626708071365177txt -Xms512m -Xmx512m -Dfile.encoding=UTF-8 -Djava.io.tmpdir=/Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/testrun/test/temp -Duser.country=DE -Duser.language=en -Duser.variant -ea worker.org.gradle.process.internal.worker.GradleWorkerMain 'Gradle Test Executor 19' +Starting process 'Gradle Test Executor 18'. Working directory: /Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/testrun/test Command: /Users/rene/.gradle/jdks/oracle_corporation-23-aarch64-os_x.2/jdk-23.jdk/Contents/Home/bin/java -Des.nativelibs.path=/Users/rene/dev/elastic/elasticsearch/libs/native/libraries/build/platform/darwin-aarch64 -Des.scripting.update.ctx_in_params=false -Des.search.rewrite_sort=true -Des.transport.cname_in_publish_address=true -Dgradle.dist.lib=/Users/rene/.gradle/wrapper/dists/gradle-8.11.1-all/2qik7nd48slq1ooc2496ixf4i/gradle-8.11.1/lib -Dgradle.user.home=/Users/rene/.gradle -Dgradle.worker.jar=/Users/rene/.gradle/caches/8.11.1/workerMain/gradle-worker.jar -Dio.netty.noKeySetOptimization=true -Dio.netty.noUnsafe=true -Dio.netty.recycler.maxCapacityPerThread=0 -Djava.awt.headless=true -Djava.locale.providers=CLDR -Djava.security.manager=allow -Djna.nosys=true -Dorg.gradle.internal.worker.tmpdir=/Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/tmp/test/work -Dtests.artifact=simdvec -Dtests.gradle=true -Dtests.hackImmutableCollections=true -Dtests.logger.level=WARN -Dtests.security.manager=true -Dtests.seed=7B469FBE8B6D0C65 -Dtests.task=:libs:simdvec:test -Dtests.testfeatures.enabled=true -Dworkspace.dir=/Users/rene/dev/elastic/elasticsearch --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.security.cert=ALL-UNNAMED --add-opens=java.base/java.nio.channels=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/javax.net.ssl=ALL-UNNAMED --add-opens=java.base/java.nio.file=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.management/java.lang.management=ALL-UNNAMED -XX:+HeapDumpOnOutOfMemoryError -esa --add-modules=jdk.incubator.vector -XX:HeapDumpPath=/Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/heapdump --patch-module=java.base=/Users/rene/dev/elastic/elasticsearch/test/immutable-collections-patch/build/jdk-patches/java.base --add-opens=java.base/java.util=ALL-UNNAMED @/Users/rene/.gradle/.tmp/gradle-worker-classpath6774643281130261042txt -Xms512m -Xmx512m -Dfile.encoding=UTF-8 -Djava.io.tmpdir=/Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/testrun/test/temp -Duser.country=DE -Duser.language=en -Duser.variant -ea worker.org.gradle.process.internal.worker.GradleWorkerMain 'Gradle Test Executor 18' +Successfully started process 'Gradle Test Executor 17' +Successfully started process 'Gradle Test Executor 18' +Successfully started process 'Gradle Test Executor 19' +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: Using incubator modules: jdk.incubator.vector + +VectorScorerFactoryTests STANDARD_ERROR + WARNING: A restricted method in + java.lang.foreign.Linker + has been called + WARNING: + java.lang.foreign.Linker::downcallHandle + has been called by + org.elasticsearch.nativeaccess.jdk.LinkerHelper + in + an unnamed module + + WARNING: Use --enable-native-access= + ALL-UNNAMED + to avoid a warning for callers in this module + WARNING: Restricted methods will be blocked in a future release unless native access is enabled + + +ESVectorUtilTests STANDARD_ERROR + WARNING: A restricted method in + java.lang.foreign.Linker + has been called + WARNING: + java.lang.foreign.Linker::downcallHandle + has been called by + org.elasticsearch.nativeaccess.jdk.LinkerHelper + in + an unnamed module + + WARNING: Use --enable-native-access= + ALL-UNNAMED + to avoid a warning for callers in this module + WARNING: Restricted methods will be blocked in a future release unless native access is enabled + + +VectorScorerFactoryTests STANDARD_OUT + [2024-12-23T09:52:40,834][INFO ][o.e.n.j.JdkVectorLibrary ] [[SUITE-VectorScorerFactoryTests-seed#[7B469FBE8B6D0C65]]] vec_caps=1 + [2024-12-23T09:52:40,837][INFO ][o.e.n.NativeAccess ] [[SUITE-VectorScorerFactoryTests-seed#[7B469FBE8B6D0C65]]] Using native vector library; to disable start with -Dorg.elasticsearch.nativeaccess.enableVectorLibrary=false + +ESVectorUtilTests STANDARD_OUT + [2024-12-23T09:52:40,837][INFO ][o.e.n.j.JdkVectorLibrary ] [[SUITE-ESVectorUtilTests-seed#[7B469FBE8B6D0C65]]] vec_caps=1 + [2024-12-23T09:52:40,840][INFO ][o.e.n.NativeAccess ] [[SUITE-ESVectorUtilTests-seed#[7B469FBE8B6D0C65]]] Using native vector library; to disable start with -Dorg.elasticsearch.nativeaccess.enableVectorLibrary=false + +VectorScorerFactoryTests STANDARD_OUT + [2024-12-23T09:52:40,842][INFO ][o.e.n.NativeAccess ] [[SUITE-VectorScorerFactoryTests-seed#[7B469FBE8B6D0C65]]] Using [jdk] native provider and native methods for [MacOS] + +ESVectorUtilTests STANDARD_OUT + [2024-12-23T09:52:40,844][INFO ][o.e.n.NativeAccess ] [[SUITE-ESVectorUtilTests-seed#[7B469FBE8B6D0C65]]] Using [jdk] native provider and native methods for [MacOS] + [2024-12-23T09:52:41,180][INFO ][o.e.s.i.v.ESVectorizationProvider] [[SUITE-ESVectorUtilTests-seed#[7B469FBE8B6D0C65]]] Java vector incubator API enabled; uses preferredBitSize=128 + +ESVectorUtilTests STANDARD_ERROR + Dec 23, 2024 9:52:41 AM org.apache.lucene.internal.vectorization.PanamaVectorizationProvider + INFO: Java vector incubator API enabled; uses preferredBitSize=128 + +VectorScorerFactoryTests STANDARD_ERROR + Dec 23, 2024 9:52:41 AM org.apache.lucene.internal.vectorization.PanamaVectorizationProvider + INFO: Java vector incubator API enabled; uses preferredBitSize=128 + +VectorScorerFactoryTests > testRandomScorerMax STANDARD_OUT + [2024-12-23T04:52:41,272][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerMax] JDK=23, os=Mac OS X, arch=aarch64 + +ESVectorUtilTests > testBitAndCount STANDARD_OUT + [2024-12-23T20:52:41,275][INFO ][o.e.s.ESVectorUtilTests ] [testBitAndCount] before test + [2024-12-23T20:52:41,282][INFO ][o.e.s.i.v.ESVectorizationProvider] [testBitAndCount] Java vector incubator API enabled; uses preferredBitSize=128 + +VectorScorerFactoryTests > testRandomScorerMax STANDARD_OUT + [2024-12-23T04:52:41,282][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerMax] before test + +ESVectorUtilTests > testBitAndCount STANDARD_OUT + [2024-12-23T20:52:41,293][INFO ][o.e.s.ESVectorUtilTests ] [testBitAndCount] after test + +VectorScorerFactoryTests > testRandomScorerMax STANDARD_OUT + [2024-12-23T04:52:41,295][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerMax] Testing testRandom-COSINE-2168.vex + +ESVectorUtilTests > testIpByteBin STANDARD_OUT + [2024-12-23T20:52:41,298][INFO ][o.e.s.ESVectorUtilTests ] [testIpByteBin] before test + +VectorScorerFactoryTests > testRandomScorerMax STANDARD_OUT + [2024-12-23T04:52:41,494][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerMax] Testing testRandom-DOT_PRODUCT-1685.vex + +ESVectorUtilTests > testIpByteBin STANDARD_OUT + [2024-12-23T20:52:41,494][INFO ][o.e.s.ESVectorUtilTests ] [testIpByteBin] after test + +ESVectorUtilTests > testIpByteBinInvariants STANDARD_OUT + [2024-12-23T20:52:41,498][INFO ][o.e.s.ESVectorUtilTests ] [testIpByteBinInvariants] before test + [2024-12-23T20:52:41,499][INFO ][o.e.s.ESVectorUtilTests ] [testIpByteBinInvariants] after test + +ESVectorUtilTests > testBasicIpByteBin STANDARD_OUT + [2024-12-23T20:52:41,502][INFO ][o.e.s.ESVectorUtilTests ] [testBasicIpByteBin] before test + [2024-12-23T20:52:41,502][INFO ][o.e.s.ESVectorUtilTests ] [testBasicIpByteBin] after test + +ESVectorUtilTests > testIpByteBit STANDARD_OUT + [2024-12-23T20:52:41,505][INFO ][o.e.s.ESVectorUtilTests ] [testIpByteBit] before test + [2024-12-23T20:52:41,505][INFO ][o.e.s.ESVectorUtilTests ] [testIpByteBit] after test + +ESVectorUtilTests > testIpFloatBit STANDARD_OUT + [2024-12-23T20:52:41,507][INFO ][o.e.s.ESVectorUtilTests ] [testIpFloatBit] before test + [2024-12-23T20:52:41,508][INFO ][o.e.s.ESVectorUtilTests ] [testIpFloatBit] after test + +VectorScorerFactoryTests > testRandomScorerMax STANDARD_OUT + [2024-12-23T04:52:41,517][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerMax] Testing testRandom-EUCLIDEAN-723.vex + [2024-12-23T04:52:41,592][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerMax] Testing testRandom-MAXIMUM_INNER_PRODUCT-2519.vex + [2024-12-23T04:52:41,628][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerMax] after test + +VectorScorerFactoryTests > testSimpleMaxChunkSizeSmall STANDARD_OUT + [2024-12-23T04:52:41,629][INFO ][o.e.s.VectorScorerFactoryTests] [testSimpleMaxChunkSizeSmall] JDK=23, os=Mac OS X, arch=aarch64 + [2024-12-23T04:52:41,632][INFO ][o.e.s.VectorScorerFactoryTests] [testSimpleMaxChunkSizeSmall] before test + [2024-12-23T04:52:41,632][INFO ][o.e.s.VectorScorerFactoryTests] [testSimpleMaxChunkSizeSmall] maxChunkSize=14 + [2024-12-23T04:52:41,645][INFO ][o.e.s.VectorScorerFactoryTests] [testSimpleMaxChunkSizeSmall] after test + +VectorScorerFactoryTests > testRace STANDARD_OUT + [2024-12-23T04:52:41,645][INFO ][o.e.s.VectorScorerFactoryTests] [testRace] JDK=23, os=Mac OS X, arch=aarch64 + [2024-12-23T04:52:41,648][INFO ][o.e.s.VectorScorerFactoryTests] [testRace] before test + [2024-12-23T04:52:41,659][INFO ][o.e.s.VectorScorerFactoryTests] [testRace] after test + +VectorScorerFactoryTests > testNonNegativeDotProduct STANDARD_OUT + [2024-12-23T04:52:41,659][INFO ][o.e.s.VectorScorerFactoryTests] [testNonNegativeDotProduct] JDK=23, os=Mac OS X, arch=aarch64 + [2024-12-23T04:52:41,662][INFO ][o.e.s.VectorScorerFactoryTests] [testNonNegativeDotProduct] before test + [2024-12-23T04:52:41,663][INFO ][o.e.s.VectorScorerFactoryTests] [testNonNegativeDotProduct] after test + +VectorScorerFactoryTests > testRandomSlice STANDARD_OUT + [2024-12-23T04:52:41,664][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] JDK=23, os=Mac OS X, arch=aarch64 + [2024-12-23T04:52:41,666][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] before test + [2024-12-23T04:52:41,667][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-0-30 + [2024-12-23T04:52:41,669][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-1-30 + [2024-12-23T04:52:41,671][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-2-30 + [2024-12-23T04:52:41,672][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-3-30 + [2024-12-23T04:52:41,674][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-4-30 + [2024-12-23T04:52:41,675][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-5-30 + [2024-12-23T04:52:41,676][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-6-30 + [2024-12-23T04:52:41,677][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-7-30 + [2024-12-23T04:52:41,679][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-8-30 + [2024-12-23T04:52:41,681][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-9-30 + [2024-12-23T04:52:41,682][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-10-30 + [2024-12-23T04:52:41,683][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-11-30 + [2024-12-23T04:52:41,684][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-12-30 + [2024-12-23T04:52:41,685][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-13-30 + [2024-12-23T04:52:41,687][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-14-30 + [2024-12-23T04:52:41,689][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-15-30 + [2024-12-23T04:52:41,690][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-16-30 + [2024-12-23T04:52:41,691][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-17-30 + [2024-12-23T04:52:41,692][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-18-30 + [2024-12-23T04:52:41,693][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-19-30 + [2024-12-23T04:52:41,694][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-20-30 + [2024-12-23T04:52:41,696][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-21-30 + [2024-12-23T04:52:41,697][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-22-30 + [2024-12-23T04:52:41,698][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-23-30 + [2024-12-23T04:52:41,699][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-24-30 + [2024-12-23T04:52:41,700][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-25-30 + [2024-12-23T04:52:41,702][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-26-30 + [2024-12-23T04:52:41,703][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-27-30 + [2024-12-23T04:52:41,703][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-28-30 + [2024-12-23T04:52:41,705][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-29-30 + [2024-12-23T04:52:41,706][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-30-30 + [2024-12-23T04:52:41,707][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-31-30 + [2024-12-23T04:52:41,708][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-32-30 + [2024-12-23T04:52:41,710][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-33-30 + [2024-12-23T04:52:41,711][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-34-30 + [2024-12-23T04:52:41,712][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-35-30 + [2024-12-23T04:52:41,713][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-36-30 + [2024-12-23T04:52:41,715][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-37-30 + [2024-12-23T04:52:41,716][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-38-30 + [2024-12-23T04:52:41,717][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-39-30 + [2024-12-23T04:52:41,718][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-40-30 + [2024-12-23T04:52:41,719][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-41-30 + [2024-12-23T04:52:41,720][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-42-30 + [2024-12-23T04:52:41,722][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-43-30 + [2024-12-23T04:52:41,723][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-44-30 + [2024-12-23T04:52:41,724][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-45-30 + [2024-12-23T04:52:41,725][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-46-30 + [2024-12-23T04:52:41,727][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-47-30 + [2024-12-23T04:52:41,727][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-48-30 + [2024-12-23T04:52:41,728][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-49-30 + [2024-12-23T04:52:41,729][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-50-30 + [2024-12-23T04:52:41,731][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-51-30 + [2024-12-23T04:52:41,732][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-52-30 + [2024-12-23T04:52:41,733][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-53-30 + [2024-12-23T04:52:41,734][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-54-30 + [2024-12-23T04:52:41,735][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-55-30 + [2024-12-23T04:52:41,736][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-56-30 + [2024-12-23T04:52:41,738][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-57-30 + [2024-12-23T04:52:41,739][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-58-30 + [2024-12-23T04:52:41,740][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-59-30 + [2024-12-23T04:52:41,741][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-60-30 + [2024-12-23T04:52:41,742][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-61-30 + [2024-12-23T04:52:41,743][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-62-30 + [2024-12-23T04:52:41,744][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-63-30 + [2024-12-23T04:52:41,746][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-64-30 + [2024-12-23T04:52:41,746][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-65-30 + [2024-12-23T04:52:41,747][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-66-30 + [2024-12-23T04:52:41,749][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-67-30 + [2024-12-23T04:52:41,750][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-68-30 + [2024-12-23T04:52:41,751][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-69-30 + [2024-12-23T04:52:41,752][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-70-30 + [2024-12-23T04:52:41,753][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-71-30 + [2024-12-23T04:52:41,755][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-72-30 + [2024-12-23T04:52:41,755][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-73-30 + [2024-12-23T04:52:41,757][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-74-30 + [2024-12-23T04:52:41,758][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-75-30 + [2024-12-23T04:52:41,759][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-76-30 + [2024-12-23T04:52:41,760][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-77-30 + [2024-12-23T04:52:41,761][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-78-30 + [2024-12-23T04:52:41,761][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-79-30 + [2024-12-23T04:52:41,762][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-80-30 + [2024-12-23T04:52:41,764][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-81-30 + [2024-12-23T04:52:41,764][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-82-30 + [2024-12-23T04:52:41,766][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-83-30 + [2024-12-23T04:52:41,767][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-84-30 + [2024-12-23T04:52:41,768][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-85-30 + [2024-12-23T04:52:41,769][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-86-30 + [2024-12-23T04:52:41,770][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-87-30 + [2024-12-23T04:52:41,771][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-88-30 + [2024-12-23T04:52:41,772][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-89-30 + [2024-12-23T04:52:41,772][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-90-30 + [2024-12-23T04:52:41,774][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-91-30 + [2024-12-23T04:52:41,774][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-92-30 + [2024-12-23T04:52:41,775][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-93-30 + [2024-12-23T04:52:41,776][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-94-30 + [2024-12-23T04:52:41,777][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-95-30 + [2024-12-23T04:52:41,778][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-96-30 + [2024-12-23T04:52:41,779][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-97-30 + [2024-12-23T04:52:41,780][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-98-30 + [2024-12-23T04:52:41,781][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] Testing testRandomSliceImpl-99-30 + [2024-12-23T04:52:41,782][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomSlice] after test + +VectorScorerFactoryTests > testRandomMin STANDARD_OUT + [2024-12-23T04:52:41,783][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomMin] JDK=23, os=Mac OS X, arch=aarch64 + [2024-12-23T04:52:41,785][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomMin] before test + [2024-12-23T04:52:41,786][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomMin] Testing testRandom-279 + [2024-12-23T04:52:41,799][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomMin] after test + +VectorScorerFactoryTests > testRandomMaxChunkSizeSmall STANDARD_OUT + [2024-12-23T04:52:41,799][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomMaxChunkSizeSmall] JDK=23, os=Mac OS X, arch=aarch64 + [2024-12-23T04:52:41,801][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomMaxChunkSizeSmall] before test + [2024-12-23T04:52:41,802][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomMaxChunkSizeSmall] maxChunkSize=118 + [2024-12-23T04:52:41,802][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomMaxChunkSizeSmall] Testing testRandom-3935 + [2024-12-23T04:52:41,916][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomMaxChunkSizeSmall] after test + +VectorScorerFactoryTests > testLarge SKIPPED + +VectorScorerFactoryTests > testRandomMax STANDARD_OUT + [2024-12-23T04:52:41,917][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomMax] JDK=23, os=Mac OS X, arch=aarch64 + [2024-12-23T04:52:41,919][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomMax] before test + [2024-12-23T04:52:41,920][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomMax] Testing testRandom-3531 + [2024-12-23T04:52:41,946][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomMax] after test + +VectorScorerFactoryTests > testRandom STANDARD_OUT + [2024-12-23T04:52:41,946][INFO ][o.e.s.VectorScorerFactoryTests] [testRandom] JDK=23, os=Mac OS X, arch=aarch64 + [2024-12-23T04:52:41,949][INFO ][o.e.s.VectorScorerFactoryTests] [testRandom] before test + [2024-12-23T04:52:41,949][INFO ][o.e.s.VectorScorerFactoryTests] [testRandom] Testing testRandom-1840 + [2024-12-23T04:52:41,980][INFO ][o.e.s.VectorScorerFactoryTests] [testRandom] after test + +VectorScorerFactoryTests > testRandomScorer STANDARD_OUT + [2024-12-23T04:52:41,981][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorer] JDK=23, os=Mac OS X, arch=aarch64 + [2024-12-23T04:52:41,985][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorer] before test + [2024-12-23T04:52:41,986][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorer] Testing testRandom-COSINE-199.vex + [2024-12-23T04:52:41,989][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorer] Testing testRandom-DOT_PRODUCT-3144.vex + [2024-12-23T04:52:41,992][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorer] Testing testRandom-EUCLIDEAN-3017.vex + [2024-12-23T04:52:42,020][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorer] Testing testRandom-MAXIMUM_INNER_PRODUCT-1915.vex + [2024-12-23T04:52:42,025][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorer] after test + +VectorScorerFactoryTests > testSimple STANDARD_OUT + [2024-12-23T04:52:42,025][INFO ][o.e.s.VectorScorerFactoryTests] [testSimple] JDK=23, os=Mac OS X, arch=aarch64 + [2024-12-23T04:52:42,027][INFO ][o.e.s.VectorScorerFactoryTests] [testSimple] before test + [2024-12-23T04:52:42,032][INFO ][o.e.s.VectorScorerFactoryTests] [testSimple] after test + +VectorScorerFactoryTests > testRandomScorerChunkSizeSmall STANDARD_OUT + [2024-12-23T04:52:42,032][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerChunkSizeSmall] JDK=23, os=Mac OS X, arch=aarch64 + [2024-12-23T04:52:42,034][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerChunkSizeSmall] before test + [2024-12-23T04:52:42,034][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerChunkSizeSmall] maxChunkSize=76 + [2024-12-23T04:52:42,035][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerChunkSizeSmall] Testing testRandom-COSINE-2044.vex + [2024-12-23T04:52:42,061][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerChunkSizeSmall] Testing testRandom-DOT_PRODUCT-159.vex + [2024-12-23T04:52:42,063][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerChunkSizeSmall] Testing testRandom-EUCLIDEAN-455.vex + [2024-12-23T04:52:42,069][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerChunkSizeSmall] Testing testRandom-MAXIMUM_INNER_PRODUCT-1088.vex + [2024-12-23T04:52:42,071][INFO ][o.e.s.VectorScorerFactoryTests] [testRandomScorerChunkSizeSmall] after test + +VectorScorerFactoryTests > testSupport STANDARD_OUT + [2024-12-23T04:52:42,072][INFO ][o.e.s.VectorScorerFactoryTests] [testSupport] JDK=23, os=Mac OS X, arch=aarch64 + [2024-12-23T04:52:42,074][INFO ][o.e.s.VectorScorerFactoryTests] [testSupport] before test + [2024-12-23T04:52:42,074][INFO ][o.e.s.VectorScorerFactoryTests] [testSupport] after test +Finished generating test XML results (0.001 secs) into: /Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/test-results/test +Generating HTML test report... +Finished generating test html results (0.002 secs) into: /Users/rene/dev/elastic/elasticsearch/libs/simdvec/build/reports/tests/test + +0 problems were found storing the configuration cache. + +See the complete report at file:///Users/rene/dev/elastic/elasticsearch/build/reports/configuration-cache/bf67ihcjtxveqczwzl7s4owb2/9o6bg5x67n4fyxqy871utzrqc/configuration-cache-report.html + +BUILD SUCCESSFUL in 4s +86 actionable tasks: 2 executed, 84 up-to-date +Watched directory hierarchies: [/Users/rene/dev/elastic/elasticsearch] + +Publishing build scan... +https://gradle-enterprise.elastic.co/s/s75qfwti22ejk + +Configuration cache entry stored. diff --git a/libs/simdvec/src/main/java/module-info.java b/libs/simdvec/src/main/java/module-info.java index 05a2e24d29fca..44f6e39d5dbab 100644 --- a/libs/simdvec/src/main/java/module-info.java +++ b/libs/simdvec/src/main/java/module-info.java @@ -1,14 +1,16 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.simdvec { requires org.elasticsearch.nativeaccess; requires org.apache.lucene.core; + requires org.elasticsearch.logging; exports org.elasticsearch.simdvec to org.elasticsearch.server; } diff --git a/libs/simdvec/src/main/java/org/elasticsearch/simdvec/ESVectorUtil.java b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/ESVectorUtil.java new file mode 100644 index 0000000000000..7fe475e86a2f5 --- /dev/null +++ b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/ESVectorUtil.java @@ -0,0 +1,134 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec; + +import org.apache.lucene.util.BitUtil; +import org.apache.lucene.util.Constants; +import org.elasticsearch.simdvec.internal.vectorization.ESVectorUtilSupport; +import org.elasticsearch.simdvec.internal.vectorization.ESVectorizationProvider; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; + +import static org.elasticsearch.simdvec.internal.vectorization.ESVectorUtilSupport.B_QUERY; + +public class ESVectorUtil { + + private static final MethodHandle BIT_COUNT_MH; + static { + try { + // For xorBitCount we stride over the values as either 64-bits (long) or 32-bits (int) at a time. + // On ARM Long::bitCount is not vectorized, and therefore produces less than optimal code, when + // compared to Integer::bitCount. While Long::bitCount is optimal on x64. See + // https://bugs.openjdk.org/browse/JDK-8336000 + BIT_COUNT_MH = Constants.OS_ARCH.equals("aarch64") + ? MethodHandles.lookup() + .findStatic(ESVectorUtil.class, "andBitCountInt", MethodType.methodType(int.class, byte[].class, byte[].class)) + : MethodHandles.lookup() + .findStatic(ESVectorUtil.class, "andBitCountLong", MethodType.methodType(int.class, byte[].class, byte[].class)); + } catch (NoSuchMethodException | IllegalAccessException e) { + throw new AssertionError(e); + } + } + + private static final ESVectorUtilSupport IMPL = ESVectorizationProvider.getInstance().getVectorUtilSupport(); + + public static long ipByteBinByte(byte[] q, byte[] d) { + if (q.length != d.length * B_QUERY) { + throw new IllegalArgumentException("vector dimensions incompatible: " + q.length + "!= " + B_QUERY + " x " + d.length); + } + return IMPL.ipByteBinByte(q, d); + } + + /** + * Compute the inner product of two vectors, where the query vector is a byte vector and the document vector is a bit vector. + * This will return the sum of the query vector values using the document vector as a mask. + * When comparing the bits with the bytes, they are done in "big endian" order. For example, if the byte vector + * is [1, 2, 3, 4, 5, 6, 7, 8] and the bit vector is [0b10000000], the inner product will be 1.0. + * @param q the query vector + * @param d the document vector + * @return the inner product of the two vectors + */ + public static int ipByteBit(byte[] q, byte[] d) { + if (q.length != d.length * Byte.SIZE) { + throw new IllegalArgumentException("vector dimensions incompatible: " + q.length + "!= " + Byte.SIZE + " x " + d.length); + } + return IMPL.ipByteBit(q, d); + } + + /** + * Compute the inner product of two vectors, where the query vector is a float vector and the document vector is a bit vector. + * This will return the sum of the query vector values using the document vector as a mask. + * When comparing the bits with the floats, they are done in "big endian" order. For example, if the float vector + * is [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0] and the bit vector is [0b10000000], the inner product will be 1.0. + * @param q the query vector + * @param d the document vector + * @return the inner product of the two vectors + */ + public static float ipFloatBit(float[] q, byte[] d) { + if (q.length != d.length * Byte.SIZE) { + throw new IllegalArgumentException("vector dimensions incompatible: " + q.length + "!= " + Byte.SIZE + " x " + d.length); + } + return IMPL.ipFloatBit(q, d); + } + + /** + * AND bit count computed over signed bytes. + * Copied from Lucene's XOR implementation + * @param a bytes containing a vector + * @param b bytes containing another vector, of the same dimension + * @return the value of the AND bit count of the two vectors + */ + public static int andBitCount(byte[] a, byte[] b) { + if (a.length != b.length) { + throw new IllegalArgumentException("vector dimensions differ: " + a.length + "!=" + b.length); + } + try { + return (int) BIT_COUNT_MH.invokeExact(a, b); + } catch (Throwable e) { + if (e instanceof Error err) { + throw err; + } else if (e instanceof RuntimeException re) { + throw re; + } else { + throw new RuntimeException(e); + } + } + } + + /** AND bit count striding over 4 bytes at a time. */ + static int andBitCountInt(byte[] a, byte[] b) { + int distance = 0, i = 0; + // limit to number of int values in the array iterating by int byte views + for (final int upperBound = a.length & -Integer.BYTES; i < upperBound; i += Integer.BYTES) { + distance += Integer.bitCount((int) BitUtil.VH_NATIVE_INT.get(a, i) & (int) BitUtil.VH_NATIVE_INT.get(b, i)); + } + // tail: + for (; i < a.length; i++) { + distance += Integer.bitCount((a[i] & b[i]) & 0xFF); + } + return distance; + } + + /** AND bit count striding over 8 bytes at a time**/ + static int andBitCountLong(byte[] a, byte[] b) { + int distance = 0, i = 0; + // limit to number of long values in the array iterating by long byte views + for (final int upperBound = a.length & -Long.BYTES; i < upperBound; i += Long.BYTES) { + distance += Long.bitCount((long) BitUtil.VH_NATIVE_LONG.get(a, i) & (long) BitUtil.VH_NATIVE_LONG.get(b, i)); + } + // tail: + for (; i < a.length; i++) { + distance += Integer.bitCount((a[i] & b[i]) & 0xFF); + } + return distance; + } +} diff --git a/libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorScorerFactory.java b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorScorerFactory.java index 88c4a59d0ffdb..e2aea6b3ebd9f 100644 --- a/libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorScorerFactory.java +++ b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorScorerFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.simdvec; diff --git a/libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorScorerFactoryImpl.java b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorScorerFactoryImpl.java index b5f5d1ef5c67d..a22d787980252 100644 --- a/libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorScorerFactoryImpl.java +++ b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorScorerFactoryImpl.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.simdvec; diff --git a/libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorSimilarityType.java b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorSimilarityType.java index 0e321771353a3..810c080f15450 100644 --- a/libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorSimilarityType.java +++ b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorSimilarityType.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.simdvec; diff --git a/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/DefaultESVectorUtilSupport.java b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/DefaultESVectorUtilSupport.java new file mode 100644 index 0000000000000..00381c8c3fb2f --- /dev/null +++ b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/DefaultESVectorUtilSupport.java @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec.internal.vectorization; + +import org.apache.lucene.util.BitUtil; +import org.apache.lucene.util.Constants; + +final class DefaultESVectorUtilSupport implements ESVectorUtilSupport { + + private static float fma(float a, float b, float c) { + if (Constants.HAS_FAST_SCALAR_FMA) { + return Math.fma(a, b, c); + } else { + return a * b + c; + } + } + + DefaultESVectorUtilSupport() {} + + @Override + public long ipByteBinByte(byte[] q, byte[] d) { + return ipByteBinByteImpl(q, d); + } + + @Override + public int ipByteBit(byte[] q, byte[] d) { + return ipByteBitImpl(q, d); + } + + @Override + public float ipFloatBit(float[] q, byte[] d) { + return ipFloatBitImpl(q, d); + } + + public static int ipByteBitImpl(byte[] q, byte[] d) { + assert q.length == d.length * Byte.SIZE; + int acc0 = 0; + int acc1 = 0; + int acc2 = 0; + int acc3 = 0; + // now combine the two vectors, summing the byte dimensions where the bit in d is `1` + for (int i = 0; i < d.length; i++) { + byte mask = d[i]; + // Make sure its just 1 or 0 + + acc0 += q[i * Byte.SIZE + 0] * ((mask >> 7) & 1); + acc1 += q[i * Byte.SIZE + 1] * ((mask >> 6) & 1); + acc2 += q[i * Byte.SIZE + 2] * ((mask >> 5) & 1); + acc3 += q[i * Byte.SIZE + 3] * ((mask >> 4) & 1); + + acc0 += q[i * Byte.SIZE + 4] * ((mask >> 3) & 1); + acc1 += q[i * Byte.SIZE + 5] * ((mask >> 2) & 1); + acc2 += q[i * Byte.SIZE + 6] * ((mask >> 1) & 1); + acc3 += q[i * Byte.SIZE + 7] * ((mask >> 0) & 1); + } + return acc0 + acc1 + acc2 + acc3; + } + + public static float ipFloatBitImpl(float[] q, byte[] d) { + assert q.length == d.length * Byte.SIZE; + float acc0 = 0; + float acc1 = 0; + float acc2 = 0; + float acc3 = 0; + // now combine the two vectors, summing the byte dimensions where the bit in d is `1` + for (int i = 0; i < d.length; i++) { + byte mask = d[i]; + acc0 = fma(q[i * Byte.SIZE + 0], (mask >> 7) & 1, acc0); + acc1 = fma(q[i * Byte.SIZE + 1], (mask >> 6) & 1, acc1); + acc2 = fma(q[i * Byte.SIZE + 2], (mask >> 5) & 1, acc2); + acc3 = fma(q[i * Byte.SIZE + 3], (mask >> 4) & 1, acc3); + + acc0 = fma(q[i * Byte.SIZE + 4], (mask >> 3) & 1, acc0); + acc1 = fma(q[i * Byte.SIZE + 5], (mask >> 2) & 1, acc1); + acc2 = fma(q[i * Byte.SIZE + 6], (mask >> 1) & 1, acc2); + acc3 = fma(q[i * Byte.SIZE + 7], (mask >> 0) & 1, acc3); + } + return acc0 + acc1 + acc2 + acc3; + } + + public static long ipByteBinByteImpl(byte[] q, byte[] d) { + long ret = 0; + int size = d.length; + for (int i = 0; i < B_QUERY; i++) { + int r = 0; + long subRet = 0; + for (final int upperBound = d.length & -Integer.BYTES; r < upperBound; r += Integer.BYTES) { + subRet += Integer.bitCount((int) BitUtil.VH_NATIVE_INT.get(q, i * size + r) & (int) BitUtil.VH_NATIVE_INT.get(d, r)); + } + for (; r < d.length; r++) { + subRet += Integer.bitCount((q[i * size + r] & d[r]) & 0xFF); + } + ret += subRet << i; + } + return ret; + } +} diff --git a/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/DefaultESVectorizationProvider.java b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/DefaultESVectorizationProvider.java new file mode 100644 index 0000000000000..6c0f7ed146b86 --- /dev/null +++ b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/DefaultESVectorizationProvider.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec.internal.vectorization; + +final class DefaultESVectorizationProvider extends ESVectorizationProvider { + private final ESVectorUtilSupport vectorUtilSupport; + + DefaultESVectorizationProvider() { + vectorUtilSupport = new DefaultESVectorUtilSupport(); + } + + @Override + public ESVectorUtilSupport getVectorUtilSupport() { + return vectorUtilSupport; + } +} diff --git a/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorUtilSupport.java b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorUtilSupport.java new file mode 100644 index 0000000000000..6938bffec5f37 --- /dev/null +++ b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorUtilSupport.java @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec.internal.vectorization; + +public interface ESVectorUtilSupport { + + short B_QUERY = 4; + + long ipByteBinByte(byte[] q, byte[] d); + + int ipByteBit(byte[] q, byte[] d); + + float ipFloatBit(float[] q, byte[] d); +} diff --git a/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorizationProvider.java b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorizationProvider.java new file mode 100644 index 0000000000000..e541c10e145bf --- /dev/null +++ b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorizationProvider.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec.internal.vectorization; + +import java.util.Objects; + +public abstract class ESVectorizationProvider { + + public static ESVectorizationProvider getInstance() { + return Objects.requireNonNull( + ESVectorizationProvider.Holder.INSTANCE, + "call to getInstance() from subclass of VectorizationProvider" + ); + } + + ESVectorizationProvider() {} + + public abstract ESVectorUtilSupport getVectorUtilSupport(); + + // visible for tests + static ESVectorizationProvider lookup(boolean testMode) { + return new DefaultESVectorizationProvider(); + } + + /** This static holder class prevents classloading deadlock. */ + private static final class Holder { + private Holder() {} + + static final ESVectorizationProvider INSTANCE = lookup(false); + } +} diff --git a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/VectorScorerFactoryImpl.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/VectorScorerFactoryImpl.java index 7c120d53a28ff..a65fe582087d9 100644 --- a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/VectorScorerFactoryImpl.java +++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/VectorScorerFactoryImpl.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.simdvec; diff --git a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorer.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorer.java index bdb4f22b3ade2..0b41436ce2242 100644 --- a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorer.java +++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.simdvec.internal; diff --git a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorerSupplier.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorerSupplier.java index b1410b03cd8ce..f6d874cd3e728 100644 --- a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorerSupplier.java +++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorerSupplier.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.simdvec.internal; diff --git a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Similarities.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Similarities.java index eea319541437b..482bbc8d8cabe 100644 --- a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Similarities.java +++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Similarities.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.simdvec.internal; diff --git a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorizationProvider.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorizationProvider.java new file mode 100644 index 0000000000000..5b7aab7ddfa48 --- /dev/null +++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorizationProvider.java @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec.internal.vectorization; + +import org.apache.lucene.util.Constants; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; + +import java.util.Locale; +import java.util.Objects; +import java.util.Optional; + +public abstract class ESVectorizationProvider { + + protected static final Logger logger = LogManager.getLogger(ESVectorizationProvider.class); + + public static ESVectorizationProvider getInstance() { + return Objects.requireNonNull( + ESVectorizationProvider.Holder.INSTANCE, + "call to getInstance() from subclass of VectorizationProvider" + ); + } + + ESVectorizationProvider() {} + + public abstract ESVectorUtilSupport getVectorUtilSupport(); + + // visible for tests + static ESVectorizationProvider lookup(boolean testMode) { + final int runtimeVersion = Runtime.version().feature(); + assert runtimeVersion >= 21; + if (runtimeVersion <= 23) { + // only use vector module with Hotspot VM + if (Constants.IS_HOTSPOT_VM == false) { + logger.warn("Java runtime is not using Hotspot VM; Java vector incubator API can't be enabled."); + return new DefaultESVectorizationProvider(); + } + // is the incubator module present and readable (JVM providers may to exclude them or it is + // build with jlink) + final var vectorMod = lookupVectorModule(); + if (vectorMod.isEmpty()) { + logger.warn( + "Java vector incubator module is not readable. " + + "For optimal vector performance, pass '--add-modules jdk.incubator.vector' to enable Vector API." + ); + return new DefaultESVectorizationProvider(); + } + vectorMod.ifPresent(ESVectorizationProvider.class.getModule()::addReads); + var impl = new PanamaESVectorizationProvider(); + logger.info( + String.format( + Locale.ENGLISH, + "Java vector incubator API enabled; uses preferredBitSize=%d", + PanamaESVectorUtilSupport.VECTOR_BITSIZE + ) + ); + return impl; + } else { + logger.warn( + "You are running with unsupported Java " + + runtimeVersion + + ". To make full use of the Vector API, please update Elasticsearch." + ); + } + return new DefaultESVectorizationProvider(); + } + + private static Optional lookupVectorModule() { + return Optional.ofNullable(ESVectorizationProvider.class.getModule().getLayer()) + .orElse(ModuleLayer.boot()) + .findModule("jdk.incubator.vector"); + } + + /** This static holder class prevents classloading deadlock. */ + private static final class Holder { + private Holder() {} + + static final ESVectorizationProvider INSTANCE = lookup(false); + } +} diff --git a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorUtilSupport.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorUtilSupport.java new file mode 100644 index 0000000000000..4de33643258e4 --- /dev/null +++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorUtilSupport.java @@ -0,0 +1,163 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec.internal.vectorization; + +import jdk.incubator.vector.ByteVector; +import jdk.incubator.vector.IntVector; +import jdk.incubator.vector.LongVector; +import jdk.incubator.vector.VectorOperators; +import jdk.incubator.vector.VectorShape; +import jdk.incubator.vector.VectorSpecies; + +import org.apache.lucene.util.Constants; + +public final class PanamaESVectorUtilSupport implements ESVectorUtilSupport { + + static final int VECTOR_BITSIZE; + + /** Whether integer vectors can be trusted to actually be fast. */ + static final boolean HAS_FAST_INTEGER_VECTORS; + + static { + // default to platform supported bitsize + VECTOR_BITSIZE = VectorShape.preferredShape().vectorBitSize(); + + // hotspot misses some SSE intrinsics, workaround it + // to be fair, they do document this thing only works well with AVX2/AVX3 and Neon + boolean isAMD64withoutAVX2 = Constants.OS_ARCH.equals("amd64") && VECTOR_BITSIZE < 256; + HAS_FAST_INTEGER_VECTORS = isAMD64withoutAVX2 == false; + } + + @Override + public long ipByteBinByte(byte[] q, byte[] d) { + // 128 / 8 == 16 + if (d.length >= 16 && HAS_FAST_INTEGER_VECTORS) { + if (VECTOR_BITSIZE >= 256) { + return ipByteBin256(q, d); + } else if (VECTOR_BITSIZE == 128) { + return ipByteBin128(q, d); + } + } + return DefaultESVectorUtilSupport.ipByteBinByteImpl(q, d); + } + + @Override + public int ipByteBit(byte[] q, byte[] d) { + return DefaultESVectorUtilSupport.ipByteBitImpl(q, d); + } + + @Override + public float ipFloatBit(float[] q, byte[] d) { + return DefaultESVectorUtilSupport.ipFloatBitImpl(q, d); + } + + private static final VectorSpecies BYTE_SPECIES_128 = ByteVector.SPECIES_128; + private static final VectorSpecies BYTE_SPECIES_256 = ByteVector.SPECIES_256; + + static long ipByteBin256(byte[] q, byte[] d) { + long subRet0 = 0; + long subRet1 = 0; + long subRet2 = 0; + long subRet3 = 0; + int i = 0; + + if (d.length >= ByteVector.SPECIES_256.vectorByteSize() * 2) { + int limit = ByteVector.SPECIES_256.loopBound(d.length); + var sum0 = LongVector.zero(LongVector.SPECIES_256); + var sum1 = LongVector.zero(LongVector.SPECIES_256); + var sum2 = LongVector.zero(LongVector.SPECIES_256); + var sum3 = LongVector.zero(LongVector.SPECIES_256); + for (; i < limit; i += ByteVector.SPECIES_256.length()) { + var vq0 = ByteVector.fromArray(BYTE_SPECIES_256, q, i).reinterpretAsLongs(); + var vq1 = ByteVector.fromArray(BYTE_SPECIES_256, q, i + d.length).reinterpretAsLongs(); + var vq2 = ByteVector.fromArray(BYTE_SPECIES_256, q, i + d.length * 2).reinterpretAsLongs(); + var vq3 = ByteVector.fromArray(BYTE_SPECIES_256, q, i + d.length * 3).reinterpretAsLongs(); + var vd = ByteVector.fromArray(BYTE_SPECIES_256, d, i).reinterpretAsLongs(); + sum0 = sum0.add(vq0.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum1 = sum1.add(vq1.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum2 = sum2.add(vq2.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum3 = sum3.add(vq3.and(vd).lanewise(VectorOperators.BIT_COUNT)); + } + subRet0 += sum0.reduceLanes(VectorOperators.ADD); + subRet1 += sum1.reduceLanes(VectorOperators.ADD); + subRet2 += sum2.reduceLanes(VectorOperators.ADD); + subRet3 += sum3.reduceLanes(VectorOperators.ADD); + } + + if (d.length - i >= ByteVector.SPECIES_128.vectorByteSize()) { + var sum0 = LongVector.zero(LongVector.SPECIES_128); + var sum1 = LongVector.zero(LongVector.SPECIES_128); + var sum2 = LongVector.zero(LongVector.SPECIES_128); + var sum3 = LongVector.zero(LongVector.SPECIES_128); + int limit = ByteVector.SPECIES_128.loopBound(d.length); + for (; i < limit; i += ByteVector.SPECIES_128.length()) { + var vq0 = ByteVector.fromArray(BYTE_SPECIES_128, q, i).reinterpretAsLongs(); + var vq1 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + d.length).reinterpretAsLongs(); + var vq2 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + d.length * 2).reinterpretAsLongs(); + var vq3 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + d.length * 3).reinterpretAsLongs(); + var vd = ByteVector.fromArray(BYTE_SPECIES_128, d, i).reinterpretAsLongs(); + sum0 = sum0.add(vq0.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum1 = sum1.add(vq1.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum2 = sum2.add(vq2.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum3 = sum3.add(vq3.and(vd).lanewise(VectorOperators.BIT_COUNT)); + } + subRet0 += sum0.reduceLanes(VectorOperators.ADD); + subRet1 += sum1.reduceLanes(VectorOperators.ADD); + subRet2 += sum2.reduceLanes(VectorOperators.ADD); + subRet3 += sum3.reduceLanes(VectorOperators.ADD); + } + // tail as bytes + for (; i < d.length; i++) { + subRet0 += Integer.bitCount((q[i] & d[i]) & 0xFF); + subRet1 += Integer.bitCount((q[i + d.length] & d[i]) & 0xFF); + subRet2 += Integer.bitCount((q[i + 2 * d.length] & d[i]) & 0xFF); + subRet3 += Integer.bitCount((q[i + 3 * d.length] & d[i]) & 0xFF); + } + return subRet0 + (subRet1 << 1) + (subRet2 << 2) + (subRet3 << 3); + } + + public static long ipByteBin128(byte[] q, byte[] d) { + long subRet0 = 0; + long subRet1 = 0; + long subRet2 = 0; + long subRet3 = 0; + int i = 0; + + var sum0 = IntVector.zero(IntVector.SPECIES_128); + var sum1 = IntVector.zero(IntVector.SPECIES_128); + var sum2 = IntVector.zero(IntVector.SPECIES_128); + var sum3 = IntVector.zero(IntVector.SPECIES_128); + int limit = ByteVector.SPECIES_128.loopBound(d.length); + for (; i < limit; i += ByteVector.SPECIES_128.length()) { + var vd = ByteVector.fromArray(BYTE_SPECIES_128, d, i).reinterpretAsInts(); + var vq0 = ByteVector.fromArray(BYTE_SPECIES_128, q, i).reinterpretAsInts(); + var vq1 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + d.length).reinterpretAsInts(); + var vq2 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + d.length * 2).reinterpretAsInts(); + var vq3 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + d.length * 3).reinterpretAsInts(); + sum0 = sum0.add(vd.and(vq0).lanewise(VectorOperators.BIT_COUNT)); + sum1 = sum1.add(vd.and(vq1).lanewise(VectorOperators.BIT_COUNT)); + sum2 = sum2.add(vd.and(vq2).lanewise(VectorOperators.BIT_COUNT)); + sum3 = sum3.add(vd.and(vq3).lanewise(VectorOperators.BIT_COUNT)); + } + subRet0 += sum0.reduceLanes(VectorOperators.ADD); + subRet1 += sum1.reduceLanes(VectorOperators.ADD); + subRet2 += sum2.reduceLanes(VectorOperators.ADD); + subRet3 += sum3.reduceLanes(VectorOperators.ADD); + // tail as bytes + for (; i < d.length; i++) { + int dValue = d[i]; + subRet0 += Integer.bitCount((dValue & q[i]) & 0xFF); + subRet1 += Integer.bitCount((dValue & q[i + d.length]) & 0xFF); + subRet2 += Integer.bitCount((dValue & q[i + 2 * d.length]) & 0xFF); + subRet3 += Integer.bitCount((dValue & q[i + 3 * d.length]) & 0xFF); + } + return subRet0 + (subRet1 << 1) + (subRet2 << 2) + (subRet3 << 3); + } +} diff --git a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorizationProvider.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorizationProvider.java new file mode 100644 index 0000000000000..62d25d79487ed --- /dev/null +++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorizationProvider.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec.internal.vectorization; + +final class PanamaESVectorizationProvider extends ESVectorizationProvider { + + private final ESVectorUtilSupport vectorUtilSupport; + + PanamaESVectorizationProvider() { + vectorUtilSupport = new PanamaESVectorUtilSupport(); + } + + @Override + public ESVectorUtilSupport getVectorUtilSupport() { + return vectorUtilSupport; + } +} diff --git a/libs/simdvec/src/main22/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorer.java b/libs/simdvec/src/main22/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorer.java index 90a7e5a23dd4c..c9659ea1af9a8 100644 --- a/libs/simdvec/src/main22/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorer.java +++ b/libs/simdvec/src/main22/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.simdvec.internal; diff --git a/libs/simdvec/src/test/java/org/elasticsearch/simdvec/AbstractVectorTestCase.java b/libs/simdvec/src/test/java/org/elasticsearch/simdvec/AbstractVectorTestCase.java index 1734bef80389d..31c5daa81f92b 100644 --- a/libs/simdvec/src/test/java/org/elasticsearch/simdvec/AbstractVectorTestCase.java +++ b/libs/simdvec/src/test/java/org/elasticsearch/simdvec/AbstractVectorTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.simdvec; diff --git a/libs/simdvec/src/test/java/org/elasticsearch/simdvec/ESVectorUtilTests.java b/libs/simdvec/src/test/java/org/elasticsearch/simdvec/ESVectorUtilTests.java new file mode 100644 index 0000000000000..368898b934c87 --- /dev/null +++ b/libs/simdvec/src/test/java/org/elasticsearch/simdvec/ESVectorUtilTests.java @@ -0,0 +1,175 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec; + +import org.elasticsearch.simdvec.internal.vectorization.BaseVectorizationTests; +import org.elasticsearch.simdvec.internal.vectorization.ESVectorizationProvider; + +import java.util.Arrays; + +import static org.elasticsearch.simdvec.internal.vectorization.ESVectorUtilSupport.B_QUERY; + +public class ESVectorUtilTests extends BaseVectorizationTests { + + static final ESVectorizationProvider defaultedProvider = BaseVectorizationTests.defaultProvider(); + static final ESVectorizationProvider defOrPanamaProvider = BaseVectorizationTests.maybePanamaProvider(); + + public void testIpByteBit() { + byte[] q = new byte[16]; + byte[] d = new byte[] { (byte) Integer.parseInt("01100010", 2), (byte) Integer.parseInt("10100111", 2) }; + random().nextBytes(q); + int expected = q[1] + q[2] + q[6] + q[8] + q[10] + q[13] + q[14] + q[15]; + assertEquals(expected, ESVectorUtil.ipByteBit(q, d)); + } + + public void testIpFloatBit() { + float[] q = new float[16]; + byte[] d = new byte[] { (byte) Integer.parseInt("01100010", 2), (byte) Integer.parseInt("10100111", 2) }; + random().nextFloat(); + float expected = q[1] + q[2] + q[6] + q[8] + q[10] + q[13] + q[14] + q[15]; + assertEquals(expected, ESVectorUtil.ipFloatBit(q, d), 1e-6); + } + + public void testBitAndCount() { + testBasicBitAndImpl(ESVectorUtil::andBitCountLong); + } + + public void testIpByteBinInvariants() { + int iterations = atLeast(10); + for (int i = 0; i < iterations; i++) { + int size = randomIntBetween(1, 10); + var d = new byte[size]; + var q = new byte[size * B_QUERY - 1]; + expectThrows(IllegalArgumentException.class, () -> ESVectorUtil.ipByteBinByte(q, d)); + } + } + + public void testBasicIpByteBin() { + testBasicIpByteBinImpl(ESVectorUtil::ipByteBinByte); + testBasicIpByteBinImpl(defaultedProvider.getVectorUtilSupport()::ipByteBinByte); + testBasicIpByteBinImpl(defOrPanamaProvider.getVectorUtilSupport()::ipByteBinByte); + } + + interface IpByteBin { + long apply(byte[] q, byte[] d); + } + + interface BitOps { + long apply(byte[] q, byte[] d); + } + + void testBasicBitAndImpl(BitOps bitAnd) { + assertEquals(0, bitAnd.apply(new byte[] { 0 }, new byte[] { 0 })); + assertEquals(0, bitAnd.apply(new byte[] { 1 }, new byte[] { 0 })); + assertEquals(0, bitAnd.apply(new byte[] { 0 }, new byte[] { 1 })); + assertEquals(1, bitAnd.apply(new byte[] { 1 }, new byte[] { 1 })); + byte[] a = new byte[31]; + byte[] b = new byte[31]; + random().nextBytes(a); + random().nextBytes(b); + int expected = scalarBitAnd(a, b); + assertEquals(expected, bitAnd.apply(a, b)); + } + + void testBasicIpByteBinImpl(IpByteBin ipByteBinFunc) { + assertEquals(15L, ipByteBinFunc.apply(new byte[] { 1, 1, 1, 1 }, new byte[] { 1 })); + assertEquals(30L, ipByteBinFunc.apply(new byte[] { 1, 2, 1, 2, 1, 2, 1, 2 }, new byte[] { 1, 2 })); + + var d = new byte[] { 1, 2, 3 }; + var q = new byte[] { 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3 }; + assert scalarIpByteBin(q, d) == 60L; // 4 + 8 + 16 + 32 + assertEquals(60L, ipByteBinFunc.apply(q, d)); + + d = new byte[] { 1, 2, 3, 4 }; + q = new byte[] { 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4 }; + assert scalarIpByteBin(q, d) == 75L; // 5 + 10 + 20 + 40 + assertEquals(75L, ipByteBinFunc.apply(q, d)); + + d = new byte[] { 1, 2, 3, 4, 5 }; + q = new byte[] { 1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5 }; + assert scalarIpByteBin(q, d) == 105L; // 7 + 14 + 28 + 56 + assertEquals(105L, ipByteBinFunc.apply(q, d)); + + d = new byte[] { 1, 2, 3, 4, 5, 6 }; + q = new byte[] { 1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6 }; + assert scalarIpByteBin(q, d) == 135L; // 9 + 18 + 36 + 72 + assertEquals(135L, ipByteBinFunc.apply(q, d)); + + d = new byte[] { 1, 2, 3, 4, 5, 6, 7 }; + q = new byte[] { 1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7 }; + assert scalarIpByteBin(q, d) == 180L; // 12 + 24 + 48 + 96 + assertEquals(180L, ipByteBinFunc.apply(q, d)); + + d = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 }; + q = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 }; + assert scalarIpByteBin(q, d) == 195L; // 13 + 26 + 52 + 104 + assertEquals(195L, ipByteBinFunc.apply(q, d)); + + d = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + q = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + assert scalarIpByteBin(q, d) == 225L; // 15 + 30 + 60 + 120 + assertEquals(225L, ipByteBinFunc.apply(q, d)); + } + + public void testIpByteBin() { + testIpByteBinImpl(ESVectorUtil::ipByteBinByte); + testIpByteBinImpl(defaultedProvider.getVectorUtilSupport()::ipByteBinByte); + testIpByteBinImpl(defOrPanamaProvider.getVectorUtilSupport()::ipByteBinByte); + } + + void testIpByteBinImpl(IpByteBin ipByteBinFunc) { + int iterations = atLeast(50); + for (int i = 0; i < iterations; i++) { + int size = random().nextInt(5000); + var d = new byte[size]; + var q = new byte[size * B_QUERY]; + random().nextBytes(d); + random().nextBytes(q); + assertEquals(scalarIpByteBin(q, d), ipByteBinFunc.apply(q, d)); + + Arrays.fill(d, Byte.MAX_VALUE); + Arrays.fill(q, Byte.MAX_VALUE); + assertEquals(scalarIpByteBin(q, d), ipByteBinFunc.apply(q, d)); + + Arrays.fill(d, Byte.MIN_VALUE); + Arrays.fill(q, Byte.MIN_VALUE); + assertEquals(scalarIpByteBin(q, d), ipByteBinFunc.apply(q, d)); + } + } + + static int scalarIpByteBin(byte[] q, byte[] d) { + int res = 0; + for (int i = 0; i < B_QUERY; i++) { + res += (popcount(q, i * d.length, d, d.length) << i); + } + return res; + } + + static int scalarBitAnd(byte[] a, byte[] b) { + int res = 0; + for (int i = 0; i < a.length; i++) { + res += Integer.bitCount((a[i] & b[i]) & 0xFF); + } + return res; + } + + public static int popcount(byte[] a, int aOffset, byte[] b, int length) { + int res = 0; + for (int j = 0; j < length; j++) { + int value = (a[aOffset + j] & b[j]) & 0xFF; + for (int k = 0; k < Byte.SIZE; k++) { + if ((value & (1 << k)) != 0) { + ++res; + } + } + } + return res; + } +} diff --git a/libs/simdvec/src/test/java/org/elasticsearch/simdvec/VectorScorerFactoryTests.java b/libs/simdvec/src/test/java/org/elasticsearch/simdvec/VectorScorerFactoryTests.java index 93c6da73f4179..db57dc936e794 100644 --- a/libs/simdvec/src/test/java/org/elasticsearch/simdvec/VectorScorerFactoryTests.java +++ b/libs/simdvec/src/test/java/org/elasticsearch/simdvec/VectorScorerFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.simdvec; @@ -236,6 +237,8 @@ void testRandomScorerImpl(long maxChunkSize, Function floatArr try (Directory dir = new MMapDirectory(createTempDir("testRandom"), maxChunkSize)) { for (var sim : List.of(COSINE, DOT_PRODUCT, EUCLIDEAN, MAXIMUM_INNER_PRODUCT)) { + // Use the random supplier for COSINE, which returns values in the normalized range + floatArraySupplier = sim == COSINE ? FLOAT_ARRAY_RANDOM_FUNC : floatArraySupplier; final int dims = randomIntBetween(1, 4096); final int size = randomIntBetween(2, 100); final float[][] vectors = new float[size][]; diff --git a/libs/simdvec/src/test/java/org/elasticsearch/simdvec/internal/vectorization/BaseVectorizationTests.java b/libs/simdvec/src/test/java/org/elasticsearch/simdvec/internal/vectorization/BaseVectorizationTests.java new file mode 100644 index 0000000000000..f2bc8a11b04aa --- /dev/null +++ b/libs/simdvec/src/test/java/org/elasticsearch/simdvec/internal/vectorization/BaseVectorizationTests.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec.internal.vectorization; + +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +public class BaseVectorizationTests extends ESTestCase { + + @Before + public void sanity() { + assert Runtime.version().feature() < 21 || ModuleLayer.boot().findModule("jdk.incubator.vector").isPresent(); + } + + public static ESVectorizationProvider defaultProvider() { + return new DefaultESVectorizationProvider(); + } + + public static ESVectorizationProvider maybePanamaProvider() { + return ESVectorizationProvider.lookup(true); + } +} diff --git a/libs/ssl-config/build.gradle b/libs/ssl-config/build.gradle index 4079d9d632709..d63df95003ab6 100644 --- a/libs/ssl-config/build.gradle +++ b/libs/ssl-config/build.gradle @@ -1,17 +1,18 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: "elasticsearch.publish" dependencies { - api project(':libs:elasticsearch-core') + api project(':libs:core') testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-ssl-config' + exclude group: 'org.elasticsearch', module: 'ssl-config' } testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" diff --git a/libs/ssl-config/src/main/java/module-info.java b/libs/ssl-config/src/main/java/module-info.java index 2bcdd8533745e..172ab6de193b3 100644 --- a/libs/ssl-config/src/main/java/module-info.java +++ b/libs/ssl-config/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.sslconfig { diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/CompositeTrustConfig.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/CompositeTrustConfig.java index 98c71a9888974..0b2bf66810586 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/CompositeTrustConfig.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/CompositeTrustConfig.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/DefaultJdkTrustConfig.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/DefaultJdkTrustConfig.java index 1ed506a8813a3..f6e330972ad19 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/DefaultJdkTrustConfig.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/DefaultJdkTrustConfig.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/DiagnosticTrustManager.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/DiagnosticTrustManager.java index 96bfa65707dbf..9737d8e38fc41 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/DiagnosticTrustManager.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/DiagnosticTrustManager.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/EmptyKeyConfig.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/EmptyKeyConfig.java index 363b47dbdc4bd..f02bd8d5c81b0 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/EmptyKeyConfig.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/EmptyKeyConfig.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/KeyStoreUtil.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/KeyStoreUtil.java index 7f5b005e28470..85bd5368b208c 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/KeyStoreUtil.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/KeyStoreUtil.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/PemKeyConfig.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/PemKeyConfig.java index 765a648bcd727..6b708fa086cd0 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/PemKeyConfig.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/PemKeyConfig.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/PemTrustConfig.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/PemTrustConfig.java index f1d7a7b6f4120..1c7295d6acc88 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/PemTrustConfig.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/PemTrustConfig.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/PemUtils.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/PemUtils.java index 9bb0643907eb5..b7418a96f180c 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/PemUtils.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/PemUtils.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslClientAuthenticationMode.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslClientAuthenticationMode.java index 03754caf58049..a27c4a6cb4d68 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslClientAuthenticationMode.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslClientAuthenticationMode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfigException.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfigException.java index 09d4fd2e6fa92..d87634d2615fa 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfigException.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfigException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfiguration.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfiguration.java index 610fb444e0a93..30d846da46156 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfiguration.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfiguration.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfigurationKeys.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfigurationKeys.java index 83dbd919a60e9..1c782a2fa5f31 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfigurationKeys.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfigurationKeys.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfigurationLoader.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfigurationLoader.java index f0f1bbfd9ea72..79bfaec6a6d11 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfigurationLoader.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfigurationLoader.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslDiagnostics.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslDiagnostics.java index e3fb59f79b41a..6cf5cfcce139a 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslDiagnostics.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslDiagnostics.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslFileUtil.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslFileUtil.java index 6a475e4a8d431..b5eee13550493 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslFileUtil.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslFileUtil.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslKeyConfig.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslKeyConfig.java index 10545329d8440..595a193572c95 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslKeyConfig.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslKeyConfig.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslTrustConfig.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslTrustConfig.java index d0e594dfc7de0..8e405d5479230 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslTrustConfig.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslTrustConfig.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslUtil.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslUtil.java index 841f46d13a135..4cf34622d596c 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslUtil.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslUtil.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslVerificationMode.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslVerificationMode.java index 1329ca276d678..cc04803bafadd 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslVerificationMode.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslVerificationMode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/StoreKeyConfig.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/StoreKeyConfig.java index ac040a27c3cbb..7952b7a89ed9a 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/StoreKeyConfig.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/StoreKeyConfig.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/StoreTrustConfig.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/StoreTrustConfig.java index 7c330aa1c2d8c..0d5c28e652f32 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/StoreTrustConfig.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/StoreTrustConfig.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/StoredCertificate.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/StoredCertificate.java index 9b66b04acdc44..73f5ab06f503c 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/StoredCertificate.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/StoredCertificate.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/TrustEverythingConfig.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/TrustEverythingConfig.java index 568dbcc58846d..32e8470de94cb 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/TrustEverythingConfig.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/TrustEverythingConfig.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/X509Field.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/X509Field.java index 64de9adb8c222..b562d112329a3 100644 --- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/X509Field.java +++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/X509Field.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/DefaultJdkTrustConfigTests.java b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/DefaultJdkTrustConfigTests.java index d8457ee16bff9..39f167fc45191 100644 --- a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/DefaultJdkTrustConfigTests.java +++ b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/DefaultJdkTrustConfigTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/KeyStoreUtilTests.java b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/KeyStoreUtilTests.java index c586ead12b002..7874869930fe5 100644 --- a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/KeyStoreUtilTests.java +++ b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/KeyStoreUtilTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/PemKeyConfigTests.java b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/PemKeyConfigTests.java index 0f695c3c525a2..b25e0c9c9af95 100644 --- a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/PemKeyConfigTests.java +++ b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/PemKeyConfigTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/PemTrustConfigTests.java b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/PemTrustConfigTests.java index af669e76bf86b..f4de800407c4f 100644 --- a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/PemTrustConfigTests.java +++ b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/PemTrustConfigTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/PemUtilsTests.java b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/PemUtilsTests.java index ca795f6b13433..fd96948275319 100644 --- a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/PemUtilsTests.java +++ b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/PemUtilsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/SslConfigurationLoaderTests.java b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/SslConfigurationLoaderTests.java index 5ec0d129b8f95..a42f041237619 100644 --- a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/SslConfigurationLoaderTests.java +++ b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/SslConfigurationLoaderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/SslConfigurationTests.java b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/SslConfigurationTests.java index d7917380aa8c5..735edddd284bd 100644 --- a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/SslConfigurationTests.java +++ b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/SslConfigurationTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/SslDiagnosticsTests.java b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/SslDiagnosticsTests.java index 57b1f62532590..ba0b67e93da61 100644 --- a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/SslDiagnosticsTests.java +++ b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/SslDiagnosticsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/StoreKeyConfigTests.java b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/StoreKeyConfigTests.java index 7c84229a88a54..f2ad3e8e33f1d 100644 --- a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/StoreKeyConfigTests.java +++ b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/StoreKeyConfigTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/StoreTrustConfigTests.java b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/StoreTrustConfigTests.java index b7f30cd4c8522..dd89d8d1b34e1 100644 --- a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/StoreTrustConfigTests.java +++ b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/StoreTrustConfigTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/X509FieldTests.java b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/X509FieldTests.java index a0de14aec887d..1efed2fea408c 100644 --- a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/X509FieldTests.java +++ b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/X509FieldTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.ssl; diff --git a/libs/tdigest/build.gradle b/libs/tdigest/build.gradle index 771df2e83d85d..b79a6ce0a486a 100644 --- a/libs/tdigest/build.gradle +++ b/libs/tdigest/build.gradle @@ -22,8 +22,11 @@ apply plugin: 'elasticsearch.build' apply plugin: 'elasticsearch.publish' dependencies { + api project(':libs:core') + api "org.apache.lucene:lucene-core:${versions.lucene}" + testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-tdigest' + exclude group: 'org.elasticsearch', module: 'tdigest' } testImplementation 'org.junit.jupiter:junit-jupiter:5.8.1' } @@ -33,7 +36,7 @@ tasks.named('forbiddenApisMain').configure { replaceSignatureFiles 'jdk-signatures' } -ext.projectLicenses.set(['The Apache Software License, Version 2.0': 'http://www.apache.org/licenses/LICENSE-2.0']) +ext.projectLicenses.set(['The Apache Software License, Version 2.0': providers.provider(() -> 'http://www.apache.org/licenses/LICENSE-2.0')]) licenseFile.set(rootProject.file('licenses/APACHE-LICENSE-2.0.txt')) tasks.withType(LicenseHeadersTask.class).configureEach { diff --git a/libs/tdigest/licenses/lucene-core-LICENSE.txt b/libs/tdigest/licenses/lucene-core-LICENSE.txt new file mode 100644 index 0000000000000..28b134f5f8e4d --- /dev/null +++ b/libs/tdigest/licenses/lucene-core-LICENSE.txt @@ -0,0 +1,475 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + +Some code in core/src/java/org/apache/lucene/util/UnicodeUtil.java was +derived from unicode conversion examples available at +http://www.unicode.org/Public/PROGRAMS/CVTUTF. Here is the copyright +from those sources: + +/* + * Copyright 2001-2004 Unicode, Inc. + * + * Disclaimer + * + * This source code is provided as is by Unicode, Inc. No claims are + * made as to fitness for any particular purpose. No warranties of any + * kind are expressed or implied. The recipient agrees to determine + * applicability of information provided. If this file has been + * purchased on magnetic or optical media from Unicode, Inc., the + * sole remedy for any claim will be exchange of defective media + * within 90 days of receipt. + * + * Limitations on Rights to Redistribute This Code + * + * Unicode, Inc. hereby grants the right to freely use the information + * supplied in this file in the creation of products supporting the + * Unicode Standard, and to make copies of this file in any form + * for internal or external distribution as long as this notice + * remains attached. + */ + + +Some code in core/src/java/org/apache/lucene/util/ArrayUtil.java was +derived from Python 2.4.2 sources available at +http://www.python.org. Full license is here: + + http://www.python.org/download/releases/2.4.2/license/ + +Some code in core/src/java/org/apache/lucene/util/UnicodeUtil.java was +derived from Python 3.1.2 sources available at +http://www.python.org. Full license is here: + + http://www.python.org/download/releases/3.1.2/license/ + +Some code in core/src/java/org/apache/lucene/util/automaton was +derived from Brics automaton sources available at +www.brics.dk/automaton/. Here is the copyright from those sources: + +/* + * Copyright (c) 2001-2009 Anders Moeller + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +The levenshtein automata tables in core/src/java/org/apache/lucene/util/automaton +were automatically generated with the moman/finenight FSA package. +Here is the copyright for those sources: + +# Copyright (c) 2010, Jean-Philippe Barrette-LaPierre, +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +Some code in core/src/java/org/apache/lucene/util/UnicodeUtil.java was +derived from ICU (http://www.icu-project.org) +The full license is available here: + http://source.icu-project.org/repos/icu/icu/trunk/license.html + +/* + * Copyright (C) 1999-2010, International Business Machines + * Corporation and others. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * provided that the above copyright notice(s) and this permission notice appear + * in all copies of the Software and that both the above copyright notice(s) and + * this permission notice appear in supporting documentation. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. + * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE BE + * LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR + * ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER + * IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT + * OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + * Except as contained in this notice, the name of a copyright holder shall not + * be used in advertising or otherwise to promote the sale, use or other + * dealings in this Software without prior written authorization of the + * copyright holder. + */ + +The following license applies to the Snowball stemmers: + +Copyright (c) 2001, Dr Martin Porter +Copyright (c) 2002, Richard Boulton +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holders nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The following license applies to the KStemmer: + +Copyright © 2003, +Center for Intelligent Information Retrieval, +University of Massachusetts, Amherst. +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +3. The names "Center for Intelligent Information Retrieval" and +"University of Massachusetts" must not be used to endorse or promote products +derived from this software without prior written permission. To obtain +permission, contact info@ciir.cs.umass.edu. + +THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF MASSACHUSETTS AND OTHER CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + +The following license applies to the Morfologik project: + +Copyright (c) 2006 Dawid Weiss +Copyright (c) 2007-2011 Dawid Weiss, Marcin Miłkowski +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name of Morfologik nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +--- + +The dictionary comes from Morfologik project. Morfologik uses data from +Polish ispell/myspell dictionary hosted at http://www.sjp.pl/slownik/en/ and +is licenced on the terms of (inter alia) LGPL and Creative Commons +ShareAlike. The part-of-speech tags were added in Morfologik project and +are not found in the data from sjp.pl. The tagset is similar to IPI PAN +tagset. + +--- + +The following license applies to the Morfeusz project, +used by org.apache.lucene.analysis.morfologik. + +BSD-licensed dictionary of Polish (SGJP) +http://sgjp.pl/morfeusz/ + +Copyright © 2011 Zygmunt Saloni, Włodzimierz Gruszczyński, + Marcin Woliński, Robert Wołosz + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the + distribution. + +THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDERS “AS IS” AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL COPYRIGHT HOLDERS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/libs/tdigest/licenses/lucene-core-NOTICE.txt b/libs/tdigest/licenses/lucene-core-NOTICE.txt new file mode 100644 index 0000000000000..1a1d51572432a --- /dev/null +++ b/libs/tdigest/licenses/lucene-core-NOTICE.txt @@ -0,0 +1,192 @@ +Apache Lucene +Copyright 2014 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Includes software from other Apache Software Foundation projects, +including, but not limited to: + - Apache Ant + - Apache Jakarta Regexp + - Apache Commons + - Apache Xerces + +ICU4J, (under analysis/icu) is licensed under an MIT styles license +and Copyright (c) 1995-2008 International Business Machines Corporation and others + +Some data files (under analysis/icu/src/data) are derived from Unicode data such +as the Unicode Character Database. See http://unicode.org/copyright.html for more +details. + +Brics Automaton (under core/src/java/org/apache/lucene/util/automaton) is +BSD-licensed, created by Anders Møller. See http://www.brics.dk/automaton/ + +The levenshtein automata tables (under core/src/java/org/apache/lucene/util/automaton) were +automatically generated with the moman/finenight FSA library, created by +Jean-Philippe Barrette-LaPierre. This library is available under an MIT license, +see http://sites.google.com/site/rrettesite/moman and +http://bitbucket.org/jpbarrette/moman/overview/ + +The class org.apache.lucene.util.WeakIdentityMap was derived from +the Apache CXF project and is Apache License 2.0. + +The Google Code Prettify is Apache License 2.0. +See http://code.google.com/p/google-code-prettify/ + +JUnit (junit-4.10) is licensed under the Common Public License v. 1.0 +See http://junit.sourceforge.net/cpl-v10.html + +This product includes code (JaspellTernarySearchTrie) from Java Spelling Checkin +g Package (jaspell): http://jaspell.sourceforge.net/ +License: The BSD License (http://www.opensource.org/licenses/bsd-license.php) + +The snowball stemmers in + analysis/common/src/java/net/sf/snowball +were developed by Martin Porter and Richard Boulton. +The snowball stopword lists in + analysis/common/src/resources/org/apache/lucene/analysis/snowball +were developed by Martin Porter and Richard Boulton. +The full snowball package is available from + http://snowball.tartarus.org/ + +The KStem stemmer in + analysis/common/src/org/apache/lucene/analysis/en +was developed by Bob Krovetz and Sergio Guzman-Lara (CIIR-UMass Amherst) +under the BSD-license. + +The Arabic,Persian,Romanian,Bulgarian, Hindi and Bengali analyzers (common) come with a default +stopword list that is BSD-licensed created by Jacques Savoy. These files reside in: +analysis/common/src/resources/org/apache/lucene/analysis/ar/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/fa/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/ro/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/bg/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/hi/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/bn/stopwords.txt +See http://members.unine.ch/jacques.savoy/clef/index.html. + +The German,Spanish,Finnish,French,Hungarian,Italian,Portuguese,Russian and Swedish light stemmers +(common) are based on BSD-licensed reference implementations created by Jacques Savoy and +Ljiljana Dolamic. These files reside in: +analysis/common/src/java/org/apache/lucene/analysis/de/GermanLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/de/GermanMinimalStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/es/SpanishLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/fi/FinnishLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchMinimalStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/hu/HungarianLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/it/ItalianLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/sv/SwedishLightStemmer.java + +The Stempel analyzer (stempel) includes BSD-licensed software developed +by the Egothor project http://egothor.sf.net/, created by Leo Galambos, Martin Kvapil, +and Edmond Nolan. + +The Polish analyzer (stempel) comes with a default +stopword list that is BSD-licensed created by the Carrot2 project. The file resides +in stempel/src/resources/org/apache/lucene/analysis/pl/stopwords.txt. +See http://project.carrot2.org/license.html. + +The SmartChineseAnalyzer source code (smartcn) was +provided by Xiaoping Gao and copyright 2009 by www.imdict.net. + +WordBreakTestUnicode_*.java (under modules/analysis/common/src/test/) +is derived from Unicode data such as the Unicode Character Database. +See http://unicode.org/copyright.html for more details. + +The Morfologik analyzer (morfologik) includes BSD-licensed software +developed by Dawid Weiss and Marcin Miłkowski (http://morfologik.blogspot.com/). + +Morfologik uses data from Polish ispell/myspell dictionary +(http://www.sjp.pl/slownik/en/) licenced on the terms of (inter alia) +LGPL and Creative Commons ShareAlike. + +Morfologic includes data from BSD-licensed dictionary of Polish (SGJP) +(http://sgjp.pl/morfeusz/) + +Servlet-api.jar and javax.servlet-*.jar are under the CDDL license, the original +source code for this can be found at http://www.eclipse.org/jetty/downloads.php + +=========================================================================== +Kuromoji Japanese Morphological Analyzer - Apache Lucene Integration +=========================================================================== + +This software includes a binary and/or source version of data from + + mecab-ipadic-2.7.0-20070801 + +which can be obtained from + + http://atilika.com/releases/mecab-ipadic/mecab-ipadic-2.7.0-20070801.tar.gz + +or + + http://jaist.dl.sourceforge.net/project/mecab/mecab-ipadic/2.7.0-20070801/mecab-ipadic-2.7.0-20070801.tar.gz + +=========================================================================== +mecab-ipadic-2.7.0-20070801 Notice +=========================================================================== + +Nara Institute of Science and Technology (NAIST), +the copyright holders, disclaims all warranties with regard to this +software, including all implied warranties of merchantability and +fitness, in no event shall NAIST be liable for +any special, indirect or consequential damages or any damages +whatsoever resulting from loss of use, data or profits, whether in an +action of contract, negligence or other tortuous action, arising out +of or in connection with the use or performance of this software. + +A large portion of the dictionary entries +originate from ICOT Free Software. The following conditions for ICOT +Free Software applies to the current dictionary as well. + +Each User may also freely distribute the Program, whether in its +original form or modified, to any third party or parties, PROVIDED +that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear +on, or be attached to, the Program, which is distributed substantially +in the same form as set out herein and that such intended +distribution, if actually made, will neither violate or otherwise +contravene any of the laws and regulations of the countries having +jurisdiction over the User or the intended distribution itself. + +NO WARRANTY + +The program was produced on an experimental basis in the course of the +research and development conducted during the project and is provided +to users as so produced on an experimental basis. Accordingly, the +program is provided without any warranty whatsoever, whether express, +implied, statutory or otherwise. The term "warranty" used herein +includes, but is not limited to, any warranty of the quality, +performance, merchantability and fitness for a particular purpose of +the program and the nonexistence of any infringement or violation of +any right of any third party. + +Each user of the program will agree and understand, and be deemed to +have agreed and understood, that there is no warranty whatsoever for +the program and, accordingly, the entire risk arising from or +otherwise connected with the program is assumed by the user. + +Therefore, neither ICOT, the copyright holder, or any other +organization that participated in or was otherwise related to the +development of the program and their respective officials, directors, +officers and other employees shall be held liable for any and all +damages, including, without limitation, general, special, incidental +and consequential damages, arising out of or otherwise in connection +with the use or inability to use the program or any product, material +or result produced or otherwise obtained by using the program, +regardless of whether they have been advised of, or otherwise had +knowledge of, the possibility of such damages at any time during the +project or thereafter. Each user will be deemed to have agreed to the +foregoing by his or her commencement of use of the program. The term +"use" as used herein includes, but is not limited to, the use, +modification, copying and distribution of the program and the +production of secondary products from the program. + +In the case where the program, whether in its original form or +modified, was distributed or delivered to or received by a user from +any person, organization or entity other than ICOT, unless it makes or +grants independently of ICOT any specific warranty to the user in +writing, such person, organization or entity, will also be exempted +from and not be held liable to the user for any such damages as noted +above as far as the program is concerned. diff --git a/libs/tdigest/src/main/java/module-info.java b/libs/tdigest/src/main/java/module-info.java index 994ff41187221..79ddbe88ab3d3 100644 --- a/libs/tdigest/src/main/java/module-info.java +++ b/libs/tdigest/src/main/java/module-info.java @@ -18,5 +18,9 @@ */ module org.elasticsearch.tdigest { + requires org.elasticsearch.base; + requires org.apache.lucene.core; + exports org.elasticsearch.tdigest; + exports org.elasticsearch.tdigest.arrays; } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLGroupTree.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLGroupTree.java index 12b2a29d3e034..66b4acc68db84 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLGroupTree.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLGroupTree.java @@ -21,65 +21,123 @@ package org.elasticsearch.tdigest; +import org.apache.lucene.util.Accountable; +import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.tdigest.arrays.TDigestArrays; +import org.elasticsearch.tdigest.arrays.TDigestDoubleArray; +import org.elasticsearch.tdigest.arrays.TDigestLongArray; + import java.util.AbstractCollection; -import java.util.Arrays; import java.util.Iterator; /** * A tree of t-digest centroids. */ -final class AVLGroupTree extends AbstractCollection { +final class AVLGroupTree extends AbstractCollection implements Releasable, Accountable { + private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(AVLGroupTree.class); + + private final TDigestArrays arrays; + private boolean closed = false; + /* For insertions into the tree */ private double centroid; private long count; - private double[] centroids; - private long[] counts; - private long[] aggregatedCounts; + private final TDigestDoubleArray centroids; + private final TDigestLongArray counts; + private final TDigestLongArray aggregatedCounts; private final IntAVLTree tree; - AVLGroupTree() { - tree = new IntAVLTree() { + static AVLGroupTree create(TDigestArrays arrays) { + arrays.adjustBreaker(SHALLOW_SIZE); + try { + return new AVLGroupTree(arrays); + } catch (Exception e) { + arrays.adjustBreaker(-SHALLOW_SIZE); + throw e; + } + } - @Override - protected void resize(int newCapacity) { - super.resize(newCapacity); - centroids = Arrays.copyOf(centroids, newCapacity); - counts = Arrays.copyOf(counts, newCapacity); - aggregatedCounts = Arrays.copyOf(aggregatedCounts, newCapacity); - } + private AVLGroupTree(TDigestArrays arrays) { + this.arrays = arrays; + + IntAVLTree tree = null; + TDigestDoubleArray centroids = null; + TDigestLongArray counts = null; + TDigestLongArray aggregatedCounts = null; + + try { + this.tree = tree = createIntAvlTree(arrays); + this.centroids = centroids = arrays.newDoubleArray(tree.capacity()); + this.counts = counts = arrays.newLongArray(tree.capacity()); + this.aggregatedCounts = aggregatedCounts = arrays.newLongArray(tree.capacity()); + + tree = null; + centroids = null; + counts = null; + aggregatedCounts = null; + } finally { + Releasables.close(tree, centroids, counts, aggregatedCounts); + } + } - @Override - protected void merge(int node) { - // two nodes are never considered equal - throw new UnsupportedOperationException(); - } + private IntAVLTree createIntAvlTree(TDigestArrays arrays) { + arrays.adjustBreaker(IntAVLTree.SHALLOW_SIZE); + try { + return new InternalIntAvlTree(arrays); + } catch (Exception e) { + arrays.adjustBreaker(-IntAVLTree.SHALLOW_SIZE); + throw e; + } + } - @Override - protected void copy(int node) { - centroids[node] = centroid; - counts[node] = count; - } + private class InternalIntAvlTree extends IntAVLTree { + private InternalIntAvlTree(TDigestArrays arrays) { + super(arrays); + } - @Override - protected int compare(int node) { - if (centroid < centroids[node]) { - return -1; - } else { - // upon equality, the newly added node is considered greater - return 1; - } - } + @Override + protected void resize(int newCapacity) { + super.resize(newCapacity); + centroids.resize(newCapacity); + counts.resize(newCapacity); + aggregatedCounts.resize(newCapacity); + } - @Override - protected void fixAggregates(int node) { - super.fixAggregates(node); - aggregatedCounts[node] = counts[node] + aggregatedCounts[left(node)] + aggregatedCounts[right(node)]; + @Override + protected void merge(int node) { + // two nodes are never considered equal + throw new UnsupportedOperationException(); + } + + @Override + protected void copy(int node) { + centroids.set(node, centroid); + counts.set(node, count); + } + + @Override + protected int compare(int node) { + if (centroid < centroids.get(node)) { + return -1; + } else { + // upon equality, the newly added node is considered greater + return 1; } + } - }; - centroids = new double[tree.capacity()]; - counts = new long[tree.capacity()]; - aggregatedCounts = new long[tree.capacity()]; + @Override + protected void fixAggregates(int node) { + super.fixAggregates(node); + aggregatedCounts.set(node, counts.get(node) + aggregatedCounts.get(left(node)) + aggregatedCounts.get(right(node))); + } + + } + + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + centroids.ramBytesUsed() + counts.ramBytesUsed() + aggregatedCounts.ramBytesUsed() + tree.ramBytesUsed(); } /** @@ -107,14 +165,14 @@ public int next(int node) { * Return the mean for the provided node. */ public double mean(int node) { - return centroids[node]; + return centroids.get(node); } /** * Return the count for the provided node. */ public long count(int node) { - return counts[node]; + return counts.get(node); } /** @@ -167,7 +225,7 @@ public int floorSum(long sum) { int floor = IntAVLTree.NIL; for (int node = tree.root(); node != IntAVLTree.NIL;) { final int left = tree.left(node); - final long leftCount = aggregatedCounts[left]; + final long leftCount = aggregatedCounts.get(left); if (leftCount <= sum) { floor = node; sum -= leftCount + count(node); @@ -199,11 +257,11 @@ public int last() { */ public long headSum(int node) { final int left = tree.left(node); - long sum = aggregatedCounts[left]; + long sum = aggregatedCounts.get(left); for (int n = node, p = tree.parent(node); p != IntAVLTree.NIL; n = p, p = tree.parent(n)) { if (n == tree.right(p)) { final int leftP = tree.left(p); - sum += counts[p] + aggregatedCounts[leftP]; + sum += counts.get(p) + aggregatedCounts.get(leftP); } } return sum; @@ -243,7 +301,7 @@ public void remove() { * Return the total count of points that have been added to the tree. */ public long sum() { - return aggregatedCounts[tree.root()]; + return aggregatedCounts.get(tree.root()); } void checkBalance() { @@ -255,11 +313,21 @@ void checkAggregates() { } private void checkAggregates(int node) { - assert aggregatedCounts[node] == counts[node] + aggregatedCounts[tree.left(node)] + aggregatedCounts[tree.right(node)]; + assert aggregatedCounts.get(node) == counts.get(node) + aggregatedCounts.get(tree.left(node)) + aggregatedCounts.get( + tree.right(node) + ); if (node != IntAVLTree.NIL) { checkAggregates(tree.left(node)); checkAggregates(tree.right(node)); } } + @Override + public void close() { + if (closed == false) { + closed = true; + arrays.adjustBreaker(-SHALLOW_SIZE); + Releasables.close(centroids, counts, aggregatedCounts, tree); + } + } } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLTreeDigest.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLTreeDigest.java index deb3407565f36..8350f17d255cd 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLTreeDigest.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLTreeDigest.java @@ -21,6 +21,10 @@ package org.elasticsearch.tdigest; +import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.tdigest.arrays.TDigestArrays; + import java.util.Collection; import java.util.Collections; import java.util.Iterator; @@ -29,6 +33,11 @@ import static org.elasticsearch.tdigest.IntAVLTree.NIL; public class AVLTreeDigest extends AbstractTDigest { + private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(AVLTreeDigest.class); + + private final TDigestArrays arrays; + private boolean closed = false; + final Random gen = new Random(); private final double compression; private AVLGroupTree summary; @@ -38,6 +47,16 @@ public class AVLTreeDigest extends AbstractTDigest { // Indicates if a sample has been added after the last compression. private boolean needsCompression; + static AVLTreeDigest create(TDigestArrays arrays, double compression) { + arrays.adjustBreaker(SHALLOW_SIZE); + try { + return new AVLTreeDigest(arrays, compression); + } catch (Exception e) { + arrays.adjustBreaker(-SHALLOW_SIZE); + throw e; + } + } + /** * A histogram structure that will record a sketch of a distribution. * @@ -46,14 +65,20 @@ public class AVLTreeDigest extends AbstractTDigest { * quantiles. Conversely, you should expect to track about 5 N centroids for this * accuracy. */ - public AVLTreeDigest(double compression) { + private AVLTreeDigest(TDigestArrays arrays, double compression) { + this.arrays = arrays; this.compression = compression; - summary = new AVLGroupTree(); + summary = AVLGroupTree.create(arrays); + } + + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + summary.ramBytesUsed(); } /** * Sets the seed for the RNG. - * In cases where a predicatable tree should be created, this function may be used to make the + * In cases where a predictable tree should be created, this function may be used to make the * randomness in this AVLTree become more deterministic. * * @param seed The random seed to use for RNG purposes @@ -148,26 +173,27 @@ public void compress() { } needsCompression = false; - AVLGroupTree centroids = summary; - this.summary = new AVLGroupTree(); + try (AVLGroupTree centroids = summary) { + this.summary = AVLGroupTree.create(arrays); - final int[] nodes = new int[centroids.size()]; - nodes[0] = centroids.first(); - for (int i = 1; i < nodes.length; ++i) { - nodes[i] = centroids.next(nodes[i - 1]); - assert nodes[i] != IntAVLTree.NIL; - } - assert centroids.next(nodes[nodes.length - 1]) == IntAVLTree.NIL; + final int[] nodes = new int[centroids.size()]; + nodes[0] = centroids.first(); + for (int i = 1; i < nodes.length; ++i) { + nodes[i] = centroids.next(nodes[i - 1]); + assert nodes[i] != IntAVLTree.NIL; + } + assert centroids.next(nodes[nodes.length - 1]) == IntAVLTree.NIL; - for (int i = centroids.size() - 1; i > 0; --i) { - final int other = gen.nextInt(i + 1); - final int tmp = nodes[other]; - nodes[other] = nodes[i]; - nodes[i] = tmp; - } + for (int i = centroids.size() - 1; i > 0; --i) { + final int other = gen.nextInt(i + 1); + final int tmp = nodes[other]; + nodes[other] = nodes[i]; + nodes[i] = tmp; + } - for (int node : nodes) { - add(centroids.mean(node), centroids.count(node)); + for (int node : nodes) { + add(centroids.mean(node), centroids.count(node)); + } } } @@ -351,4 +377,13 @@ public int byteSize() { compress(); return 64 + summary.size() * 13; } + + @Override + public void close() { + if (closed == false) { + closed = true; + arrays.adjustBreaker(-SHALLOW_SIZE); + Releasables.close(summary); + } + } } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/Dist.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/Dist.java index 087deaedc7d75..02fb7a8376aa4 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/Dist.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/Dist.java @@ -21,6 +21,8 @@ package org.elasticsearch.tdigest; +import org.elasticsearch.tdigest.arrays.TDigestDoubleArray; + import java.util.List; import java.util.function.Function; @@ -102,6 +104,10 @@ public static double cdf(final double x, List data) { return cdf(x, data.size(), data::get); } + public static double cdf(final double x, TDigestDoubleArray data) { + return cdf(x, data.size(), data::get); + } + private static double quantile(final double q, final int length, Function elementGetter) { if (length == 0) { return Double.NaN; @@ -133,4 +139,8 @@ public static double quantile(final double q, double[] data) { public static double quantile(final double q, List data) { return quantile(q, data.size(), data::get); } + + public static double quantile(final double q, TDigestDoubleArray data) { + return quantile(q, data.size(), data::get); + } } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/HybridDigest.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/HybridDigest.java index 07a12381e2a71..cf743db49acdd 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/HybridDigest.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/HybridDigest.java @@ -19,6 +19,10 @@ package org.elasticsearch.tdigest; +import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.tdigest.arrays.TDigestArrays; + import java.util.Collection; /** @@ -31,6 +35,10 @@ * bounded memory allocation and acceptable speed and accuracy for larger ones. */ public class HybridDigest extends AbstractTDigest { + private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(HybridDigest.class); + + private final TDigestArrays arrays; + private boolean closed = false; // See MergingDigest's compression param. private final double compression; @@ -39,11 +47,21 @@ public class HybridDigest extends AbstractTDigest { private final long maxSortingSize; // This is set to null when the implementation switches to MergingDigest. - private SortingDigest sortingDigest = new SortingDigest(); + private SortingDigest sortingDigest; // This gets initialized when the implementation switches to MergingDigest. private MergingDigest mergingDigest; + static HybridDigest create(TDigestArrays arrays, double compression) { + arrays.adjustBreaker(SHALLOW_SIZE); + try { + return new HybridDigest(arrays, compression); + } catch (Exception e) { + arrays.adjustBreaker(-SHALLOW_SIZE); + throw e; + } + } + /** * Creates a hybrid digest that uses a {@link SortingDigest} for up to {@param maxSortingSize} samples, * then switches to a {@link MergingDigest}. @@ -51,9 +69,11 @@ public class HybridDigest extends AbstractTDigest { * @param compression The compression factor for the MergingDigest * @param maxSortingSize The sample size limit for switching from a {@link SortingDigest} to a {@link MergingDigest} implementation */ - HybridDigest(double compression, long maxSortingSize) { + private HybridDigest(TDigestArrays arrays, double compression, long maxSortingSize) { + this.arrays = arrays; this.compression = compression; this.maxSortingSize = maxSortingSize; + this.sortingDigest = TDigest.createSortingDigest(arrays); } /** @@ -62,11 +82,18 @@ public class HybridDigest extends AbstractTDigest { * * @param compression The compression factor for the MergingDigest */ - HybridDigest(double compression) { + private HybridDigest(TDigestArrays arrays, double compression) { // The default maxSortingSize is calculated so that the SortingDigest will have comparable size with the MergingDigest // at the point where implementations switch, e.g. for default compression 100 SortingDigest allocates ~16kB and MergingDigest // allocates ~15kB. - this(compression, Math.round(compression) * 20); + this(arrays, compression, Math.round(compression) * 20); + } + + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + (sortingDigest != null ? sortingDigest.ramBytesUsed() : 0) + (mergingDigest != null + ? mergingDigest.ramBytesUsed() + : 0); } @Override @@ -98,12 +125,13 @@ public void reserve(long size) { // Check if we need to switch implementations. assert sortingDigest != null; if (sortingDigest.size() + size >= maxSortingSize) { - mergingDigest = new MergingDigest(compression); - for (double value : sortingDigest.values) { - mergingDigest.add(value); + mergingDigest = TDigest.createMergingDigest(arrays, compression); + for (int i = 0; i < sortingDigest.values.size(); i++) { + mergingDigest.add(sortingDigest.values.get(i)); } mergingDigest.reserve(size); // Release the allocated SortingDigest. + sortingDigest.close(); sortingDigest = null; } else { sortingDigest.reserve(size); @@ -190,4 +218,13 @@ public int byteSize() { } return sortingDigest.byteSize(); } + + @Override + public void close() { + if (closed == false) { + closed = true; + arrays.adjustBreaker(-SHALLOW_SIZE); + Releasables.close(sortingDigest, mergingDigest); + } + } } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/IntAVLTree.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/IntAVLTree.java index f2f71b13a2eef..c86642e757caa 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/IntAVLTree.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/IntAVLTree.java @@ -21,7 +21,13 @@ package org.elasticsearch.tdigest; -import java.util.Arrays; +import org.apache.lucene.util.Accountable; +import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.tdigest.arrays.TDigestArrays; +import org.elasticsearch.tdigest.arrays.TDigestByteArray; +import org.elasticsearch.tdigest.arrays.TDigestIntArray; /** * An AVL-tree structure stored in parallel arrays. @@ -29,8 +35,8 @@ * want to add data to the nodes, typically by using arrays and node * identifiers as indices. */ -abstract class IntAVLTree { - +abstract class IntAVLTree implements Releasable, Accountable { + static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(IntAVLTree.class); /** * We use 0 instead of -1 so that left(NIL) works without * condition. @@ -42,24 +48,51 @@ static int oversize(int size) { return size + (size >>> 3); } + private final TDigestArrays arrays; + private boolean closed = false; + private final NodeAllocator nodeAllocator; private int root; - private int[] parent; - private int[] left; - private int[] right; - private byte[] depth; + private final TDigestIntArray parent; + private final TDigestIntArray left; + private final TDigestIntArray right; + private final TDigestByteArray depth; - IntAVLTree(int initialCapacity) { - nodeAllocator = new NodeAllocator(); + IntAVLTree(TDigestArrays arrays, int initialCapacity) { + this.arrays = arrays; root = NIL; - parent = new int[initialCapacity]; - left = new int[initialCapacity]; - right = new int[initialCapacity]; - depth = new byte[initialCapacity]; + + NodeAllocator nodeAllocator = null; + TDigestIntArray parent = null; + TDigestIntArray left = null; + TDigestIntArray right = null; + TDigestByteArray depth = null; + + try { + this.nodeAllocator = nodeAllocator = NodeAllocator.create(arrays); + this.parent = parent = arrays.newIntArray(initialCapacity); + this.left = left = arrays.newIntArray(initialCapacity); + this.right = right = arrays.newIntArray(initialCapacity); + this.depth = depth = arrays.newByteArray(initialCapacity); + + nodeAllocator = null; + parent = null; + left = null; + right = null; + depth = null; + } finally { + Releasables.close(nodeAllocator, parent, left, right, depth); + } } - IntAVLTree() { - this(16); + IntAVLTree(TDigestArrays arrays) { + this(arrays, 16); + } + + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + nodeAllocator.ramBytesUsed() + parent.ramBytesUsed() + left.ramBytesUsed() + right.ramBytesUsed() + depth + .ramBytesUsed(); } /** @@ -74,7 +107,7 @@ public int root() { * can hold. */ public int capacity() { - return parent.length; + return parent.size(); } /** @@ -82,10 +115,10 @@ public int capacity() { * newCapacity (excluded). */ protected void resize(int newCapacity) { - parent = Arrays.copyOf(parent, newCapacity); - left = Arrays.copyOf(left, newCapacity); - right = Arrays.copyOf(right, newCapacity); - depth = Arrays.copyOf(depth, newCapacity); + parent.resize(newCapacity); + left.resize(newCapacity); + right.resize(newCapacity); + depth.resize(newCapacity); } /** @@ -99,28 +132,28 @@ public int size() { * Return the parent of the provided node. */ public int parent(int node) { - return parent[node]; + return parent.get(node); } /** * Return the left child of the provided node. */ public int left(int node) { - return left[node]; + return left.get(node); } /** * Return the right child of the provided node. */ public int right(int node) { - return right[node]; + return right.get(node); } /** * Return the depth nodes that are stored below node including itself. */ public int depth(int node) { - return depth[node]; + return depth.get(node); } /** @@ -493,23 +526,23 @@ private void rotateRight(int n) { private void parent(int node, int parent) { assert node != NIL; - this.parent[node] = parent; + this.parent.set(node, parent); } private void left(int node, int left) { assert node != NIL; - this.left[node] = left; + this.left.set(node, left); } private void right(int node, int right) { assert node != NIL; - this.right[node] = right; + this.right.set(node, right); } private void depth(int node, int depth) { assert node != NIL; assert depth >= 0 && depth <= Byte.MAX_VALUE; - this.depth[node] = (byte) depth; + this.depth.set(node, (byte) depth); } void checkBalance(int node) { @@ -526,42 +559,85 @@ void checkBalance(int node) { /** * A stack of int values. */ - private static class IntStack { + private static class IntStack implements Releasable, Accountable { + private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(IntStack.class); + + private final TDigestArrays arrays; + private boolean closed = false; - private int[] stack; + private final TDigestIntArray stack; private int size; - IntStack() { - stack = new int[0]; + IntStack(TDigestArrays arrays) { + this.arrays = arrays; + stack = arrays.newIntArray(0); size = 0; } + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + stack.ramBytesUsed(); + } + int size() { return size; } int pop() { - return stack[--size]; + int value = stack.get(--size); + stack.resize(size); + return value; } void push(int v) { - if (size >= stack.length) { - final int newLength = oversize(size + 1); - stack = Arrays.copyOf(stack, newLength); - } - stack[size++] = v; + stack.resize(++size); + stack.set(size - 1, v); } + @Override + public void close() { + if (closed == false) { + closed = true; + arrays.adjustBreaker(-SHALLOW_SIZE); + stack.close(); + } + } } - private static class NodeAllocator { + private static class NodeAllocator implements Releasable, Accountable { + private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(NodeAllocator.class); + + private final TDigestArrays arrays; + private boolean closed = false; private int nextNode; private final IntStack releasedNodes; - NodeAllocator() { + static NodeAllocator create(TDigestArrays arrays) { + arrays.adjustBreaker(SHALLOW_SIZE); + try { + return new NodeAllocator(arrays); + } catch (Exception e) { + arrays.adjustBreaker(-SHALLOW_SIZE); + throw e; + } + } + + private NodeAllocator(TDigestArrays arrays) { + this.arrays = arrays; nextNode = NIL + 1; - releasedNodes = new IntStack(); + arrays.adjustBreaker(IntStack.SHALLOW_SIZE); + try { + releasedNodes = new IntStack(arrays); + } catch (Exception e) { + arrays.adjustBreaker(-IntStack.SHALLOW_SIZE); + throw e; + } + } + + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + releasedNodes.ramBytesUsed(); } int newNode() { @@ -581,6 +657,22 @@ int size() { return nextNode - releasedNodes.size() - 1; } + @Override + public void close() { + if (closed == false) { + closed = true; + arrays.adjustBreaker(-SHALLOW_SIZE); + releasedNodes.close(); + } + } } + @Override + public void close() { + if (closed == false) { + closed = true; + arrays.adjustBreaker(-SHALLOW_SIZE); + Releasables.close(nodeAllocator, parent, left, right, depth); + } + } } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java index fc22bda52e104..06724b049f821 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java @@ -21,6 +21,12 @@ package org.elasticsearch.tdigest; +import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.tdigest.arrays.TDigestArrays; +import org.elasticsearch.tdigest.arrays.TDigestDoubleArray; +import org.elasticsearch.tdigest.arrays.TDigestIntArray; + import java.util.AbstractCollection; import java.util.Collection; import java.util.Iterator; @@ -62,6 +68,11 @@ * what the AVLTreeDigest uses and no dynamic allocation is required at all. */ public class MergingDigest extends AbstractTDigest { + private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(MergingDigest.class); + + private final TDigestArrays arrays; + private boolean closed = false; + private int mergeCount = 0; private final double publicCompression; @@ -70,26 +81,26 @@ public class MergingDigest extends AbstractTDigest { // points to the first unused centroid private int lastUsedCell; - // sum_i weight[i] See also unmergedWeight + // sum_i weight.get(i) See also unmergedWeight private double totalWeight = 0; // number of points that have been added to each merged centroid - private final double[] weight; + private final TDigestDoubleArray weight; // mean of points added to each merged centroid - private final double[] mean; + private final TDigestDoubleArray mean; - // sum_i tempWeight[i] + // sum_i tempWeight.get(i) private double unmergedWeight = 0; // this is the index of the next temporary centroid // this is a more Java-like convention than lastUsedCell uses private int tempUsed = 0; - private final double[] tempWeight; - private final double[] tempMean; + private final TDigestDoubleArray tempWeight; + private final TDigestDoubleArray tempMean; // array used for sorting the temp centroids. This is a field // to avoid allocations during operation - private final int[] order; + private final TDigestIntArray order; // if true, alternate upward and downward merge passes public boolean useAlternatingSort = true; @@ -102,6 +113,26 @@ public class MergingDigest extends AbstractTDigest { // weight limits. public static boolean useWeightLimit = true; + static MergingDigest create(TDigestArrays arrays, double compression) { + arrays.adjustBreaker(SHALLOW_SIZE); + try { + return new MergingDigest(arrays, compression); + } catch (Exception e) { + arrays.adjustBreaker(-SHALLOW_SIZE); + throw e; + } + } + + static MergingDigest create(TDigestArrays arrays, double compression, int bufferSize, int size) { + arrays.adjustBreaker(SHALLOW_SIZE); + try { + return new MergingDigest(arrays, compression, bufferSize, size); + } catch (Exception e) { + arrays.adjustBreaker(-SHALLOW_SIZE); + throw e; + } + } + /** * Allocates a buffer merging t-digest. This is the normally used constructor that * allocates default sized internal arrays. Other versions are available, but should @@ -109,8 +140,8 @@ public class MergingDigest extends AbstractTDigest { * * @param compression The compression factor */ - public MergingDigest(double compression) { - this(compression, -1); + private MergingDigest(TDigestArrays arrays, double compression) { + this(arrays, compression, -1); } /** @@ -119,9 +150,9 @@ public MergingDigest(double compression) { * @param compression Compression factor for t-digest. Same as 1/\delta in the paper. * @param bufferSize How many samples to retain before merging. */ - public MergingDigest(double compression, int bufferSize) { + private MergingDigest(TDigestArrays arrays, double compression, int bufferSize) { // we can guarantee that we only need ceiling(compression). - this(compression, bufferSize, -1); + this(arrays, compression, bufferSize, -1); } /** @@ -131,7 +162,9 @@ public MergingDigest(double compression, int bufferSize) { * @param bufferSize Number of temporary centroids * @param size Size of main buffer */ - public MergingDigest(double compression, int bufferSize, int size) { + private MergingDigest(TDigestArrays arrays, double compression, int bufferSize, int size) { + this.arrays = arrays; + // ensure compression >= 10 // default size = 2 * ceil(compression) // default bufferSize = 5 * size @@ -205,25 +238,42 @@ public MergingDigest(double compression, int bufferSize, int size) { bufferSize = 2 * size; } - weight = new double[size]; - mean = new double[size]; - - tempWeight = new double[bufferSize]; - tempMean = new double[bufferSize]; - order = new int[bufferSize]; + TDigestDoubleArray weight = null; + TDigestDoubleArray mean = null; + TDigestDoubleArray tempWeight = null; + TDigestDoubleArray tempMean = null; + TDigestIntArray order = null; + + try { + this.weight = weight = arrays.newDoubleArray(size); + this.mean = mean = arrays.newDoubleArray(size); + + this.tempWeight = tempWeight = arrays.newDoubleArray(bufferSize); + this.tempMean = tempMean = arrays.newDoubleArray(bufferSize); + this.order = order = arrays.newIntArray(bufferSize); + } catch (Exception e) { + Releasables.close(weight, mean, tempWeight, tempMean, order); + throw e; + } lastUsedCell = 0; } + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + weight.ramBytesUsed() + mean.ramBytesUsed() + tempWeight.ramBytesUsed() + tempMean.ramBytesUsed() + order + .ramBytesUsed(); + } + @Override public void add(double x, long w) { checkValue(x); - if (tempUsed >= tempWeight.length - lastUsedCell - 1) { + if (tempUsed >= tempWeight.size() - lastUsedCell - 1) { mergeNewValues(); } int where = tempUsed++; - tempWeight[where] = w; - tempMean[where] = x; + tempWeight.set(where, w); + tempMean.set(where, x); unmergedWeight += w; if (x < min) { min = x; @@ -252,23 +302,20 @@ private void mergeNewValues(double compression) { } private void merge( - double[] incomingMean, - double[] incomingWeight, + TDigestDoubleArray incomingMean, + TDigestDoubleArray incomingWeight, int incomingCount, - int[] incomingOrder, + TDigestIntArray incomingOrder, double unmergedWeight, boolean runBackwards, double compression ) { // when our incoming buffer fills up, we combine our existing centroids with the incoming data, // and then reduce the centroids by merging if possible - System.arraycopy(mean, 0, incomingMean, incomingCount, lastUsedCell); - System.arraycopy(weight, 0, incomingWeight, incomingCount, lastUsedCell); + incomingMean.set(incomingCount, mean, 0, lastUsedCell); + incomingWeight.set(incomingCount, weight, 0, lastUsedCell); incomingCount += lastUsedCell; - if (incomingOrder == null) { - incomingOrder = new int[incomingCount]; - } Sort.stableSort(incomingOrder, incomingMean, incomingCount); totalWeight += unmergedWeight; @@ -280,8 +327,8 @@ private void merge( // start by copying the least incoming value to the normal buffer lastUsedCell = 0; - mean[lastUsedCell] = incomingMean[incomingOrder[0]]; - weight[lastUsedCell] = incomingWeight[incomingOrder[0]]; + mean.set(lastUsedCell, incomingMean.get(incomingOrder.get(0))); + weight.set(lastUsedCell, incomingWeight.get(incomingOrder.get(0))); double wSoFar = 0; // weight will contain all zeros after this loop @@ -290,8 +337,8 @@ private void merge( double k1 = scale.k(0, normalizer); double wLimit = totalWeight * scale.q(k1 + 1, normalizer); for (int i = 1; i < incomingCount; i++) { - int ix = incomingOrder[i]; - double proposedWeight = weight[lastUsedCell] + incomingWeight[ix]; + int ix = incomingOrder.get(i); + double proposedWeight = weight.get(lastUsedCell) + incomingWeight.get(ix); double projectedW = wSoFar + proposedWeight; boolean addThis; if (useWeightLimit) { @@ -305,7 +352,7 @@ private void merge( // force first and last centroid to never merge addThis = false; } - if (lastUsedCell == mean.length - 1) { + if (lastUsedCell == mean.size() - 1) { // use the last centroid, there's no more addThis = true; } @@ -313,22 +360,26 @@ private void merge( if (addThis) { // next point will fit // so merge into existing centroid - weight[lastUsedCell] += incomingWeight[ix]; - mean[lastUsedCell] = mean[lastUsedCell] + (incomingMean[ix] - mean[lastUsedCell]) * incomingWeight[ix] - / weight[lastUsedCell]; - incomingWeight[ix] = 0; + weight.set(lastUsedCell, weight.get(lastUsedCell) + incomingWeight.get(ix)); + mean.set( + lastUsedCell, + mean.get(lastUsedCell) + (incomingMean.get(ix) - mean.get(lastUsedCell)) * incomingWeight.get(ix) / weight.get( + lastUsedCell + ) + ); + incomingWeight.set(ix, 0); } else { // didn't fit ... move to next output, copy out first centroid - wSoFar += weight[lastUsedCell]; + wSoFar += weight.get(lastUsedCell); if (useWeightLimit == false) { k1 = scale.k(wSoFar / totalWeight, normalizer); wLimit = totalWeight * scale.q(k1 + 1, normalizer); } lastUsedCell++; - mean[lastUsedCell] = incomingMean[ix]; - weight[lastUsedCell] = incomingWeight[ix]; - incomingWeight[ix] = 0; + mean.set(lastUsedCell, incomingMean.get(ix)); + weight.set(lastUsedCell, incomingWeight.get(ix)); + incomingWeight.set(ix, 0); } } // points to next empty cell @@ -337,7 +388,7 @@ private void merge( // sanity check double sum = 0; for (int i = 0; i < lastUsedCell; i++) { - sum += weight[i]; + sum += weight.get(i); } assert sum == totalWeight; if (runBackwards) { @@ -345,8 +396,8 @@ private void merge( Sort.reverse(weight, 0, lastUsedCell); } if (totalWeight > 0) { - min = Math.min(min, mean[0]); - max = Math.max(max, mean[lastUsedCell - 1]); + min = Math.min(min, mean.get(0)); + max = Math.max(max, mean.get(lastUsedCell - 1)); } } @@ -387,8 +438,8 @@ public double cdf(double x) { // we have one or more centroids == x, treat them as one // dw will accumulate the weight of all of the centroids at x double dw = 0; - for (int i = 0; i < lastUsedCell && Double.compare(mean[i], x) == 0; i++) { - dw += weight[i]; + for (int i = 0; i < lastUsedCell && Double.compare(mean.get(i), x) == 0; i++) { + dw += weight.get(i); } return dw / 2.0 / size(); } @@ -398,31 +449,32 @@ public double cdf(double x) { } if (x == max) { double dw = 0; - for (int i = lastUsedCell - 1; i >= 0 && Double.compare(mean[i], x) == 0; i--) { - dw += weight[i]; + for (int i = lastUsedCell - 1; i >= 0 && Double.compare(mean.get(i), x) == 0; i--) { + dw += weight.get(i); } return (size() - dw / 2.0) / size(); } // initially, we set left width equal to right width - double left = (mean[1] - mean[0]) / 2; + double left = (mean.get(1) - mean.get(0)) / 2; double weightSoFar = 0; for (int i = 0; i < lastUsedCell - 1; i++) { - double right = (mean[i + 1] - mean[i]) / 2; - if (x < mean[i] + right) { - double value = (weightSoFar + weight[i] * interpolate(x, mean[i] - left, mean[i] + right)) / size(); + double right = (mean.get(i + 1) - mean.get(i)) / 2; + if (x < mean.get(i) + right) { + double value = (weightSoFar + weight.get(i) * interpolate(x, mean.get(i) - left, mean.get(i) + right)) / size(); return Math.max(value, 0.0); } - weightSoFar += weight[i]; + weightSoFar += weight.get(i); left = right; } // for the last element, assume right width is same as left int lastOffset = lastUsedCell - 1; - double right = (mean[lastOffset] - mean[lastOffset - 1]) / 2; - if (x < mean[lastOffset] + right) { - return (weightSoFar + weight[lastOffset] * interpolate(x, mean[lastOffset] - right, mean[lastOffset] + right)) / size(); + double right = (mean.get(lastOffset) - mean.get(lastOffset - 1)) / 2; + if (x < mean.get(lastOffset) + right) { + return (weightSoFar + weight.get(lastOffset) * interpolate(x, mean.get(lastOffset) - right, mean.get(lastOffset) + right)) + / size(); } return 1; } @@ -440,7 +492,7 @@ public double quantile(double q) { return Double.NaN; } else if (lastUsedCell == 1) { // with one data point, all quantiles lead to Rome - return mean[0]; + return mean.get(0); } // we know that there are at least two centroids now @@ -458,40 +510,40 @@ public double quantile(double q) { return max; } - double weightSoFar = weight[0] / 2; + double weightSoFar = weight.get(0) / 2; // if the left centroid has more than one sample, we still know // that one sample occurred at min so we can do some interpolation - if (weight[0] > 1 && index < weightSoFar) { + if (weight.get(0) > 1 && index < weightSoFar) { // there is a single sample at min so we interpolate with less weight - return weightedAverage(min, weightSoFar - index, mean[0], index); + return weightedAverage(min, weightSoFar - index, mean.get(0), index); } // if the right-most centroid has more than one sample, we still know // that one sample occurred at max so we can do some interpolation - if (weight[n - 1] > 1 && totalWeight - index <= weight[n - 1] / 2) { - return max - (totalWeight - index - 1) / (weight[n - 1] / 2 - 1) * (max - mean[n - 1]); + if (weight.get(n - 1) > 1 && totalWeight - index <= weight.get(n - 1) / 2) { + return max - (totalWeight - index - 1) / (weight.get(n - 1) / 2 - 1) * (max - mean.get(n - 1)); } // in between extremes we interpolate between centroids for (int i = 0; i < n - 1; i++) { - double dw = (weight[i] + weight[i + 1]) / 2; + double dw = (weight.get(i) + weight.get(i + 1)) / 2; if (weightSoFar + dw > index) { // centroids i and i+1 bracket our current point double z1 = index - weightSoFar; double z2 = weightSoFar + dw - index; - return weightedAverage(mean[i], z2, mean[i + 1], z1); + return weightedAverage(mean.get(i), z2, mean.get(i + 1), z1); } weightSoFar += dw; } - assert weight[n - 1] >= 1; - assert index >= totalWeight - weight[n - 1]; + assert weight.get(n - 1) >= 1; + assert index >= totalWeight - weight.get(n - 1); // Interpolate between the last mean and the max. double z1 = index - weightSoFar; - double z2 = weight[n - 1] / 2.0 - z1; - return weightedAverage(mean[n - 1], z1, max, z2); + double z2 = weight.get(n - 1) / 2.0 - z1; + return weightedAverage(mean.get(n - 1), z1, max, z2); } @Override @@ -518,7 +570,7 @@ public boolean hasNext() { @Override public Centroid next() { - Centroid rc = new Centroid(mean[i], (long) weight[i]); + Centroid rc = new Centroid(mean.get(i), (long) weight.get(i)); i++; return rc; } @@ -553,7 +605,7 @@ public void setScaleFunction(ScaleFunction scaleFunction) { @Override public int byteSize() { - return 48 + 8 * (mean.length + weight.length + tempMean.length + tempWeight.length) + 4 * order.length; + return 48 + 8 * (mean.size() + weight.size() + tempMean.size() + tempWeight.size()) + 4 * order.size(); } @Override @@ -568,4 +620,13 @@ public String toString() { + "-" + (useTwoLevelCompression ? "twoLevel" : "oneLevel"); } + + @Override + public void close() { + if (closed == false) { + closed = true; + arrays.adjustBreaker(-SHALLOW_SIZE); + Releasables.close(weight, mean, tempWeight, tempMean, order); + } + } } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/Sort.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/Sort.java index c62ae54f93c2c..1c54b0327690c 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/Sort.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/Sort.java @@ -21,7 +21,9 @@ package org.elasticsearch.tdigest; -import java.util.Arrays; +import org.elasticsearch.tdigest.arrays.TDigestDoubleArray; +import org.elasticsearch.tdigest.arrays.TDigestIntArray; + import java.util.Random; /** @@ -37,206 +39,14 @@ public class Sort { * @param values The values to sort. * @param n The number of values to sort */ - public static void stableSort(int[] order, double[] values, int n) { + public static void stableSort(TDigestIntArray order, TDigestDoubleArray values, int n) { for (int i = 0; i < n; i++) { - order[i] = i; + order.set(i, i); } stableQuickSort(order, values, 0, n, 64); stableInsertionSort(order, values, 0, n, 64); } - /** - * Two-key quick sort on (values, weights) using an index array - * - * @param order Indexes into values - * @param values The values to sort. - * @param weights The secondary sort key - * @param n The number of values to sort - * @return true if the values were already sorted - */ - public static boolean sort(int[] order, double[] values, double[] weights, int n) { - if (weights == null) { - weights = Arrays.copyOf(values, values.length); - } - boolean r = sort(order, values, weights, 0, n); - // now adjust all runs with equal value so that bigger weights are nearer - // the median - double medianWeight = 0; - for (int i = 0; i < n; i++) { - medianWeight += weights[i]; - } - medianWeight = medianWeight / 2; - int i = 0; - double soFar = 0; - double nextGroup = 0; - while (i < n) { - int j = i; - while (j < n && values[order[j]] == values[order[i]]) { - double w = weights[order[j]]; - nextGroup += w; - j++; - } - if (j > i + 1) { - if (soFar >= medianWeight) { - // entire group is in last half, reverse the order - reverse(order, i, j - i); - } else if (nextGroup > medianWeight) { - // group straddles the median, but not necessarily evenly - // most elements are probably unit weight if there are many - double[] scratch = new double[j - i]; - - double netAfter = nextGroup + soFar - 2 * medianWeight; - // heuristically adjust weights to roughly balance around median - double max = weights[order[j - 1]]; - for (int k = j - i - 1; k >= 0; k--) { - double weight = weights[order[i + k]]; - if (netAfter < 0) { - // sort in normal order - scratch[k] = weight; - netAfter += weight; - } else { - // sort reversed, but after normal items - scratch[k] = 2 * max + 1 - weight; - netAfter -= weight; - } - } - // sort these balanced weights - int[] sub = new int[j - i]; - sort(sub, scratch, scratch, 0, j - i); - int[] tmp = Arrays.copyOfRange(order, i, j); - for (int k = 0; k < j - i; k++) { - order[i + k] = tmp[sub[k]]; - } - } - } - soFar = nextGroup; - i = j; - } - return r; - } - - /** - * Two-key quick sort on (values, weights) using an index array - * - * @param order Indexes into values - * @param values The values to sort - * @param weights The weights that define the secondary ordering - * @param start The first element to sort - * @param n The number of values to sort - * @return True if the values were in order without sorting - */ - private static boolean sort(int[] order, double[] values, double[] weights, int start, int n) { - boolean inOrder = true; - for (int i = start; i < start + n; i++) { - if (inOrder && i < start + n - 1) { - inOrder = values[i] < values[i + 1] || (values[i] == values[i + 1] && weights[i] <= weights[i + 1]); - } - order[i] = i; - } - if (inOrder) { - return true; - } - quickSort(order, values, weights, start, start + n, 64); - insertionSort(order, values, weights, start, start + n, 64); - return false; - } - - /** - * Standard two-key quick sort on (values, weights) except that sorting is done on an index array - * rather than the values themselves - * - * @param order The pre-allocated index array - * @param values The values to sort - * @param weights The weights (secondary key) - * @param start The beginning of the values to sort - * @param end The value after the last value to sort - * @param limit The minimum size to recurse down to. - */ - private static void quickSort(int[] order, double[] values, double[] weights, int start, int end, int limit) { - // the while loop implements tail-recursion to avoid excessive stack calls on nasty cases - while (end - start > limit) { - - // pivot by a random element - int pivotIndex = start + prng.nextInt(end - start); - double pivotValue = values[order[pivotIndex]]; - double pivotWeight = weights[order[pivotIndex]]; - - // move pivot to beginning of array - swap(order, start, pivotIndex); - - // we use a three way partition because many duplicate values is an important case - - int low = start + 1; // low points to first value not known to be equal to pivotValue - int high = end; // high points to first value > pivotValue - int i = low; // i scans the array - while (i < high) { - // invariant: (values,weights)[order[k]] == (pivotValue, pivotWeight) for k in [0..low) - // invariant: (values,weights)[order[k]] < (pivotValue, pivotWeight) for k in [low..i) - // invariant: (values,weights)[order[k]] > (pivotValue, pivotWeight) for k in [high..end) - // in-loop: i < high - // in-loop: low < high - // in-loop: i >= low - double vi = values[order[i]]; - double wi = weights[order[i]]; - if (vi == pivotValue && wi == pivotWeight) { - if (low != i) { - swap(order, low, i); - } else { - i++; - } - low++; - } else if (vi > pivotValue || (vi == pivotValue && wi > pivotWeight)) { - high--; - swap(order, i, high); - } else { - // vi < pivotValue || (vi == pivotValue && wi < pivotWeight) - i++; - } - } - // invariant: (values,weights)[order[k]] == (pivotValue, pivotWeight) for k in [0..low) - // invariant: (values,weights)[order[k]] < (pivotValue, pivotWeight) for k in [low..i) - // invariant: (values,weights)[order[k]] > (pivotValue, pivotWeight) for k in [high..end) - // assert i == high || low == high therefore, we are done with partition - - // at this point, i==high, from [start,low) are == pivot, [low,high) are < and [high,end) are > - // we have to move the values equal to the pivot into the middle. To do this, we swap pivot - // values into the top end of the [low,high) range stopping when we run out of destinations - // or when we run out of values to copy - int from = start; - int to = high - 1; - for (i = 0; from < low && to >= low; i++) { - swap(order, from++, to--); - } - if (from == low) { - // ran out of things to copy. This means that the last destination is the boundary - low = to + 1; - } else { - // ran out of places to copy to. This means that there are uncopied pivots and the - // boundary is at the beginning of those - low = from; - } - - // checkPartition(order, values, pivotValue, start, low, high, end); - - // now recurse, but arrange it so we handle the longer limit by tail recursion - // we have to sort the pivot values because they may have different weights - // we can't do that, however until we know how much weight is in the left and right - if (low - start < end - high) { - // left side is smaller - quickSort(order, values, weights, start, low, limit); - - // this is really a way to do - // quickSort(order, values, high, end, limit); - start = high; - } else { - quickSort(order, values, weights, high, end, limit); - // this is really a way to do - // quickSort(order, values, start, low, limit); - end = low; - } - } - } - /** * Stabilized quick sort on an index array. This is a normal quick sort that uses the * original index as a secondary key. Since we are really just sorting an index array @@ -248,14 +58,14 @@ private static void quickSort(int[] order, double[] values, double[] weights, in * @param end The value after the last value to sort * @param limit The minimum size to recurse down to. */ - private static void stableQuickSort(int[] order, double[] values, int start, int end, int limit) { + private static void stableQuickSort(TDigestIntArray order, TDigestDoubleArray values, int start, int end, int limit) { // the while loop implements tail-recursion to avoid excessive stack calls on nasty cases while (end - start > limit) { // pivot by a random element int pivotIndex = start + prng.nextInt(end - start); - double pivotValue = values[order[pivotIndex]]; - int pv = order[pivotIndex]; + double pivotValue = values.get(order.get(pivotIndex)); + int pv = order.get(pivotIndex); // move pivot to beginning of array swap(order, start, pivotIndex); @@ -272,8 +82,8 @@ private static void stableQuickSort(int[] order, double[] values, int start, int // in-loop: i < high // in-loop: low < high // in-loop: i >= low - double vi = values[order[i]]; - int pi = order[i]; + double vi = values.get(order.get(i)); + int pi = order.get(i); if (vi == pivotValue && pi == pv) { if (low != i) { swap(order, low, i); @@ -333,247 +143,10 @@ private static void stableQuickSort(int[] order, double[] values, int start, int } } - /** - * Quick sort in place of several paired arrays. On return, - * keys[...] is in order and the values[] arrays will be - * reordered as well in the same way. - * - * @param key Values to sort on - * @param values The auxiliary values to sort. - */ - public static void sort(double[] key, double[]... values) { - sort(key, 0, key.length, values); - } - - /** - * Quick sort using an index array. On return, - * values[order[i]] is in order as i goes start..n - * @param key Values to sort on - * @param start The first element to sort - * @param n The number of values to sort - * @param values The auxiliary values to sort. - */ - public static void sort(double[] key, int start, int n, double[]... values) { - quickSort(key, values, start, start + n, 8); - insertionSort(key, values, start, start + n, 8); - } - - /** - * Standard quick sort except that sorting rearranges parallel arrays - * - * @param key Values to sort on - * @param values The auxiliary values to sort. - * @param start The beginning of the values to sort - * @param end The value after the last value to sort - * @param limit The minimum size to recurse down to. - */ - private static void quickSort(double[] key, double[][] values, int start, int end, int limit) { - // the while loop implements tail-recursion to avoid excessive stack calls on nasty cases - while (end - start > limit) { - - // median of three values for the pivot - int a = start; - int b = (start + end) / 2; - int c = end - 1; - - int pivotIndex; - double pivotValue; - double va = key[a]; - double vb = key[b]; - double vc = key[c]; - - if (va > vb) { - if (vc > va) { - // vc > va > vb - pivotIndex = a; - pivotValue = va; - } else { - // va > vb, va >= vc - if (vc < vb) { - // va > vb > vc - pivotIndex = b; - pivotValue = vb; - } else { - // va >= vc >= vb - pivotIndex = c; - pivotValue = vc; - } - } - } else { - // vb >= va - if (vc > vb) { - // vc > vb >= va - pivotIndex = b; - pivotValue = vb; - } else { - // vb >= va, vb >= vc - if (vc < va) { - // vb >= va > vc - pivotIndex = a; - pivotValue = va; - } else { - // vb >= vc >= va - pivotIndex = c; - pivotValue = vc; - } - } - } - - // move pivot to beginning of array - swap(start, pivotIndex, key, values); - - // we use a three way partition because many duplicate values is an important case - - int low = start + 1; // low points to first value not known to be equal to pivotValue - int high = end; // high points to first value > pivotValue - int i = low; // i scans the array - while (i < high) { - // invariant: values[order[k]] == pivotValue for k in [0..low) - // invariant: values[order[k]] < pivotValue for k in [low..i) - // invariant: values[order[k]] > pivotValue for k in [high..end) - // in-loop: i < high - // in-loop: low < high - // in-loop: i >= low - double vi = key[i]; - if (vi == pivotValue) { - if (low != i) { - swap(low, i, key, values); - } else { - i++; - } - low++; - } else if (vi > pivotValue) { - high--; - swap(i, high, key, values); - } else { - // vi < pivotValue - i++; - } - } - // invariant: values[order[k]] == pivotValue for k in [0..low) - // invariant: values[order[k]] < pivotValue for k in [low..i) - // invariant: values[order[k]] > pivotValue for k in [high..end) - // assert i == high || low == high therefore, we are done with partition - - // at this point, i==high, from [start,low) are == pivot, [low,high) are < and [high,end) are > - // we have to move the values equal to the pivot into the middle. To do this, we swap pivot - // values into the top end of the [low,high) range stopping when we run out of destinations - // or when we run out of values to copy - int from = start; - int to = high - 1; - for (i = 0; from < low && to >= low; i++) { - swap(from++, to--, key, values); - } - if (from == low) { - // ran out of things to copy. This means that the last destination is the boundary - low = to + 1; - } else { - // ran out of places to copy to. This means that there are uncopied pivots and the - // boundary is at the beginning of those - low = from; - } - - // checkPartition(order, values, pivotValue, start, low, high, end); - - // now recurse, but arrange it so we handle the longer limit by tail recursion - if (low - start < end - high) { - quickSort(key, values, start, low, limit); - - // this is really a way to do - // quickSort(order, values, high, end, limit); - start = high; - } else { - quickSort(key, values, high, end, limit); - // this is really a way to do - // quickSort(order, values, start, low, limit); - end = low; - } - } - } - - /** - * Limited range insertion sort. We assume that no element has to move more than limit steps - * because quick sort has done its thing. This version works on parallel arrays of keys and values. - * - * @param key The array of keys - * @param values The values we are sorting - * @param start The starting point of the sort - * @param end The ending point of the sort - * @param limit The largest amount of disorder - */ - private static void insertionSort(double[] key, double[][] values, int start, int end, int limit) { - // loop invariant: all values start ... i-1 are ordered - for (int i = start + 1; i < end; i++) { - double v = key[i]; - int m = Math.max(i - limit, start); - for (int j = i; j >= m; j--) { - if (j == m || key[j - 1] <= v) { - if (j < i) { - System.arraycopy(key, j, key, j + 1, i - j); - key[j] = v; - for (double[] value : values) { - double tmp = value[i]; - System.arraycopy(value, j, value, j + 1, i - j); - value[j] = tmp; - } - } - break; - } - } - } - } - - private static void swap(int[] order, int i, int j) { - int t = order[i]; - order[i] = order[j]; - order[j] = t; - } - - private static void swap(int i, int j, double[] key, double[]... values) { - double t = key[i]; - key[i] = key[j]; - key[j] = t; - - for (int k = 0; k < values.length; k++) { - t = values[k][i]; - values[k][i] = values[k][j]; - values[k][j] = t; - } - } - - /** - * Limited range insertion sort with primary and secondary key. We assume that no - * element has to move more than limit steps because quick sort has done its thing. - * - * If weights (the secondary key) is null, then only the primary key is used. - * - * This sort is inherently stable. - * - * @param order The permutation index - * @param values The values we are sorting - * @param weights The secondary key for sorting - * @param start Where to start the sort - * @param n How many elements to sort - * @param limit The largest amount of disorder - */ - private static void insertionSort(int[] order, double[] values, double[] weights, int start, int n, int limit) { - for (int i = start + 1; i < n; i++) { - int t = order[i]; - double v = values[order[i]]; - double w = weights == null ? 0 : weights[order[i]]; - int m = Math.max(i - limit, start); - // values in [start, i) are ordered - // scan backwards to find where to stick t - for (int j = i; j >= m; j--) { - if (j == 0 || values[order[j - 1]] < v || (values[order[j - 1]] == v && (weights == null || weights[order[j - 1]] <= w))) { - if (j < i) { - System.arraycopy(order, j, order, j + 1, i - j); - order[j] = t; - } - break; - } - } - } + private static void swap(TDigestIntArray order, int i, int j) { + int t = order.get(i); + order.set(i, order.get(j)); + order.set(j, t); } /** @@ -587,19 +160,19 @@ private static void insertionSort(int[] order, double[] values, double[] weights * @param n How many elements to sort * @param limit The largest amount of disorder */ - private static void stableInsertionSort(int[] order, double[] values, int start, int n, int limit) { + private static void stableInsertionSort(TDigestIntArray order, TDigestDoubleArray values, int start, int n, int limit) { for (int i = start + 1; i < n; i++) { - int t = order[i]; - double v = values[order[i]]; - int vi = order[i]; + int t = order.get(i); + double v = values.get(order.get(i)); + int vi = order.get(i); int m = Math.max(i - limit, start); // values in [start, i) are ordered // scan backwards to find where to stick t for (int j = i; j >= m; j--) { - if (j == 0 || values[order[j - 1]] < v || (values[order[j - 1]] == v && (order[j - 1] <= vi))) { + if (j == 0 || values.get(order.get(j - 1)) < v || (values.get(order.get(j - 1)) == v && (order.get(j - 1) <= vi))) { if (j < i) { - System.arraycopy(order, j, order, j + 1, i - j); - order[j] = t; + order.set(j + 1, order, j, i - j); + order.set(j, t); } break; } @@ -608,41 +181,32 @@ private static void stableInsertionSort(int[] order, double[] values, int start, } /** - * Reverses an array in-place. - * - * @param order The array to reverse - */ - public static void reverse(int[] order) { - reverse(order, 0, order.length); - } - - /** - * Reverses part of an array. See {@link #reverse(int[])} + * Reverses part of an array. * * @param order The array containing the data to reverse. * @param offset Where to start reversing. * @param length How many elements to reverse */ - public static void reverse(int[] order, int offset, int length) { + public static void reverse(TDigestIntArray order, int offset, int length) { for (int i = 0; i < length / 2; i++) { - int t = order[offset + i]; - order[offset + i] = order[offset + length - i - 1]; - order[offset + length - i - 1] = t; + int t = order.get(offset + i); + order.set(offset + i, order.get(offset + length - i - 1)); + order.set(offset + length - i - 1, t); } } /** - * Reverses part of an array. See {@link #reverse(int[])} + * Reverses part of an array. * * @param order The array containing the data to reverse. * @param offset Where to start reversing. * @param length How many elements to reverse */ - public static void reverse(double[] order, int offset, int length) { + public static void reverse(TDigestDoubleArray order, int offset, int length) { for (int i = 0; i < length / 2; i++) { - double t = order[offset + i]; - order[offset + i] = order[offset + length - i - 1]; - order[offset + length - i - 1] = t; + double t = order.get(offset + i); + order.set(offset + i, order.get(offset + length - i - 1)); + order.set(offset + length - i - 1, t); } } } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/SortingDigest.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/SortingDigest.java index 92f770cbb7569..6912982ae6f8f 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/SortingDigest.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/SortingDigest.java @@ -19,10 +19,13 @@ package org.elasticsearch.tdigest; +import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.tdigest.arrays.TDigestArrays; +import org.elasticsearch.tdigest.arrays.TDigestDoubleArray; + import java.util.AbstractCollection; -import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.Iterator; /** @@ -31,17 +34,41 @@ * samples, at the expense of allocating much more memory. */ public class SortingDigest extends AbstractTDigest { + private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(SortingDigest.class); + + private final TDigestArrays arrays; + private boolean closed = false; // Tracks all samples. Gets sorted on quantile and cdf calls. - final ArrayList values = new ArrayList<>(); + final TDigestDoubleArray values; // Indicates if all values have been sorted. private boolean isSorted = true; + static SortingDigest create(TDigestArrays arrays) { + arrays.adjustBreaker(SHALLOW_SIZE); + try { + return new SortingDigest(arrays); + } catch (Exception e) { + arrays.adjustBreaker(-SHALLOW_SIZE); + throw e; + } + } + + private SortingDigest(TDigestArrays arrays) { + this.arrays = arrays; + values = arrays.newDoubleArray(0); + } + + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + values.ramBytesUsed(); + } + @Override public void add(double x, long w) { checkValue(x); - isSorted = isSorted && (values.isEmpty() || values.get(values.size() - 1) <= x); + isSorted = isSorted && (values.size() == 0 || values.get(values.size() - 1) <= x); for (int i = 0; i < w; i++) { values.add(x); } @@ -52,7 +79,7 @@ public void add(double x, long w) { @Override public void compress() { if (isSorted == false) { - Collections.sort(values); + values.sort(); isSorted = true; } } @@ -133,4 +160,13 @@ public void reserve(long size) { public int byteSize() { return values.size() * 8; } + + @Override + public void close() { + if (closed == false) { + closed = true; + arrays.adjustBreaker(-SHALLOW_SIZE); + Releasables.close(values); + } + } } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/TDigest.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/TDigest.java index 296ed57a4d960..cf232b4a23a87 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/TDigest.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/TDigest.java @@ -21,6 +21,10 @@ package org.elasticsearch.tdigest; +import org.apache.lucene.util.Accountable; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.tdigest.arrays.TDigestArrays; + import java.util.Collection; import java.util.Locale; @@ -35,7 +39,7 @@ * - test coverage roughly at 90% * - easy to adapt for use with map-reduce */ -public abstract class TDigest { +public abstract class TDigest implements Releasable, Accountable { protected ScaleFunction scale = ScaleFunction.K_2; double min = Double.POSITIVE_INFINITY; double max = Double.NEGATIVE_INFINITY; @@ -48,8 +52,8 @@ public abstract class TDigest { * The number of centroids retained will be a smallish (usually less than 10) multiple of this number. * @return the MergingDigest */ - public static TDigest createMergingDigest(double compression) { - return new MergingDigest(compression); + public static MergingDigest createMergingDigest(TDigestArrays arrays, double compression) { + return MergingDigest.create(arrays, compression); } /** @@ -61,8 +65,8 @@ public static TDigest createMergingDigest(double compression) { * The number of centroids retained will be a smallish (usually less than 10) multiple of this number. * @return the AvlTreeDigest */ - public static TDigest createAvlTreeDigest(double compression) { - return new AVLTreeDigest(compression); + public static AVLTreeDigest createAvlTreeDigest(TDigestArrays arrays, double compression) { + return AVLTreeDigest.create(arrays, compression); } /** @@ -71,8 +75,8 @@ public static TDigest createAvlTreeDigest(double compression) { * * @return the SortingDigest */ - public static TDigest createSortingDigest() { - return new SortingDigest(); + public static SortingDigest createSortingDigest(TDigestArrays arrays) { + return SortingDigest.create(arrays); } /** @@ -84,8 +88,8 @@ public static TDigest createSortingDigest() { * The number of centroids retained will be a smallish (usually less than 10) multiple of this number. * @return the HybridDigest */ - public static TDigest createHybridDigest(double compression) { - return new HybridDigest(compression); + public static HybridDigest createHybridDigest(TDigestArrays arrays, double compression) { + return HybridDigest.create(arrays, compression); } /** diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestArrays.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestArrays.java new file mode 100644 index 0000000000000..e444eeda458e9 --- /dev/null +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestArrays.java @@ -0,0 +1,37 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * This project is based on a modification of https://github.com/tdunning/t-digest which is licensed under the Apache 2.0 License. + */ + +package org.elasticsearch.tdigest.arrays; + +/** + * Minimal interface for BigArrays-like classes used within TDigest. + */ +public interface TDigestArrays { + void adjustBreaker(long size); + + TDigestDoubleArray newDoubleArray(int initialSize); + + TDigestIntArray newIntArray(int initialSize); + + TDigestLongArray newLongArray(int initialSize); + + TDigestByteArray newByteArray(int initialSize); +} diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestByteArray.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestByteArray.java new file mode 100644 index 0000000000000..3416ace3bd095 --- /dev/null +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestByteArray.java @@ -0,0 +1,41 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * This project is based on a modification of https://github.com/tdunning/t-digest which is licensed under the Apache 2.0 License. + */ + +package org.elasticsearch.tdigest.arrays; + +import org.apache.lucene.util.Accountable; +import org.elasticsearch.core.Releasable; + +/** + * Minimal interface for ByteArray-like classes used within TDigest. + */ +public interface TDigestByteArray extends Releasable, Accountable { + int size(); + + byte get(int index); + + void set(int index, byte value); + + /** + * Resizes the array. If the new size is bigger than the current size, the new elements are set to 0. + */ + void resize(int newSize); +} diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestDoubleArray.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestDoubleArray.java new file mode 100644 index 0000000000000..707b983358e36 --- /dev/null +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestDoubleArray.java @@ -0,0 +1,61 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * This project is based on a modification of https://github.com/tdunning/t-digest which is licensed under the Apache 2.0 License. + */ + +package org.elasticsearch.tdigest.arrays; + +import org.apache.lucene.util.Accountable; +import org.elasticsearch.core.Releasable; + +/** + * Minimal interface for DoubleArray-like classes used within TDigest. + */ +public interface TDigestDoubleArray extends Releasable, Accountable { + int size(); + + double get(int index); + + void set(int index, double value); + + void add(double value); + + void ensureCapacity(int requiredCapacity); + + /** + * Resizes the array. If the new size is bigger than the current size, the new elements are set to 0. + */ + void resize(int newSize); + + /** + * Copies {@code len} elements from {@code buf} to this array. + */ + default void set(int index, TDigestDoubleArray buf, int offset, int len) { + assert index >= 0 && index + len <= this.size(); + assert buf != this : "This method doesn't ensure that the copy from itself will be correct"; + for (int i = len - 1; i >= 0; i--) { + this.set(index + i, buf.get(offset + i)); + } + } + + /** + * Sorts the array in place in ascending order. + */ + void sort(); +} diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestIntArray.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestIntArray.java new file mode 100644 index 0000000000000..b131e194e8be4 --- /dev/null +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestIntArray.java @@ -0,0 +1,56 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * This project is based on a modification of https://github.com/tdunning/t-digest which is licensed under the Apache 2.0 License. + */ + +package org.elasticsearch.tdigest.arrays; + +import org.apache.lucene.util.Accountable; +import org.elasticsearch.core.Releasable; + +/** + * Minimal interface for IntArray-like classes used within TDigest. + */ +public interface TDigestIntArray extends Releasable, Accountable { + int size(); + + int get(int index); + + void set(int index, int value); + + /** + * Resizes the array. If the new size is bigger than the current size, the new elements are set to 0. + */ + void resize(int newSize); + + /** + * Copies {@code len} elements from {@code buf} to this array. + *

    + * As this method will be used to insert elements from itself in an insertion sort, + * the copy must be made in reverse order, from offset+len-1 to offset. + *

    + */ + default void set(int index, TDigestIntArray buf, int offset, int len) { + assert index >= 0 && index + len <= this.size(); + assert buf != this || index >= offset : "To set to itself, the destination index must be greater than the source offset"; + for (int i = len - 1; i >= 0; i--) { + this.set(index + i, buf.get(offset + i)); + } + } +} diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestLongArray.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestLongArray.java new file mode 100644 index 0000000000000..a41742dabb205 --- /dev/null +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestLongArray.java @@ -0,0 +1,41 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * This project is based on a modification of https://github.com/tdunning/t-digest which is licensed under the Apache 2.0 License. + */ + +package org.elasticsearch.tdigest.arrays; + +import org.apache.lucene.util.Accountable; +import org.elasticsearch.core.Releasable; + +/** + * Minimal interface for LongArray-like classes used within TDigest. + */ +public interface TDigestLongArray extends Releasable, Accountable { + int size(); + + long get(int index); + + void set(int index, long value); + + /** + * Resizes the array. If the new size is bigger than the current size, the new elements are set to 0. + */ + void resize(int newSize); +} diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLGroupTreeTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLGroupTreeTests.java index 972a9e3b36878..bbaa829fd37e0 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLGroupTreeTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLGroupTreeTests.java @@ -21,86 +21,88 @@ package org.elasticsearch.tdigest; -import org.elasticsearch.test.ESTestCase; - -public class AVLGroupTreeTests extends ESTestCase { +public class AVLGroupTreeTests extends TDigestTestCase { public void testSimpleAdds() { - AVLGroupTree x = new AVLGroupTree(); - assertEquals(IntAVLTree.NIL, x.floor(34)); - assertEquals(IntAVLTree.NIL, x.first()); - assertEquals(IntAVLTree.NIL, x.last()); - assertEquals(0, x.size()); - assertEquals(0, x.sum()); + try (AVLGroupTree x = AVLGroupTree.create(arrays())) { + assertEquals(IntAVLTree.NIL, x.floor(34)); + assertEquals(IntAVLTree.NIL, x.first()); + assertEquals(IntAVLTree.NIL, x.last()); + assertEquals(0, x.size()); + assertEquals(0, x.sum()); - x.add(new Centroid(1)); - assertEquals(1, x.sum()); - Centroid centroid = new Centroid(2); - centroid.add(3, 1); - centroid.add(4, 1); - x.add(centroid); + x.add(new Centroid(1)); + assertEquals(1, x.sum()); + Centroid centroid = new Centroid(2); + centroid.add(3, 1); + centroid.add(4, 1); + x.add(centroid); - assertEquals(2, x.size()); - assertEquals(4, x.sum()); + assertEquals(2, x.size()); + assertEquals(4, x.sum()); + } } public void testBalancing() { - AVLGroupTree x = new AVLGroupTree(); - for (int i = 0; i < 101; i++) { - x.add(new Centroid(i)); - } + try (AVLGroupTree x = AVLGroupTree.create(arrays())) { + for (int i = 0; i < 101; i++) { + x.add(new Centroid(i)); + } - assertEquals(101, x.size()); - assertEquals(101, x.sum()); + assertEquals(101, x.size()); + assertEquals(101, x.sum()); - x.checkBalance(); - x.checkAggregates(); + x.checkBalance(); + x.checkAggregates(); + } } public void testFloor() { // mostly tested in other tests - AVLGroupTree x = new AVLGroupTree(); - for (int i = 0; i < 101; i++) { - x.add(new Centroid(i / 2)); - } + try (AVLGroupTree x = AVLGroupTree.create(arrays())) { + for (int i = 0; i < 101; i++) { + x.add(new Centroid(i / 2)); + } - assertEquals(IntAVLTree.NIL, x.floor(-30)); + assertEquals(IntAVLTree.NIL, x.floor(-30)); - for (Centroid centroid : x) { - assertEquals(centroid.mean(), x.mean(x.floor(centroid.mean() + 0.1)), 0); + for (Centroid centroid : x) { + assertEquals(centroid.mean(), x.mean(x.floor(centroid.mean() + 0.1)), 0); + } } } public void testHeadSum() { - AVLGroupTree x = new AVLGroupTree(); - for (int i = 0; i < 1000; ++i) { - x.add(randomDouble(), randomIntBetween(1, 10)); - } - long sum = 0; - long last = -1; - for (int node = x.first(); node != IntAVLTree.NIL; node = x.next(node)) { - assertEquals(sum, x.headSum(node)); - sum += x.count(node); - last = x.count(node); + try (AVLGroupTree x = AVLGroupTree.create(arrays())) { + for (int i = 0; i < 1000; ++i) { + x.add(randomDouble(), randomIntBetween(1, 10)); + } + long sum = 0; + long last = -1; + for (int node = x.first(); node != IntAVLTree.NIL; node = x.next(node)) { + assertEquals(sum, x.headSum(node)); + sum += x.count(node); + last = x.count(node); + } + assertEquals(last, x.count(x.last())); } - assertEquals(last, x.count(x.last())); } public void testFloorSum() { - AVLGroupTree x = new AVLGroupTree(); - int total = 0; - for (int i = 0; i < 1000; ++i) { - int count = randomIntBetween(1, 10); - x.add(randomDouble(), count); - total += count; - } - assertEquals(IntAVLTree.NIL, x.floorSum(-1)); - for (long i = 0; i < total + 10; ++i) { - final int floorNode = x.floorSum(i); - assertTrue(x.headSum(floorNode) <= i); - final int next = x.next(floorNode); - assertTrue(next == IntAVLTree.NIL || x.headSum(next) > i); + try (AVLGroupTree x = AVLGroupTree.create(arrays())) { + int total = 0; + for (int i = 0; i < 1000; ++i) { + int count = randomIntBetween(1, 10); + x.add(randomDouble(), count); + total += count; + } + assertEquals(IntAVLTree.NIL, x.floorSum(-1)); + for (long i = 0; i < total + 10; ++i) { + final int floorNode = x.floorSum(i); + assertTrue(x.headSum(floorNode) <= i); + final int next = x.next(floorNode); + assertTrue(next == IntAVLTree.NIL || x.headSum(next) > i); + } } } - } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLTreeDigestTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLTreeDigestTests.java index 7fd3e58da04f9..e7cac9fbef725 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLTreeDigestTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLTreeDigestTests.java @@ -25,7 +25,7 @@ public class AVLTreeDigestTests extends TDigestTests { protected DigestFactory factory(final double compression) { return () -> { - AVLTreeDigest digest = new AVLTreeDigest(compression); + AVLTreeDigest digest = AVLTreeDigest.create(arrays(), compression); digest.setRandomSeed(randomLong()); return digest; }; diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AlternativeMergeTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AlternativeMergeTests.java index b9d36f4c945fe..0639c8a0acbd1 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AlternativeMergeTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AlternativeMergeTests.java @@ -21,14 +21,12 @@ package org.elasticsearch.tdigest; -import org.elasticsearch.test.ESTestCase; - import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Random; -public class AlternativeMergeTests extends ESTestCase { +public class AlternativeMergeTests extends TDigestTestCase { /** * Computes size using the alternative scaling limit for both an idealized merge and for * a MergingDigest. @@ -36,48 +34,51 @@ public class AlternativeMergeTests extends ESTestCase { public void testMerges() { for (int n : new int[] { 100, 1000, 10000, 100000 }) { for (double compression : new double[] { 50, 100, 200, 400 }) { - MergingDigest mergingDigest = new MergingDigest(compression); - AVLTreeDigest treeDigest = new AVLTreeDigest(compression); - List data = new ArrayList<>(); - Random gen = random(); - for (int i = 0; i < n; i++) { - double x = gen.nextDouble(); - data.add(x); - mergingDigest.add(x); - treeDigest.add(x); - } - Collections.sort(data); - List counts = new ArrayList<>(); - double soFar = 0; - double current = 0; - for (Double x : data) { - double q = (soFar + (current + 1.0) / 2) / n; - if (current == 0 || current + 1 < n * Math.PI / compression * Math.sqrt(q * (1 - q))) { - current += 1; - } else { + try ( + MergingDigest mergingDigest = TDigest.createMergingDigest(arrays(), compression); + AVLTreeDigest treeDigest = TDigest.createAvlTreeDigest(arrays(), compression); + ) { + List data = new ArrayList<>(); + Random gen = random(); + for (int i = 0; i < n; i++) { + double x = gen.nextDouble(); + data.add(x); + mergingDigest.add(x); + treeDigest.add(x); + } + Collections.sort(data); + List counts = new ArrayList<>(); + double soFar = 0; + double current = 0; + for (Double x : data) { + double q = (soFar + (current + 1.0) / 2) / n; + if (current == 0 || current + 1 < n * Math.PI / compression * Math.sqrt(q * (1 - q))) { + current += 1; + } else { + counts.add(current); + soFar += current; + current = 1; + } + } + if (current > 0) { counts.add(current); - soFar += current; - current = 1; } + soFar = 0; + for (Double count : counts) { + soFar += count; + } + assertEquals(n, soFar, 0); + soFar = 0; + for (Centroid c : mergingDigest.centroids()) { + soFar += c.count(); + } + assertEquals(n, soFar, 0); + soFar = 0; + for (Centroid c : treeDigest.centroids()) { + soFar += c.count(); + } + assertEquals(n, soFar, 0); } - if (current > 0) { - counts.add(current); - } - soFar = 0; - for (Double count : counts) { - soFar += count; - } - assertEquals(n, soFar, 0); - soFar = 0; - for (Centroid c : mergingDigest.centroids()) { - soFar += c.count(); - } - assertEquals(n, soFar, 0); - soFar = 0; - for (Centroid c : treeDigest.centroids()) { - soFar += c.count(); - } - assertEquals(n, soFar, 0); } } } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTests.java index 68b07f1096eea..ac39bf0f7e8b5 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTests.java @@ -21,16 +21,17 @@ package org.elasticsearch.tdigest; -import org.elasticsearch.test.ESTestCase; - -public abstract class BigCountTests extends ESTestCase { +public abstract class BigCountTests extends TDigestTestCase { public void testBigMerge() { - TDigest digest = createDigest(); - for (int i = 0; i < 5; i++) { - digest.add(getDigest()); - double actual = digest.quantile(0.5); - assertEquals("Count = " + digest.size(), 3000, actual, 0.001); + try (TDigest digest = createDigest()) { + for (int i = 0; i < 5; i++) { + try (TDigest digestToMerge = getDigest()) { + digest.add(digestToMerge); + } + double actual = digest.quantile(0.5); + assertEquals("Count = " + digest.size(), 3000, actual, 0.001); + } } } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsMergingDigestTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsMergingDigestTests.java index d11cc76820823..7a7094691fb95 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsMergingDigestTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsMergingDigestTests.java @@ -24,6 +24,6 @@ public class BigCountTestsMergingDigestTests extends BigCountTests { @Override public TDigest createDigest() { - return new MergingDigest(100); + return TDigest.createMergingDigest(arrays(), 100); } } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsTreeDigestTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsTreeDigestTests.java index 765b7c98d7df4..2978e1c98bcdb 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsTreeDigestTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsTreeDigestTests.java @@ -24,6 +24,6 @@ public class BigCountTestsTreeDigestTests extends BigCountTests { @Override public TDigest createDigest() { - return new AVLTreeDigest(100); + return TDigest.createAvlTreeDigest(arrays(), 100); } } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/ComparisonTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/ComparisonTests.java index 61a546fe3dd3d..b9b22bf1f8480 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/ComparisonTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/ComparisonTests.java @@ -21,12 +21,12 @@ package org.elasticsearch.tdigest; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.core.Releasables; import java.util.Arrays; import java.util.function.Supplier; -public class ComparisonTests extends ESTestCase { +public class ComparisonTests extends TDigestTestCase { private static final int SAMPLE_COUNT = 1_000_000; @@ -39,10 +39,10 @@ public class ComparisonTests extends ESTestCase { private void loadData(Supplier sampleGenerator) { final int COMPRESSION = 100; - avlTreeDigest = TDigest.createAvlTreeDigest(COMPRESSION); - mergingDigest = TDigest.createMergingDigest(COMPRESSION); - sortingDigest = TDigest.createSortingDigest(); - hybridDigest = TDigest.createHybridDigest(COMPRESSION); + avlTreeDigest = TDigest.createAvlTreeDigest(arrays(), COMPRESSION); + mergingDigest = TDigest.createMergingDigest(arrays(), COMPRESSION); + sortingDigest = TDigest.createSortingDigest(arrays()); + hybridDigest = TDigest.createHybridDigest(arrays(), COMPRESSION); samples = new double[SAMPLE_COUNT]; for (int i = 0; i < SAMPLE_COUNT; i++) { @@ -55,6 +55,10 @@ private void loadData(Supplier sampleGenerator) { Arrays.sort(samples); } + private void releaseData() { + Releasables.close(avlTreeDigest, mergingDigest, sortingDigest, hybridDigest); + } + public void testRandomDenseDistribution() { loadData(() -> random().nextDouble()); @@ -67,6 +71,8 @@ public void testRandomDenseDistribution() { assertEquals(String.valueOf(percentile), expected, mergingDigest.quantile(q), accuracy); assertEquals(String.valueOf(percentile), expected, hybridDigest.quantile(q), accuracy); } + + releaseData(); } public void testRandomSparseDistribution() { @@ -81,6 +87,8 @@ public void testRandomSparseDistribution() { assertEquals(String.valueOf(percentile), expected, mergingDigest.quantile(q), accuracy); assertEquals(String.valueOf(percentile), expected, hybridDigest.quantile(q), accuracy); } + + releaseData(); } public void testDenseGaussianDistribution() { @@ -101,6 +109,8 @@ public void testDenseGaussianDistribution() { assertEquals(expectedMedian, avlTreeDigest.quantile(0.5), 0.01); assertEquals(expectedMedian, mergingDigest.quantile(0.5), 0.01); assertEquals(expectedMedian, hybridDigest.quantile(0.5), 0.01); + + releaseData(); } public void testSparseGaussianDistribution() { @@ -122,5 +132,7 @@ public void testSparseGaussianDistribution() { assertEquals(expectedMedian, avlTreeDigest.quantile(0.5), 5000); assertEquals(expectedMedian, mergingDigest.quantile(0.5), 5000); assertEquals(expectedMedian, hybridDigest.quantile(0.5), 5000); + + releaseData(); } } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/HybridDigestTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/HybridDigestTests.java index 019dbdd830182..96adb7b13203d 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/HybridDigestTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/HybridDigestTests.java @@ -24,6 +24,6 @@ public class HybridDigestTests extends TDigestTests { protected DigestFactory factory(final double compression) { - return () -> new HybridDigest(compression); + return () -> HybridDigest.create(arrays(), compression); } } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/IntAVLTreeTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/IntAVLTreeTests.java index 733639978593f..53c4664cbcc0d 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/IntAVLTreeTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/IntAVLTreeTests.java @@ -21,7 +21,7 @@ package org.elasticsearch.tdigest; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.tdigest.arrays.TDigestArrays; import java.util.Arrays; import java.util.Iterator; @@ -29,7 +29,7 @@ import java.util.Random; import java.util.TreeMap; -public class IntAVLTreeTests extends ESTestCase { +public class IntAVLTreeTests extends TDigestTestCase { static class IntegerBag extends IntAVLTree { @@ -37,7 +37,10 @@ static class IntegerBag extends IntAVLTree { int[] values; int[] counts; - IntegerBag() { + IntegerBag(TDigestArrays arrays) { + super(arrays); + // We adjust the breaker after creation as this is just a test class + arrays.adjustBreaker(IntAVLTree.SHALLOW_SIZE); values = new int[capacity()]; counts = new int[capacity()]; } @@ -87,53 +90,54 @@ protected void merge(int node) { public void testDualAdd() { Random r = random(); TreeMap map = new TreeMap<>(); - IntegerBag bag = new IntegerBag(); - for (int i = 0; i < 100000; ++i) { - final int v = r.nextInt(100000); - if (map.containsKey(v)) { - map.put(v, map.get(v) + 1); - assertFalse(bag.addValue(v)); - } else { - map.put(v, 1); - assertTrue(bag.addValue(v)); + try (IntegerBag bag = new IntegerBag(arrays())) { + for (int i = 0; i < 100000; ++i) { + final int v = r.nextInt(100000); + if (map.containsKey(v)) { + map.put(v, map.get(v) + 1); + assertFalse(bag.addValue(v)); + } else { + map.put(v, 1); + assertTrue(bag.addValue(v)); + } } + Iterator> it = map.entrySet().iterator(); + for (int node = bag.first(bag.root()); node != IntAVLTree.NIL; node = bag.next(node)) { + final Map.Entry next = it.next(); + assertEquals(next.getKey().intValue(), bag.values[node]); + assertEquals(next.getValue().intValue(), bag.counts[node]); + } + assertFalse(it.hasNext()); } - Iterator> it = map.entrySet().iterator(); - for (int node = bag.first(bag.root()); node != IntAVLTree.NIL; node = bag.next(node)) { - final Map.Entry next = it.next(); - assertEquals(next.getKey().intValue(), bag.values[node]); - assertEquals(next.getValue().intValue(), bag.counts[node]); - } - assertFalse(it.hasNext()); } public void testDualAddRemove() { Random r = random(); TreeMap map = new TreeMap<>(); - IntegerBag bag = new IntegerBag(); - for (int i = 0; i < 100000; ++i) { - final int v = r.nextInt(1000); - if (r.nextBoolean()) { - // add - if (map.containsKey(v)) { - map.put(v, map.get(v) + 1); - assertFalse(bag.addValue(v)); + try (IntegerBag bag = new IntegerBag(arrays())) { + for (int i = 0; i < 100000; ++i) { + final int v = r.nextInt(1000); + if (r.nextBoolean()) { + // add + if (map.containsKey(v)) { + map.put(v, map.get(v) + 1); + assertFalse(bag.addValue(v)); + } else { + map.put(v, 1); + assertTrue(bag.addValue(v)); + } } else { - map.put(v, 1); - assertTrue(bag.addValue(v)); + // remove + assertEquals(map.remove(v) != null, bag.removeValue(v)); } - } else { - // remove - assertEquals(map.remove(v) != null, bag.removeValue(v)); } + Iterator> it = map.entrySet().iterator(); + for (int node = bag.first(bag.root()); node != IntAVLTree.NIL; node = bag.next(node)) { + final Map.Entry next = it.next(); + assertEquals(next.getKey().intValue(), bag.values[node]); + assertEquals(next.getValue().intValue(), bag.counts[node]); + } + assertFalse(it.hasNext()); } - Iterator> it = map.entrySet().iterator(); - for (int node = bag.first(bag.root()); node != IntAVLTree.NIL; node = bag.next(node)) { - final Map.Entry next = it.next(); - assertEquals(next.getKey().intValue(), bag.values[node]); - assertEquals(next.getValue().intValue(), bag.counts[node]); - } - assertFalse(it.hasNext()); } - } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MedianTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MedianTests.java index b95c81b3c6144..524c1df2a8c92 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MedianTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MedianTests.java @@ -21,52 +21,54 @@ package org.elasticsearch.tdigest; -import org.elasticsearch.test.ESTestCase; - -public class MedianTests extends ESTestCase { +public class MedianTests extends TDigestTestCase { public void testAVL() { double[] data = new double[] { 7, 15, 36, 39, 40, 41 }; - TDigest digest = new AVLTreeDigest(100); - for (double value : data) { - digest.add(value); - } + try (TDigest digest = TDigest.createAvlTreeDigest(arrays(), 100)) { + for (double value : data) { + digest.add(value); + } - assertEquals(37.5, digest.quantile(0.5), 0); - assertEquals(0.5, digest.cdf(37.5), 0); + assertEquals(37.5, digest.quantile(0.5), 0); + assertEquals(0.5, digest.cdf(37.5), 0); + } } public void testMergingDigest() { double[] data = new double[] { 7, 15, 36, 39, 40, 41 }; - TDigest digest = new MergingDigest(100); - for (double value : data) { - digest.add(value); - } + try (TDigest digest = TDigest.createMergingDigest(arrays(), 100)) { + for (double value : data) { + digest.add(value); + } - assertEquals(37.5, digest.quantile(0.5), 0); - assertEquals(0.5, digest.cdf(37.5), 0); + assertEquals(37.5, digest.quantile(0.5), 0); + assertEquals(0.5, digest.cdf(37.5), 0); + } } public void testSortingDigest() { double[] data = new double[] { 7, 15, 36, 39, 40, 41 }; - TDigest digest = new SortingDigest(); - for (double value : data) { - digest.add(value); - } + try (TDigest digest = TDigest.createSortingDigest(arrays())) { + for (double value : data) { + digest.add(value); + } - assertEquals(37.5, digest.quantile(0.5), 0); - assertEquals(0.5, digest.cdf(37.5), 0); + assertEquals(37.5, digest.quantile(0.5), 0); + assertEquals(0.5, digest.cdf(37.5), 0); + } } public void testHybridDigest() { double[] data = new double[] { 7, 15, 36, 39, 40, 41 }; - TDigest digest = new HybridDigest(100); - for (double value : data) { - digest.add(value); - } + try (TDigest digest = TDigest.createHybridDigest(arrays(), 100)) { + for (double value : data) { + digest.add(value); + } - assertEquals(37.5, digest.quantile(0.5), 0); - assertEquals(0.5, digest.cdf(37.5), 0); + assertEquals(37.5, digest.quantile(0.5), 0); + assertEquals(0.5, digest.cdf(37.5), 0); + } } public void testReferenceWikipedia() { diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MergingDigestTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MergingDigestTests.java index 9fadf2218f203..18ef7984242ff 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MergingDigestTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MergingDigestTests.java @@ -21,6 +21,7 @@ package org.elasticsearch.tdigest; +import org.elasticsearch.core.Releasables; import org.junit.Assert; import java.util.ArrayList; @@ -33,45 +34,47 @@ public class MergingDigestTests extends TDigestTests { protected DigestFactory factory(final double compression) { - return () -> new MergingDigest(compression); + return () -> MergingDigest.create(arrays(), compression); } public void testNanDueToBadInitialization() { int compression = 100; int factor = 5; - MergingDigest md = new MergingDigest(compression, (factor + 1) * compression, compression); + try (MergingDigest md = MergingDigest.create(arrays(), compression, (factor + 1) * compression, compression)) { - final int M = 10; - List mds = new ArrayList<>(); - for (int i = 0; i < M; ++i) { - mds.add(new MergingDigest(compression, (factor + 1) * compression, compression)); - } + final int M = 10; + List mds = new ArrayList<>(); + for (int i = 0; i < M; ++i) { + mds.add(MergingDigest.create(arrays(), compression, (factor + 1) * compression, compression)); + } - // Fill all digests with values (0,10,20,...,80). - List raw = new ArrayList<>(); - for (int i = 0; i < 9; ++i) { - double x = 10 * i; - md.add(x); - raw.add(x); - for (int j = 0; j < M; ++j) { - mds.get(j).add(x); + // Fill all digests with values (0,10,20,...,80). + List raw = new ArrayList<>(); + for (int i = 0; i < 9; ++i) { + double x = 10 * i; + md.add(x); raw.add(x); + for (int j = 0; j < M; ++j) { + mds.get(j).add(x); + raw.add(x); + } } - } - Collections.sort(raw); + Collections.sort(raw); - // Merge all mds one at a time into md. - for (int i = 0; i < M; ++i) { - md.add(mds.get(i)); - } - Assert.assertFalse(Double.isNaN(md.quantile(0.01))); + // Merge all mds one at a time into md. + for (int i = 0; i < M; ++i) { + md.add(mds.get(i)); + } + Assert.assertFalse(Double.isNaN(md.quantile(0.01))); - for (double q : new double[] { 0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.90, 0.95, 0.99 }) { - double est = md.quantile(q); - double actual = Dist.quantile(q, raw); - double qx = md.cdf(actual); - Assert.assertEquals(q, qx, 0.5); - Assert.assertEquals(est, actual, 3.8); + for (double q : new double[] { 0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.90, 0.95, 0.99 }) { + double est = md.quantile(q); + double actual = Dist.quantile(q, raw); + double qx = md.cdf(actual); + Assert.assertEquals(q, qx, 0.5); + Assert.assertEquals(est, actual, 3.8); + } + Releasables.close(mds); } } @@ -79,86 +82,90 @@ public void testNanDueToBadInitialization() { * Verifies interpolation between a singleton and a larger centroid. */ public void testSingleMultiRange() { - TDigest digest = factory(100).create(); - digest.setScaleFunction(ScaleFunction.K_0); - for (int i = 0; i < 100; i++) { - digest.add(1); - digest.add(2); - digest.add(3); - } - // this check is, of course true, but it also forces merging before we change scale - assertTrue(digest.centroidCount() < 300); - digest.add(0); - // we now have a digest with a singleton first, then a heavier centroid next - Iterator ix = digest.centroids().iterator(); - Centroid first = ix.next(); - Centroid second = ix.next(); - assertEquals(1, first.count()); - assertEquals(0, first.mean(), 0); - // assertTrue(second.count() > 1); - assertEquals(1.0, second.mean(), 0); + try (TDigest digest = factory(100).create()) { + digest.setScaleFunction(ScaleFunction.K_0); + for (int i = 0; i < 100; i++) { + digest.add(1); + digest.add(2); + digest.add(3); + } + // this check is, of course true, but it also forces merging before we change scale + assertTrue(digest.centroidCount() < 300); + digest.add(0); + // we now have a digest with a singleton first, then a heavier centroid next + Iterator ix = digest.centroids().iterator(); + Centroid first = ix.next(); + Centroid second = ix.next(); + assertEquals(1, first.count()); + assertEquals(0, first.mean(), 0); + // assertTrue(second.count() > 1); + assertEquals(1.0, second.mean(), 0); - assertEquals(0.00166, digest.cdf(0), 1e-5); - assertEquals(0.00166, digest.cdf(1e-10), 1e-5); - assertEquals(0.0025, digest.cdf(0.25), 1e-5); + assertEquals(0.00166, digest.cdf(0), 1e-5); + assertEquals(0.00166, digest.cdf(1e-10), 1e-5); + assertEquals(0.0025, digest.cdf(0.25), 1e-5); + } } /** * Make sure that the first and last centroids have unit weight */ public void testSingletonsAtEnds() { - TDigest d = new MergingDigest(50); - Random gen = random(); - double[] data = new double[100]; - for (int i = 0; i < data.length; i++) { - data[i] = Math.floor(gen.nextGaussian() * 3); - } - for (int i = 0; i < 100; i++) { - for (double x : data) { - d.add(x); + try (TDigest d = MergingDigest.create(arrays(), 50)) { + Random gen = random(); + double[] data = new double[100]; + for (int i = 0; i < data.length; i++) { + data[i] = Math.floor(gen.nextGaussian() * 3); } - } - long last = 0; - for (Centroid centroid : d.centroids()) { - if (last == 0) { - assertEquals(1, centroid.count()); + for (int i = 0; i < 100; i++) { + for (double x : data) { + d.add(x); + } + } + long last = 0; + for (Centroid centroid : d.centroids()) { + if (last == 0) { + assertEquals(1, centroid.count()); + } + last = centroid.count(); } - last = centroid.count(); + assertEquals(1, last); } - assertEquals(1, last); } /** * Verify centroid sizes. */ public void testFill() { - MergingDigest x = new MergingDigest(300); - Random gen = random(); - ScaleFunction scale = x.getScaleFunction(); - double compression = x.compression(); - for (int i = 0; i < 1000000; i++) { - x.add(gen.nextGaussian()); - } - double q0 = 0; - int i = 0; - for (Centroid centroid : x.centroids()) { - double q1 = q0 + (double) centroid.count() / x.size(); - double dk = scale.k(q1, compression, x.size()) - scale.k(q0, compression, x.size()); - if (centroid.count() > 1) { - assertTrue(String.format(Locale.ROOT, "K-size for centroid %d at %.3f is %.3f", i, centroid.mean(), dk), dk <= 1); + try (MergingDigest x = MergingDigest.create(arrays(), 300)) { + Random gen = random(); + ScaleFunction scale = x.getScaleFunction(); + double compression = x.compression(); + for (int i = 0; i < 1000000; i++) { + x.add(gen.nextGaussian()); + } + double q0 = 0; + int i = 0; + for (Centroid centroid : x.centroids()) { + double q1 = q0 + (double) centroid.count() / x.size(); + double dk = scale.k(q1, compression, x.size()) - scale.k(q0, compression, x.size()); + if (centroid.count() > 1) { + assertTrue(String.format(Locale.ROOT, "K-size for centroid %d at %.3f is %.3f", i, centroid.mean(), dk), dk <= 1); + } + q0 = q1; + i++; } - q0 = q1; - i++; } } public void testLargeInputSmallCompression() { - MergingDigest td = new MergingDigest(10); - for (int i = 0; i < 10_000_000; i++) { - td.add(between(0, 3_600_000)); + try (MergingDigest td = MergingDigest.create(arrays(), 10)) { + for (int i = 0; i < 10_000_000; i++) { + td.add(between(0, 3_600_000)); + } + assertTrue(td.centroidCount() < 100); + assertTrue(td.quantile(0.00001) < 100_000); + assertTrue(td.quantile(0.99999) > 3_000_000); } - assertTrue(td.centroidCount() < 100); - assertTrue(td.quantile(0.00001) < 100_000); - assertTrue(td.quantile(0.99999) > 3_000_000); } } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/SortTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/SortTests.java index 7b0d867d21205..f12004d3d6d02 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/SortTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/SortTests.java @@ -21,58 +21,59 @@ package org.elasticsearch.tdigest; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.tdigest.arrays.TDigestIntArray; import java.util.HashMap; -import java.util.Locale; import java.util.Map; import java.util.Random; -public class SortTests extends ESTestCase { - +public class SortTests extends TDigestTestCase { public void testReverse() { - int[] x = new int[0]; + TDigestIntArray x = arrays().newIntArray(0); // don't crash with no input - Sort.reverse(x); + Sort.reverse(x, 0, x.size()); // reverse stuff! - x = new int[] { 1, 2, 3, 4, 5 }; - Sort.reverse(x); + x.close(); + x = arrays().newIntArray(new int[] { 1, 2, 3, 4, 5 }); + Sort.reverse(x, 0, x.size()); for (int i = 0; i < 5; i++) { - assertEquals(5 - i, x[i]); + assertEquals(5 - i, x.get(i)); } // reverse some stuff back Sort.reverse(x, 1, 3); - assertEquals(5, x[0]); - assertEquals(2, x[1]); - assertEquals(3, x[2]); - assertEquals(4, x[3]); - assertEquals(1, x[4]); + assertEquals(5, x.get(0)); + assertEquals(2, x.get(1)); + assertEquals(3, x.get(2)); + assertEquals(4, x.get(3)); + assertEquals(1, x.get(4)); // another no-op Sort.reverse(x, 3, 0); - assertEquals(5, x[0]); - assertEquals(2, x[1]); - assertEquals(3, x[2]); - assertEquals(4, x[3]); - assertEquals(1, x[4]); - - x = new int[] { 1, 2, 3, 4, 5, 6 }; - Sort.reverse(x); + assertEquals(5, x.get(0)); + assertEquals(2, x.get(1)); + assertEquals(3, x.get(2)); + assertEquals(4, x.get(3)); + assertEquals(1, x.get(4)); + + x.close(); + x = arrays().newIntArray(new int[] { 1, 2, 3, 4, 5, 6 }); + Sort.reverse(x, 0, x.size()); for (int i = 0; i < 6; i++) { - assertEquals(6 - i, x[i]); + assertEquals(6 - i, x.get(i)); } + x.close(); } public void testEmpty() { - Sort.sort(new int[] {}, new double[] {}, null, 0); + sort(new int[0], new double[0], 0); } public void testOne() { int[] order = new int[1]; - Sort.sort(order, new double[] { 1 }, new double[] { 1 }, 1); + sort(order, new double[] { 1 }, 1); assertEquals(0, order[0]); } @@ -80,7 +81,7 @@ public void testIdentical() { int[] order = new int[6]; double[] values = new double[6]; - Sort.sort(order, values, null, values.length); + sort(order, values, values.length); checkOrder(order, values); } @@ -92,58 +93,10 @@ public void testRepeated() { values[i] = Math.rint(10 * ((double) i / n)) / 10.0; } - Sort.sort(order, values, null, values.length); + sort(order, values, values.length); checkOrder(order, values); } - public void testRepeatedSortByWeight() { - // this needs to be long enough to force coverage of both quicksort and insertion sort - // (i.e. >64) - int n = 125; - int[] order = new int[n]; - double[] values = new double[n]; - double[] weights = new double[n]; - double totalWeight = 0; - - // generate evenly distributed values and weights - for (int i = 0; i < n; i++) { - int k = ((i + 5) * 37) % n; - values[i] = Math.floor(k / 25.0); - weights[i] = (k % 25) + 1; - totalWeight += weights[i]; - } - - // verify: test weights should be evenly distributed - double[] tmp = new double[5]; - for (int i = 0; i < n; i++) { - tmp[(int) values[i]] += weights[i]; - } - for (double v : tmp) { - assertEquals(totalWeight / tmp.length, v, 0); - } - - // now sort ... - Sort.sort(order, values, weights, n); - - // and verify our somewhat unusual ordering of the result - // within the first two quintiles, value is constant, weights increase within each quintile - int delta = order.length / 5; - double sum = checkSubOrder(0.0, order, values, weights, 0, delta, 1); - assertEquals(totalWeight * 0.2, sum, 0); - sum = checkSubOrder(sum, order, values, weights, delta, 2 * delta, 1); - assertEquals(totalWeight * 0.4, sum, 0); - - // in the middle quintile, weights go up and then down after the median - sum = checkMidOrder(totalWeight / 2, sum, order, values, weights, 2 * delta, 3 * delta); - assertEquals(totalWeight * 0.6, sum, 0); - - // in the last two quintiles, weights decrease - sum = checkSubOrder(sum, order, values, weights, 3 * delta, 4 * delta, -1); - assertEquals(totalWeight * 0.8, sum, 0); - sum = checkSubOrder(sum, order, values, weights, 4 * delta, 5 * delta, -1); - assertEquals(totalWeight, sum, 0); - } - public void testStableSort() { // this needs to be long enough to force coverage of both quicksort and insertion sort // (i.e. >64) @@ -172,7 +125,7 @@ public void testStableSort() { } // now sort ... - Sort.stableSort(order, values, n); + sort(order, values, n); // and verify stability of the ordering // values must be in order and they must appear in their original ordering @@ -184,37 +137,6 @@ public void testStableSort() { } } - private double checkMidOrder(double medianWeight, double sofar, int[] order, double[] values, double[] weights, int start, int end) { - double value = values[order[start]]; - double last = 0; - assertTrue(sofar < medianWeight); - for (int i = start; i < end; i++) { - assertEquals(value, values[order[i]], 0); - double w = weights[order[i]]; - assertTrue(w > 0); - if (sofar > medianWeight) { - w = 2 * medianWeight - w; - } - assertTrue(w >= last); - sofar += weights[order[i]]; - } - assertTrue(sofar > medianWeight); - return sofar; - } - - private double checkSubOrder(double sofar, int[] order, double[] values, double[] weights, int start, int end, int ordering) { - double lastWeight = weights[order[start]] * ordering; - double value = values[order[start]]; - for (int i = start; i < end; i++) { - assertEquals(value, values[order[i]], 0); - double newOrderedWeight = weights[order[i]] * ordering; - assertTrue(newOrderedWeight >= lastWeight); - lastWeight = newOrderedWeight; - sofar += weights[order[i]]; - } - return sofar; - } - public void testShort() { int[] order = new int[6]; double[] values = new double[6]; @@ -224,19 +146,19 @@ public void testShort() { values[i] = 1; } - Sort.sort(order, values, null, values.length); + sort(order, values, values.length); checkOrder(order, values); values[0] = 0.8; values[1] = 0.3; - Sort.sort(order, values, null, values.length); + sort(order, values, values.length); checkOrder(order, values); values[5] = 1.5; values[4] = 1.2; - Sort.sort(order, values, null, values.length); + sort(order, values, values.length); checkOrder(order, values); } @@ -246,7 +168,7 @@ public void testLonger() { for (int i = 0; i < 20; i++) { values[i] = (i * 13) % 20; } - Sort.sort(order, values, null, values.length); + sort(order, values, values.length); checkOrder(order, values); } @@ -270,35 +192,10 @@ public void testMultiPivots() { values[24] = 25; values[26] = 25; - Sort.sort(order, values, null, values.length); + sort(order, values, values.length); checkOrder(order, values); } - public void testMultiPivotsInPlace() { - // more pivots than low split on first pass - // multiple pivots, but more low data on second part of recursion - double[] keys = new double[30]; - for (int i = 0; i < 9; i++) { - keys[i] = i + 20 * (i % 2); - } - - for (int i = 9; i < 20; i++) { - keys[i] = 10; - } - - for (int i = 20; i < 30; i++) { - keys[i] = i - 20 * (i % 2); - } - keys[29] = 29; - keys[24] = 25; - keys[26] = 25; - - double[] v = valuesFromKeys(keys, 0); - - Sort.sort(keys, v); - checkOrder(keys, 0, keys.length, v); - } - public void testRandomized() { Random rand = random(); @@ -309,98 +206,11 @@ public void testRandomized() { values[i] = rand.nextDouble(); } - Sort.sort(order, values, null, values.length); + sort(order, values, values.length); checkOrder(order, values); } } - public void testRandomizedShortSort() { - Random rand = random(); - - for (int k = 0; k < 100; k++) { - double[] keys = new double[30]; - for (int i = 0; i < 10; i++) { - keys[i] = i; - } - for (int i = 10; i < 20; i++) { - keys[i] = rand.nextDouble(); - } - for (int i = 20; i < 30; i++) { - keys[i] = i; - } - double[] v0 = valuesFromKeys(keys, 0); - double[] v1 = valuesFromKeys(keys, 1); - - Sort.sort(keys, 10, 10, v0, v1); - checkOrder(keys, 10, 10, v0, v1); - checkValues(keys, 0, keys.length, v0, v1); - for (int i = 0; i < 10; i++) { - assertEquals(i, keys[i], 0); - } - for (int i = 20; i < 30; i++) { - assertEquals(i, keys[i], 0); - } - } - } - - /** - * Generates a vector of values corresponding to a vector of keys. - * - * @param keys A vector of keys - * @param k Which value vector to generate - * @return The new vector containing frac(key_i * 3 * 5^k) - */ - private double[] valuesFromKeys(double[] keys, int k) { - double[] r = new double[keys.length]; - double scale = 3; - for (int i = 0; i < k; i++) { - scale = scale * 5; - } - for (int i = 0; i < keys.length; i++) { - r[i] = fractionalPart(keys[i] * scale); - } - return r; - } - - /** - * Verifies that keys are in order and that each value corresponds to the keys - * - * @param key Array of keys - * @param start The starting offset of keys and values to check - * @param length The number of keys and values to check - * @param values Arrays of associated values. Value_{ki} = frac(key_i * 3 * 5^k) - */ - private void checkOrder(double[] key, int start, int length, double[]... values) { - assert start + length <= key.length; - - for (int i = start; i < start + length - 1; i++) { - assertTrue(String.format(Locale.ROOT, "bad ordering at %d, %f > %f", i, key[i], key[i + 1]), key[i] <= key[i + 1]); - } - - checkValues(key, start, length, values); - } - - private void checkValues(double[] key, int start, int length, double[]... values) { - double scale = 3; - for (int k = 0; k < values.length; k++) { - double[] v = values[k]; - assertEquals(key.length, v.length); - for (int i = start; i < length; i++) { - assertEquals( - String.format(Locale.ROOT, "value %d not correlated, key=%.5f, k=%d, v=%.5f", i, key[i], k, values[k][i]), - fractionalPart(key[i] * scale), - values[k][i], - 0 - ); - } - scale = scale * 5; - } - } - - private double fractionalPart(double v) { - return v - Math.floor(v); - } - private void checkOrder(int[] order, double[] values) { double previous = -Double.MAX_VALUE; Map counts = new HashMap(); @@ -418,4 +228,10 @@ private void checkOrder(int[] order, double[] values) { assertEquals(1, entry.getValue().intValue()); } } + + private void sort(int[] order, double[] values, int n) { + try (var wrappedOrder = arrays().newIntArray(order); var wrappedValues = arrays().newDoubleArray(values);) { + Sort.stableSort(wrappedOrder, wrappedValues, n); + } + } } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/SortingDigestTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/SortingDigestTests.java index 1c1dbbfa28ae9..ea38959019f0b 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/SortingDigestTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/SortingDigestTests.java @@ -24,7 +24,7 @@ public class SortingDigestTests extends TDigestTests { protected DigestFactory factory(final double compression) { - return SortingDigest::new; + return () -> SortingDigest.create(arrays()); } // Make this test a noop to avoid OOMs. diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestReleasingTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestReleasingTests.java new file mode 100644 index 0000000000000..fbd423e3d0a37 --- /dev/null +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestReleasingTests.java @@ -0,0 +1,89 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * This project is based on a modification of https://github.com/tdunning/t-digest which is licensed under the Apache 2.0 License. + */ + +package org.elasticsearch.tdigest; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.search.aggregations.metrics.MemoryTrackingTDigestArrays; +import org.elasticsearch.tdigest.arrays.TDigestArrays; +import org.elasticsearch.test.ESTestCase; + +import java.util.List; +import java.util.function.Function; +import java.util.function.Supplier; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; + +public class TDigestReleasingTests extends ESTestCase { + @ParametersFactory + public static Iterable parameters() { + return List.of( + makeTDigestParams("Hybrid", (arrays) -> TDigest.createHybridDigest(arrays, 100)), + makeTDigestParams("Merging", (arrays) -> TDigest.createMergingDigest(arrays, 100)), + makeTDigestParams("Sorting", TDigest::createSortingDigest), + makeTDigestParams("AvlTree", (arrays) -> TDigest.createAvlTreeDigest(arrays, 100)) + ); + } + + public record TestCase(String name, CircuitBreaker breaker, Supplier tDigestSupplier) { + @Override + public String toString() { + return name; + } + } + + private static Object[] makeTDigestParams(String name, Function tDigestSupplier) { + var breaker = newLimitedBreaker(ByteSizeValue.ofMb(100)); + return new Object[] { new TestCase(name, breaker, () -> tDigestSupplier.apply(new MemoryTrackingTDigestArrays(breaker))) }; + } + + private final TestCase testCase; + + public TDigestReleasingTests(TestCase testCase) { + this.testCase = testCase; + } + + public void testRelease() { + var breaker = testCase.breaker; + assertThat(breaker.getUsed(), equalTo(0L)); + + var tDigest = testCase.tDigestSupplier.get(); + assertThat(breaker.getUsed(), greaterThan(0L)); + assertThat(breaker.getUsed(), equalTo(tDigest.ramBytesUsed())); + + for (int i = 0; i < 10_000; i++) { + tDigest.add(randomDoubleBetween(-Double.MAX_VALUE, Double.MAX_VALUE, true)); + } + assertThat(breaker.getUsed(), greaterThan(0L)); + assertThat(breaker.getUsed(), equalTo(tDigest.ramBytesUsed())); + + tDigest.close(); + assertThat("close() must release all memory", breaker.getUsed(), equalTo(0L)); + + tDigest.close(); + assertThat("close() must be idempotent", breaker.getUsed(), equalTo(0L)); + } + +} diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTestCase.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTestCase.java new file mode 100644 index 0000000000000..d4240a8e633d1 --- /dev/null +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTestCase.java @@ -0,0 +1,114 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * This project is based on a modification of https://github.com/tdunning/t-digest which is licensed under the Apache 2.0 License. + */ + +package org.elasticsearch.tdigest; + +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.search.aggregations.metrics.MemoryTrackingTDigestArrays; +import org.elasticsearch.tdigest.arrays.TDigestArrays; +import org.elasticsearch.tdigest.arrays.TDigestByteArray; +import org.elasticsearch.tdigest.arrays.TDigestDoubleArray; +import org.elasticsearch.tdigest.arrays.TDigestIntArray; +import org.elasticsearch.tdigest.arrays.TDigestLongArray; +import org.elasticsearch.test.ESTestCase; +import org.junit.After; + +import java.util.Collection; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Base class for TDigest tests that require {@link TDigestArrays} instances. + *

    + * This class provides arrays that will be automatically closed after the test. + * It will also test that all memory have been freed, as the arrays use a counting CircuitBreaker. + *

    + */ +public abstract class TDigestTestCase extends ESTestCase { + private final Collection trackedArrays = ConcurrentHashMap.newKeySet(); + + /** + * Create a new TDigestArrays instance with a limited breaker. This method may be called multiple times. + * + *

    + * The arrays created by this method will be automatically released after the test. + *

    + */ + protected MemoryTrackingTDigestArrays arrays() { + return new MemoryTrackingTDigestArrays(newLimitedBreaker(ByteSizeValue.ofMb(100))); + } + + /** + * Release all arrays before {@link ESTestCase} checks for unreleased bytes. + */ + @After + public void releaseArrays() { + Releasables.close(trackedArrays); + trackedArrays.clear(); + } + + private T register(T releasable) { + trackedArrays.add(releasable); + return releasable; + } + + protected final class DelegatingTDigestArrays implements TDigestArrays { + private final MemoryTrackingTDigestArrays delegate; + + DelegatingTDigestArrays() { + this.delegate = new MemoryTrackingTDigestArrays(newLimitedBreaker(ByteSizeValue.ofMb(100))); + } + + public TDigestDoubleArray newDoubleArray(double[] data) { + return register(delegate.newDoubleArray(data)); + } + + @Override + public void adjustBreaker(long size) { + delegate.adjustBreaker(size); + } + + @Override + public TDigestDoubleArray newDoubleArray(int size) { + return register(delegate.newDoubleArray(size)); + } + + public TDigestIntArray newIntArray(int[] data) { + return register(delegate.newIntArray(data)); + } + + @Override + public TDigestIntArray newIntArray(int size) { + return register(delegate.newIntArray(size)); + } + + @Override + public TDigestLongArray newLongArray(int size) { + return register(delegate.newLongArray(size)); + } + + @Override + public TDigestByteArray newByteArray(int initialSize) { + return register(delegate.newByteArray(initialSize)); + } + } +} diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTests.java index 815346100532c..24dac8242bee9 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTests.java @@ -21,8 +21,6 @@ package org.elasticsearch.tdigest; -import org.elasticsearch.test.ESTestCase; - import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -32,7 +30,7 @@ /** * Base test case for TDigests, just extend this class and implement the abstract methods. */ -public abstract class TDigestTests extends ESTestCase { +public abstract class TDigestTests extends TDigestTestCase { public interface DigestFactory { TDigest create(); @@ -61,6 +59,7 @@ public void testBigJump() { assertEquals(0.95, digest.cdf(500_000), 1e-5); assertEquals(0.975, digest.cdf(1_000_000), 1e-5); + digest.close(); digest = factory(80).create(); digest.setScaleFunction(ScaleFunction.K_0); @@ -74,21 +73,23 @@ public void testBigJump() { assertEquals(19.0, digest.quantile(0.915), 0.1); assertEquals(19.0, digest.quantile(0.935), 0.1); assertEquals(1_000_000.0, digest.quantile(0.965), 0.1); + digest.close(); } public void testSmallCountQuantile() { List data = List.of(15.0, 20.0, 32.0, 60.0); - TDigest td = factory(200).create(); - for (Double datum : data) { - td.add(datum); + try (TDigest td = factory(200).create()) { + for (Double datum : data) { + td.add(datum); + } + assertEquals(15.0, td.quantile(0.00), 1e-5); + assertEquals(16.0, td.quantile(0.10), 1.0); + assertEquals(18.0, td.quantile(0.25), 1.0); + assertEquals(26.0, td.quantile(0.50), 1e-5); + assertEquals(42.0, td.quantile(0.75), 4.0); + assertEquals(55.0, td.quantile(0.90), 5.0); + assertEquals(60.0, td.quantile(1.00), 1e-5); } - assertEquals(15.0, td.quantile(0.00), 1e-5); - assertEquals(16.0, td.quantile(0.10), 1.0); - assertEquals(18.0, td.quantile(0.25), 1.0); - assertEquals(26.0, td.quantile(0.50), 1e-5); - assertEquals(42.0, td.quantile(0.75), 4.0); - assertEquals(55.0, td.quantile(0.90), 5.0); - assertEquals(60.0, td.quantile(1.00), 1e-5); } public void testExplicitSkewedData() { @@ -125,35 +126,37 @@ public void testExplicitSkewedData() { 51242, 54241 }; - TDigest digest = factory().create(); - for (double x : data) { - digest.add(x); - } + try (TDigest digest = factory().create()) { + for (double x : data) { + digest.add(x); + } - assertEquals(Dist.quantile(0.5, data), digest.quantile(0.5), 0); + assertEquals(Dist.quantile(0.5, data), digest.quantile(0.5), 0); + } } public void testQuantile() { double[] samples = new double[] { 1.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 3.0, 3.0, 3.0, 3.0, 4.0, 5.0, 6.0, 7.0 }; - TDigest hist1 = factory().create(); - List data = new ArrayList<>(); + try (TDigest hist1 = factory().create(); TDigest hist2 = factory().create()) { + List data = new ArrayList<>(); - for (int j = 0; j < 100; j++) { - for (double x : samples) { - data.add(x); - hist1.add(x); + for (int j = 0; j < 100; j++) { + for (double x : samples) { + data.add(x); + hist1.add(x); + } } + + hist1.compress(); + hist2.add(hist1); + Collections.sort(data); + hist2.compress(); + double x1 = hist1.quantile(0.5); + double x2 = hist2.quantile(0.5); + assertEquals(Dist.quantile(0.5, data), x1, 0.25); + assertEquals(x1, x2, 0.01); } - TDigest hist2 = factory().create(); - hist1.compress(); - hist2.add(hist1); - Collections.sort(data); - hist2.compress(); - double x1 = hist1.quantile(0.5); - double x2 = hist2.quantile(0.5); - assertEquals(Dist.quantile(0.5, data), x1, 0.25); - assertEquals(x1, x2, 0.01); } /** @@ -161,43 +164,45 @@ public void testQuantile() { */ public void testSingletonQuantiles() { double[] data = new double[11]; - TDigest digest = factory().create(); - for (int i = 0; i < data.length; i++) { - digest.add(i); - data[i] = i; - } + try (TDigest digest = factory().create()) { + for (int i = 0; i < data.length; i++) { + digest.add(i); + data[i] = i; + } - for (double x = digest.getMin() - 0.1; x <= digest.getMax() + 0.1; x += 1e-3) { - assertEquals(String.valueOf(x), Dist.cdf(x, data), digest.cdf(x), 0.1); - } + for (double x = digest.getMin() - 0.1; x <= digest.getMax() + 0.1; x += 1e-3) { + assertEquals(String.valueOf(x), Dist.cdf(x, data), digest.cdf(x), 0.1); + } - for (int i = 0; i <= 1000; i++) { - double q = 0.001 * i; - double dist = Dist.quantile(q, data); - double td = digest.quantile(q); - assertEquals(String.valueOf(q), dist, td, 0.5); + for (int i = 0; i <= 1000; i++) { + double q = 0.001 * i; + double dist = Dist.quantile(q, data); + double td = digest.quantile(q); + assertEquals(String.valueOf(q), dist, td, 0.5); + } } } public void testCentroidsWithIncreasingWeights() { ArrayList data = new ArrayList<>(); - TDigest digest = factory().create(); - for (int i = 1; i <= 10; i++) { - digest.add(i, i); - for (int j = 0; j < i; j++) { - data.add((double) i); + try (TDigest digest = factory().create()) { + for (int i = 1; i <= 10; i++) { + digest.add(i, i); + for (int j = 0; j < i; j++) { + data.add((double) i); + } } - } - for (double x = digest.getMin() - 0.1; x <= digest.getMax() + 0.1; x += 1e-3) { - assertEquals(String.valueOf(x), Dist.cdf(x, data), digest.cdf(x), 0.5); - } + for (double x = digest.getMin() - 0.1; x <= digest.getMax() + 0.1; x += 1e-3) { + assertEquals(String.valueOf(x), Dist.cdf(x, data), digest.cdf(x), 0.5); + } - for (int i = 0; i <= 1000; i++) { - double q = 0.001 * i; - double dist = Dist.quantile(q, data); - double td = digest.quantile(q); - assertEquals(String.valueOf(q), dist, td, 0.75); + for (int i = 0; i <= 1000; i++) { + double q = 0.001 * i; + double dist = Dist.quantile(q, data); + double td = digest.quantile(q); + assertEquals(String.valueOf(q), dist, td, 0.75); + } } } @@ -205,15 +210,16 @@ public void testCentroidsWithIncreasingWeights() { * Verifies behavior involving interpolation between singleton centroids. */ public void testSingleSingleRange() { - TDigest digest = factory().create(); - digest.add(1); - digest.add(2); - digest.add(3); + try (TDigest digest = factory().create()) { + digest.add(1); + digest.add(2); + digest.add(3); - // verify the cdf is a step between singletons - assertEquals(0.5 / 3.0, digest.cdf(1), 0); - assertEquals(1.5 / 3.0, digest.cdf(2), 0); - assertEquals(2.5 / 3.0, digest.cdf(3), 0); + // verify the cdf is a step between singletons + assertEquals(0.5 / 3.0, digest.cdf(1), 0); + assertEquals(1.5 / 3.0, digest.cdf(2), 0); + assertEquals(2.5 / 3.0, digest.cdf(3), 0); + } } /** @@ -242,6 +248,7 @@ public void testSingletonAtEnd() { // normally min == mean[0] because weight[0] == 1 // we can force this not to be true for testing + digest.close(); digest = factory().create(); digest.setScaleFunction(ScaleFunction.K_0); for (int i = 0; i < 100; i++) { @@ -280,219 +287,229 @@ public void testSingletonAtEnd() { assertEquals(4, digest.quantile(1), 0); assertEquals(last.mean(), 4, 0); + digest.close(); } public void testFewRepeatedValues() { - TDigest d = factory().create(); - for (int i = 0; i < 2; ++i) { - d.add(9000); - } - for (int i = 0; i < 11; ++i) { - d.add(3000); - } - for (int i = 0; i < 26; ++i) { - d.add(1000); - } + try (TDigest d = factory().create()) { + for (int i = 0; i < 2; ++i) { + d.add(9000); + } + for (int i = 0; i < 11; ++i) { + d.add(3000); + } + for (int i = 0; i < 26; ++i) { + d.add(1000); + } - assertEquals(3000.0, d.quantile(0.90), 1e-5); - assertEquals(4500.0, d.quantile(0.95), 2000); - assertEquals(8500.0, d.quantile(0.97), 500); - assertEquals(9000.0, d.quantile(0.98), 1e-5); - assertEquals(9000.0, d.quantile(1.00), 1e-5); + assertEquals(3000.0, d.quantile(0.90), 1e-5); + assertEquals(4500.0, d.quantile(0.95), 2000); + assertEquals(8500.0, d.quantile(0.97), 500); + assertEquals(9000.0, d.quantile(0.98), 1e-5); + assertEquals(9000.0, d.quantile(1.00), 1e-5); + } } public void testSingleValue() { Random rand = random(); - final TDigest digest = factory().create(); - final double value = rand.nextDouble() * 1000; - digest.add(value); + try (TDigest digest = factory().create()) { + final double value = rand.nextDouble() * 1000; + digest.add(value); - assertEquals(value, digest.quantile(0.0), 0); - assertEquals(value, digest.quantile(1.0), 0); - assertEquals(value, digest.quantile(rand.nextDouble()), 0); + assertEquals(value, digest.quantile(0.0), 0); + assertEquals(value, digest.quantile(1.0), 0); + assertEquals(value, digest.quantile(rand.nextDouble()), 0); - assertEquals(0.0, digest.cdf(value - 1e-5), 0.0); - assertEquals(1.0, digest.cdf(value + 1e5), 0.0); - assertEquals(0.5, digest.cdf(value), 0.0); + assertEquals(0.0, digest.cdf(value - 1e-5), 0.0); + assertEquals(1.0, digest.cdf(value + 1e5), 0.0); + assertEquals(0.5, digest.cdf(value), 0.0); + } } public void testFewValues() { // When there are few values in the tree, quantiles should be exact - final TDigest digest = factory().create(); - final Random r = random(); - final int length = r.nextInt(10); - final List values = new ArrayList<>(); - for (int i = 0; i < length; ++i) { - final double value; - if (i == 0 || r.nextBoolean()) { - value = r.nextDouble() * 100; - } else { - // introduce duplicates - value = values.get(i - 1); + try (TDigest digest = factory().create()) { + final Random r = random(); + final int length = r.nextInt(10); + final List values = new ArrayList<>(); + for (int i = 0; i < length; ++i) { + final double value; + if (i == 0 || r.nextBoolean()) { + value = r.nextDouble() * 100; + } else { + // introduce duplicates + value = values.get(i - 1); + } + digest.add(value); + values.add(value); + } + Collections.sort(values); + + // for this value of the compression, the tree shouldn't have merged any node + assertEquals(digest.centroids().size(), values.size()); + for (double q : new double[] { 0, 1e-10, 0.5, 1 - 1e-10, 1 }) { + double q1 = Dist.quantile(q, values); + double q2 = digest.quantile(q); + assertEquals(String.valueOf(q), q1, q2, q1); } - digest.add(value); - values.add(value); - } - Collections.sort(values); - - // for this value of the compression, the tree shouldn't have merged any node - assertEquals(digest.centroids().size(), values.size()); - for (double q : new double[] { 0, 1e-10, 0.5, 1 - 1e-10, 1 }) { - double q1 = Dist.quantile(q, values); - double q2 = digest.quantile(q); - assertEquals(String.valueOf(q), q1, q2, q1); } } public void testEmptyDigest() { - TDigest digest = factory().create(); - assertEquals(0, digest.centroids().size()); - assertEquals(0, digest.size()); - assertTrue(Double.isNaN(digest.quantile(random().nextDouble()))); - assertTrue(Double.isNaN(digest.cdf(0))); + try (TDigest digest = factory().create()) { + assertEquals(0, digest.centroids().size()); + assertEquals(0, digest.size()); + assertTrue(Double.isNaN(digest.quantile(random().nextDouble()))); + assertTrue(Double.isNaN(digest.cdf(0))); + } } public void testMoreThan2BValues() { - final TDigest digest = factory().create(); - // carefully build a t-digest that is as if we added 3 uniform values from [0,1] - double n = 3e9; - double q0 = 0; - for (int i = 0; i < 200 && q0 < 1 - 1e-10; ++i) { - double k0 = digest.scale.k(q0, digest.compression(), n); - double q = digest.scale.q(k0 + 1, digest.compression(), n); - int m = (int) Math.max(1, n * (q - q0)); - digest.add((q + q0) / 2, m); - q0 = q0 + m / n; - } - digest.compress(); - assertEquals(3_000_000_000L, digest.size()); - assertTrue(digest.size() > Integer.MAX_VALUE); - final double[] quantiles = new double[] { 0, 0.1, 0.5, 0.9, 1 }; - double prev = Double.NEGATIVE_INFINITY; - for (double q : quantiles) { - final double v = digest.quantile(q); - assertTrue(String.format(Locale.ROOT, "q=%.1f, v=%.4f, pref=%.4f", q, v, prev), v >= prev); - prev = v; + try (TDigest digest = factory().create()) { + // carefully build a t-digest that is as if we added 3 uniform values from [0,1] + double n = 3e9; + double q0 = 0; + for (int i = 0; i < 200 && q0 < 1 - 1e-10; ++i) { + double k0 = digest.scale.k(q0, digest.compression(), n); + double q = digest.scale.q(k0 + 1, digest.compression(), n); + int m = (int) Math.max(1, n * (q - q0)); + digest.add((q + q0) / 2, m); + q0 = q0 + m / n; + } + digest.compress(); + assertEquals(3_000_000_000L, digest.size()); + assertTrue(digest.size() > Integer.MAX_VALUE); + final double[] quantiles = new double[] { 0, 0.1, 0.5, 0.9, 1 }; + double prev = Double.NEGATIVE_INFINITY; + for (double q : quantiles) { + final double v = digest.quantile(q); + assertTrue(String.format(Locale.ROOT, "q=%.1f, v=%.4f, pref=%.4f", q, v, prev), v >= prev); + prev = v; + } } } public void testSorted() { - final TDigest digest = factory().create(); - Random gen = random(); - for (int i = 0; i < 10000; ++i) { - int w = 1 + gen.nextInt(10); - double x = gen.nextDouble(); - for (int j = 0; j < w; j++) { - digest.add(x); + try (TDigest digest = factory().create()) { + Random gen = random(); + for (int i = 0; i < 10000; ++i) { + int w = 1 + gen.nextInt(10); + double x = gen.nextDouble(); + for (int j = 0; j < w; j++) { + digest.add(x); + } } - } - Centroid previous = null; - for (Centroid centroid : digest.centroids()) { - if (previous != null) { - if (previous.mean() <= centroid.mean()) { - assertTrue(Double.compare(previous.mean(), centroid.mean()) <= 0); + Centroid previous = null; + for (Centroid centroid : digest.centroids()) { + if (previous != null) { + if (previous.mean() <= centroid.mean()) { + assertTrue(Double.compare(previous.mean(), centroid.mean()) <= 0); + } } + previous = centroid; } - previous = centroid; } } public void testNaN() { - final TDigest digest = factory().create(); - Random gen = random(); - final int iters = gen.nextInt(100); - for (int i = 0; i < iters; ++i) { - digest.add(gen.nextDouble(), 1 + gen.nextInt(10)); - } - try { - // both versions should fail - if (gen.nextBoolean()) { - digest.add(Double.NaN); - } else { - digest.add(Double.NaN, 1); + try (TDigest digest = factory().create()) { + Random gen = random(); + final int iters = gen.nextInt(100); + for (int i = 0; i < iters; ++i) { + digest.add(gen.nextDouble(), 1 + gen.nextInt(10)); + } + try { + // both versions should fail + if (gen.nextBoolean()) { + digest.add(Double.NaN); + } else { + digest.add(Double.NaN, 1); + } + fail("NaN should be an illegal argument"); + } catch (IllegalArgumentException e) { + // expected } - fail("NaN should be an illegal argument"); - } catch (IllegalArgumentException e) { - // expected } } public void testMidPointRule() { - TDigest dist = factory(200).create(); - dist.add(1); - dist.add(2); - - for (int i = 0; i < 1000; i++) { + try (TDigest dist = factory(200).create()) { dist.add(1); dist.add(2); - if (i % 8 == 0) { - String message = String.format(Locale.ROOT, "i = %d", i); - assertEquals(message, 0, dist.cdf(1 - 1e-9), 0); - assertEquals(message, 0.3, dist.cdf(1), 0.2); - assertEquals(message, 0.8, dist.cdf(2), 0.2); - assertEquals(message, 1, dist.cdf(2 + 1e-9), 0); - - assertEquals(1.0, dist.quantile(0.0), 1e-5); - assertEquals(1.0, dist.quantile(0.1), 1e-5); - assertEquals(1.0, dist.quantile(0.2), 1e-5); - - assertTrue(dist.quantile(0.5) > 1.0); - assertTrue(dist.quantile(0.5) < 2.0); - - assertEquals(2.0, dist.quantile(0.7), 1e-5); - assertEquals(2.0, dist.quantile(0.8), 1e-5); - assertEquals(2.0, dist.quantile(0.9), 1e-5); - assertEquals(2.0, dist.quantile(1.0), 1e-5); + + for (int i = 0; i < 1000; i++) { + dist.add(1); + dist.add(2); + if (i % 8 == 0) { + String message = String.format(Locale.ROOT, "i = %d", i); + assertEquals(message, 0, dist.cdf(1 - 1e-9), 0); + assertEquals(message, 0.3, dist.cdf(1), 0.2); + assertEquals(message, 0.8, dist.cdf(2), 0.2); + assertEquals(message, 1, dist.cdf(2 + 1e-9), 0); + + assertEquals(1.0, dist.quantile(0.0), 1e-5); + assertEquals(1.0, dist.quantile(0.1), 1e-5); + assertEquals(1.0, dist.quantile(0.2), 1e-5); + + assertTrue(dist.quantile(0.5) > 1.0); + assertTrue(dist.quantile(0.5) < 2.0); + + assertEquals(2.0, dist.quantile(0.7), 1e-5); + assertEquals(2.0, dist.quantile(0.8), 1e-5); + assertEquals(2.0, dist.quantile(0.9), 1e-5); + assertEquals(2.0, dist.quantile(1.0), 1e-5); + } } } - } public void testThreePointExample() { - TDigest tdigest = factory().create(); - double x0 = 0.18615591526031494; - double x1 = 0.4241943657398224; - double x2 = 0.8813006281852722; - - tdigest.add(x0); - tdigest.add(x1); - tdigest.add(x2); - - double p10 = tdigest.quantile(0.1); - double p50 = tdigest.quantile(0.5); - double p90 = tdigest.quantile(0.9); - double p95 = tdigest.quantile(0.95); - double p99 = tdigest.quantile(0.99); - - assertTrue(Double.compare(p10, p50) <= 0); - assertTrue(Double.compare(p50, p90) <= 0); - assertTrue(Double.compare(p90, p95) <= 0); - assertTrue(Double.compare(p95, p99) <= 0); - - assertEquals(x0, tdigest.quantile(0.0), 0); - assertEquals(x2, tdigest.quantile(1.0), 0); - - assertTrue(String.valueOf(p10), Double.compare(x0, p10) <= 0); - assertTrue(String.valueOf(p10), Double.compare(x1, p10) >= 0); - assertTrue(String.valueOf(p99), Double.compare(x1, p99) <= 0); - assertTrue(String.valueOf(p99), Double.compare(x2, p99) >= 0); + try (TDigest tdigest = factory().create()) { + double x0 = 0.18615591526031494; + double x1 = 0.4241943657398224; + double x2 = 0.8813006281852722; + + tdigest.add(x0); + tdigest.add(x1); + tdigest.add(x2); + + double p10 = tdigest.quantile(0.1); + double p50 = tdigest.quantile(0.5); + double p90 = tdigest.quantile(0.9); + double p95 = tdigest.quantile(0.95); + double p99 = tdigest.quantile(0.99); + + assertTrue(Double.compare(p10, p50) <= 0); + assertTrue(Double.compare(p50, p90) <= 0); + assertTrue(Double.compare(p90, p95) <= 0); + assertTrue(Double.compare(p95, p99) <= 0); + + assertEquals(x0, tdigest.quantile(0.0), 0); + assertEquals(x2, tdigest.quantile(1.0), 0); + + assertTrue(String.valueOf(p10), Double.compare(x0, p10) <= 0); + assertTrue(String.valueOf(p10), Double.compare(x1, p10) >= 0); + assertTrue(String.valueOf(p99), Double.compare(x1, p99) <= 0); + assertTrue(String.valueOf(p99), Double.compare(x2, p99) >= 0); + } } public void testSingletonInACrowd() { - TDigest dist = factory().create(); - for (int i = 0; i < 10000; i++) { - dist.add(10); + try (TDigest dist = factory().create()) { + for (int i = 0; i < 10000; i++) { + dist.add(10); + } + dist.add(20); + dist.compress(); + + // The actual numbers depend on how the digest get constructed. + // A singleton on the right boundary yields much better accuracy, e.g. q(0.9999) == 10. + // Otherwise, quantiles above 0.9 use interpolation between 10 and 20, thus returning higher values. + assertEquals(10.0, dist.quantile(0), 0); + assertEquals(10.0, dist.quantile(0.9), 0); + assertEquals(19.0, dist.quantile(0.99999), 1); + assertEquals(20.0, dist.quantile(1), 0); } - dist.add(20); - dist.compress(); - - // The actual numbers depend on how the digest get constructed. - // A singleton on the right boundary yields much better accuracy, e.g. q(0.9999) == 10. - // Otherwise, quantiles above 0.9 use interpolation between 10 and 20, thus returning higher values. - assertEquals(10.0, dist.quantile(0), 0); - assertEquals(10.0, dist.quantile(0.9), 0); - assertEquals(19.0, dist.quantile(0.99999), 1); - assertEquals(20.0, dist.quantile(1), 0); } public void testScaling() { @@ -505,41 +522,43 @@ public void testScaling() { Collections.sort(data); for (double compression : new double[] { 10, 20, 50, 100, 200, 500, 1000 }) { - TDigest dist = factory(compression).create(); - for (Double x : data) { - dist.add(x); - } - dist.compress(); - - for (double q : new double[] { 0.001, 0.01, 0.1, 0.5 }) { - double estimate = dist.quantile(q); - double actual = data.get((int) (q * data.size())); - if (Double.compare(estimate, 0) != 0) { - assertTrue(Double.compare(Math.abs(actual - estimate) / estimate, 1) < 0); - } else { - assertEquals(Double.compare(estimate, 0), 0); + try (TDigest dist = factory(compression).create()) { + for (Double x : data) { + dist.add(x); + } + dist.compress(); + + for (double q : new double[] { 0.001, 0.01, 0.1, 0.5 }) { + double estimate = dist.quantile(q); + double actual = data.get((int) (q * data.size())); + if (Double.compare(estimate, 0) != 0) { + assertTrue(Double.compare(Math.abs(actual - estimate) / estimate, 1) < 0); + } else { + assertEquals(Double.compare(estimate, 0), 0); + } } } } } public void testMonotonicity() { - TDigest digest = factory().create(); - final Random gen = random(); - for (int i = 0; i < 100000; i++) { - digest.add(gen.nextDouble()); - } + try (TDigest digest = factory().create()) { + final Random gen = random(); + for (int i = 0; i < 100000; i++) { + digest.add(gen.nextDouble()); + } - double lastQuantile = -1; - double lastX = -1; - for (double z = 0; z <= 1; z += 1e-4) { - double x = digest.quantile(z); - assertTrue("q: " + z + " x: " + x + " last: " + lastX, Double.compare(x, lastX) >= 0); - lastX = x; + double lastQuantile = -1; + double lastX = -1; + for (double z = 0; z <= 1; z += 1e-4) { + double x = digest.quantile(z); + assertTrue("q: " + z + " x: " + x + " last: " + lastX, Double.compare(x, lastX) >= 0); + lastX = x; - double q = digest.cdf(z); - assertTrue("Q: " + z, Double.compare(q, lastQuantile) >= 0); - lastQuantile = q; + double q = digest.cdf(z); + assertTrue("Q: " + z, Double.compare(q, lastQuantile) >= 0); + lastQuantile = q; + } } } } diff --git a/libs/x-content/build.gradle b/libs/x-content/build.gradle index 15a79364559a2..1cf18d46e7610 100644 --- a/libs/x-content/build.gradle +++ b/libs/x-content/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.build' @@ -11,14 +12,14 @@ apply plugin: 'elasticsearch.publish' apply plugin: 'elasticsearch.embedded-providers' embeddedProviders { - impl 'x-content', project(':libs:elasticsearch-x-content:impl') + impl 'x-content', project(':libs:x-content:impl') } dependencies { - api project(':libs:elasticsearch-core') + api project(':libs:core') testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-x-content' + exclude group: 'org.elasticsearch', module: 'x-content' } } diff --git a/libs/x-content/impl/build.gradle b/libs/x-content/impl/build.gradle index 6cf278e826d4c..35e122d336c68 100644 --- a/libs/x-content/impl/build.gradle +++ b/libs/x-content/impl/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.build' @@ -15,8 +16,8 @@ base { String jacksonVersion = "2.17.2" dependencies { - compileOnly project(':libs:elasticsearch-core') - compileOnly project(':libs:elasticsearch-x-content') + compileOnly project(':libs:core') + compileOnly project(':libs:x-content') implementation "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${jacksonVersion}" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${jacksonVersion}" @@ -24,7 +25,7 @@ dependencies { implementation "org.yaml:snakeyaml:${versions.snakeyaml}" testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'elasticsearch-x-content' + exclude group: 'org.elasticsearch', module: 'x-content' } } diff --git a/libs/x-content/impl/src/main/java/module-info.java b/libs/x-content/impl/src/main/java/module-info.java index 0fe9a360c0713..61ca74e4effd2 100644 --- a/libs/x-content/impl/src/main/java/module-info.java +++ b/libs/x-content/impl/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.xcontent.impl { diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/XContentImplUtils.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/XContentImplUtils.java index 3e3fc12f9c16a..558aafa34bac9 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/XContentImplUtils.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/XContentImplUtils.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.provider; diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/XContentParserConfigurationImpl.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/XContentParserConfigurationImpl.java index 2f31cf991dc3b..70adc59b9c6a9 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/XContentParserConfigurationImpl.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/XContentParserConfigurationImpl.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.provider; @@ -18,6 +19,8 @@ import org.elasticsearch.xcontent.provider.filtering.FilterPathBasedFilter; import org.elasticsearch.xcontent.support.filtering.FilterPath; +import java.util.ArrayList; +import java.util.List; import java.util.Set; public class XContentParserConfigurationImpl implements XContentParserConfiguration { @@ -105,12 +108,41 @@ public XContentParserConfiguration withFiltering( Set excludeStrings, boolean filtersMatchFieldNamesWithDots ) { + return withFiltering(null, includeStrings, excludeStrings, filtersMatchFieldNamesWithDots); + } + + public XContentParserConfiguration withFiltering( + String prefixPath, + Set includeStrings, + Set excludeStrings, + boolean filtersMatchFieldNamesWithDots + ) { + FilterPath[] includePaths = FilterPath.compile(includeStrings); + FilterPath[] excludePaths = FilterPath.compile(excludeStrings); + + if (prefixPath != null) { + if (includePaths != null) { + List includeFilters = new ArrayList<>(); + for (var incl : includePaths) { + incl.matches(prefixPath, includeFilters, true); + } + includePaths = includeFilters.isEmpty() ? null : includeFilters.toArray(FilterPath[]::new); + } + + if (excludePaths != null) { + List excludeFilters = new ArrayList<>(); + for (var excl : excludePaths) { + excl.matches(prefixPath, excludeFilters, true); + } + excludePaths = excludeFilters.isEmpty() ? null : excludeFilters.toArray(FilterPath[]::new); + } + } return new XContentParserConfigurationImpl( registry, deprecationHandler, restApiVersion, - FilterPath.compile(includeStrings), - FilterPath.compile(excludeStrings), + includePaths, + excludePaths, filtersMatchFieldNamesWithDots ); } diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/XContentProviderImpl.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/XContentProviderImpl.java index a99d8f0c5604f..5b82b79e6b023 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/XContentProviderImpl.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/XContentProviderImpl.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.provider; diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/cbor/CborXContentGenerator.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/cbor/CborXContentGenerator.java index 86f1eacde7d23..e527054cb9796 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/cbor/CborXContentGenerator.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/cbor/CborXContentGenerator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.provider.cbor; diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/cbor/CborXContentImpl.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/cbor/CborXContentImpl.java index 3aa8323eb5495..690799721d9c0 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/cbor/CborXContentImpl.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/cbor/CborXContentImpl.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.provider.cbor; diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/cbor/CborXContentParser.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/cbor/CborXContentParser.java index 837ad2461e9df..23cc14303f393 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/cbor/CborXContentParser.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/cbor/CborXContentParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.provider.cbor; diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/filtering/FilterPathBasedFilter.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/filtering/FilterPathBasedFilter.java index ea307a1b5da6b..4562afa8af693 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/filtering/FilterPathBasedFilter.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/filtering/FilterPathBasedFilter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.provider.filtering; @@ -95,6 +96,41 @@ public TokenFilter includeProperty(String name) { return filter; } + /** + * This is overridden in order to keep empty arrays in nested exclusions - see #109668. + *

    + * If we are excluding contents, we only want to exclude based on property name - but empty arrays in themselves do not have a property + * name. If the empty array were to be excluded, it should be done by excluding the parent. + *

    + * Note though that the expected behavior seems to be ambiguous if contentsFiltered is true - that is, that the filter has pruned all + * the contents of a given array, such that we are left with the empty array. The behavior below drops that array, for at the time of + * writing, not doing so would cause assertions in JsonXContentFilteringTests to fail, which expect this behavior. Yet it is not obvious + * if dropping the empty array in this case is correct. For example, one could expect this sort of behavior: + *

      + *
    • Document:
      { "myArray": [ { "myField": "myValue" } ]}
    • + *
    • Filter:
      { "exclude": "myArray.myField" }
    • + *
    + * From the user's perspective, this could reasonably yield either of: + *
      + *
    1. { "myArray": []}
    2. + *
    3. Removing {@code myArray} entirely.
    4. + *
    + */ + @Override + public boolean includeEmptyArray(boolean contentsFiltered) { + return inclusive == false && contentsFiltered == false; + } + + /** + * This is overridden in order to keep empty objects in nested exclusions - see #109668. + *

    + * The same logic applies to this as to {@link #includeEmptyArray(boolean)}, only for nested objects instead of nested arrays. + */ + @Override + public boolean includeEmptyObject(boolean contentsFiltered) { + return inclusive == false && contentsFiltered == false; + } + @Override protected boolean _includeScalar() { return inclusive == false; diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonStringEncoderImpl.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonStringEncoderImpl.java index ac1b53e56bfd3..4f209e29b96cb 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonStringEncoderImpl.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonStringEncoderImpl.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.provider.json; diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentGenerator.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentGenerator.java index 09cbdf2d571cd..31190f2c1b73f 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentGenerator.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentGenerator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.provider.json; diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentImpl.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentImpl.java index 4e04230a7486e..c842e3bbc50f4 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentImpl.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentImpl.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.provider.json; diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentParser.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentParser.java index 63191084ca837..38ef8bc2e4ef0 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentParser.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.provider.json; @@ -107,7 +108,11 @@ public String text() throws IOException { if (currentToken().isValue() == false) { throwOnNoText(); } - return parser.getText(); + try { + return parser.getText(); + } catch (JsonParseException e) { + throw newXContentParseException(e); + } } private void throwOnNoText() { diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/smile/SmileXContentGenerator.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/smile/SmileXContentGenerator.java index 3a3c83fe7c998..89f2d9b0928b5 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/smile/SmileXContentGenerator.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/smile/SmileXContentGenerator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.provider.smile; diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/smile/SmileXContentImpl.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/smile/SmileXContentImpl.java index 83528980c2b52..a2139b519d0a0 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/smile/SmileXContentImpl.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/smile/SmileXContentImpl.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.provider.smile; diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/smile/SmileXContentParser.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/smile/SmileXContentParser.java index e8b2681e8dfba..4af70c6e11b45 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/smile/SmileXContentParser.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/smile/SmileXContentParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.provider.smile; diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/yaml/YamlXContentGenerator.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/yaml/YamlXContentGenerator.java index 6fa457a25f152..ff276e63387ae 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/yaml/YamlXContentGenerator.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/yaml/YamlXContentGenerator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.provider.yaml; diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/yaml/YamlXContentImpl.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/yaml/YamlXContentImpl.java index 6e1496bfffd7b..69d9acc50cae9 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/yaml/YamlXContentImpl.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/yaml/YamlXContentImpl.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.provider.yaml; diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/yaml/YamlXContentParser.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/yaml/YamlXContentParser.java index f93d949990733..8538d02a41352 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/yaml/YamlXContentParser.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/yaml/YamlXContentParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.provider.yaml; diff --git a/libs/x-content/impl/src/test/java/org/elasticsearch/xcontent/internal/CborTests.java b/libs/x-content/impl/src/test/java/org/elasticsearch/xcontent/internal/CborTests.java index c144ebe14d8a6..f1c963451b7f7 100644 --- a/libs/x-content/impl/src/test/java/org/elasticsearch/xcontent/internal/CborTests.java +++ b/libs/x-content/impl/src/test/java/org/elasticsearch/xcontent/internal/CborTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.internal; diff --git a/libs/x-content/impl/src/test/java/org/elasticsearch/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java b/libs/x-content/impl/src/test/java/org/elasticsearch/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java index 8ac72489870cd..84f35ede20236 100644 --- a/libs/x-content/impl/src/test/java/org/elasticsearch/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java +++ b/libs/x-content/impl/src/test/java/org/elasticsearch/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.support.filtering; diff --git a/libs/x-content/src/main/java/module-info.java b/libs/x-content/src/main/java/module-info.java index ff3c0b5cf6232..4a6dcb7e41408 100644 --- a/libs/x-content/src/main/java/module-info.java +++ b/libs/x-content/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/AbstractObjectParser.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/AbstractObjectParser.java index 32347297fea73..244e1270fe530 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/AbstractObjectParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/AbstractObjectParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/ConstructingObjectParser.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/ConstructingObjectParser.java index fe8fe262660ee..be13207702627 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/ConstructingObjectParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/ConstructingObjectParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/ContextParser.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/ContextParser.java index 75b8bcc52e22a..01a9a59563042 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/ContextParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/ContextParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/CopyingXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/CopyingXContentParser.java index b8e6e1330e0c2..fcfe49d96aeac 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/CopyingXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/CopyingXContentParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/DeprecationHandler.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/DeprecationHandler.java index 285d4cfe01092..e5bc5453836fd 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/DeprecationHandler.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/DeprecationHandler.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/ErrorOnUnknown.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/ErrorOnUnknown.java index 8ffbdcb55a3f5..1fd986ebeabef 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/ErrorOnUnknown.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/ErrorOnUnknown.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/FilterXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/FilterXContentParser.java index 96d186dd612b0..a9616a6702aa0 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/FilterXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/FilterXContentParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/FilterXContentParserWrapper.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/FilterXContentParserWrapper.java index 8436d073155e9..8bb2591479909 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/FilterXContentParserWrapper.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/FilterXContentParserWrapper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/FlatteningXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/FlatteningXContentParser.java index 9395f07fca79e..fee47734f1cc5 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/FlatteningXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/FlatteningXContentParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/InstantiatingObjectParser.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/InstantiatingObjectParser.java index c46c36f8afae8..cf6cbed0916a3 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/InstantiatingObjectParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/InstantiatingObjectParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/MediaType.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/MediaType.java index 9884e0cdc85b5..f4e3f7e0238ba 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/MediaType.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/MediaType.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/MediaTypeRegistry.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/MediaTypeRegistry.java index 643b48d54fcdd..88724062bb452 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/MediaTypeRegistry.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/MediaTypeRegistry.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/NamedObjectNotFoundException.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/NamedObjectNotFoundException.java index c4cd944d436e1..8b2c208bc29f4 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/NamedObjectNotFoundException.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/NamedObjectNotFoundException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/NamedXContentRegistry.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/NamedXContentRegistry.java index 02caac775c5e3..1b4c0d42764cb 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/NamedXContentRegistry.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/NamedXContentRegistry.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/ObjectParser.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/ObjectParser.java index f16542a3f1886..b28e883476bce 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/ObjectParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/ObjectParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/ObjectPath.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/ObjectPath.java index 128caf39d2634..59b6cafd854e4 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/ObjectPath.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/ObjectPath.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/ParseField.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/ParseField.java index 52966e16b14f1..5cbee0c5981d5 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/ParseField.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/ParseField.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/ParsedMediaType.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/ParsedMediaType.java index 3e0c4d517e935..a887c3202dd11 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/ParsedMediaType.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/ParsedMediaType.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/ParserConstructor.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/ParserConstructor.java index 3760f4cc258fa..d6a66ed9da918 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/ParserConstructor.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/ParserConstructor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/ToXContent.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/ToXContent.java index 139758cd51e99..a2836b05801df 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/ToXContent.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/ToXContent.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/ToXContentFragment.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/ToXContentFragment.java index 461194bb6ca54..77bba3630ae3d 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/ToXContentFragment.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/ToXContentFragment.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/ToXContentObject.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/ToXContentObject.java index ffb5c03f32b1c..e4efebac2934c 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/ToXContentObject.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/ToXContentObject.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContent.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContent.java index 56eb308eaebae..3325287fd6abf 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContent.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContent.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilder.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilder.java index 1be4594b097a6..6f0b473b5ba1f 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilder.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; @@ -39,6 +40,7 @@ import java.util.ServiceLoader; import java.util.Set; import java.util.function.Function; +import java.util.function.LongFunction; /** * A utility to build XContent (ie json). @@ -106,13 +108,15 @@ public static XContentBuilder builder(XContentType xContentType, Set inc private static final Map, Writer> WRITERS; private static final Map, HumanReadableTransformer> HUMAN_READABLE_TRANSFORMERS; private static final Map, Function> DATE_TRANSFORMERS; + private static final LongFunction UNIX_EPOCH_MILLIS_FORMATTER; + static { Map, Writer> writers = new HashMap<>(); writers.put(Boolean.class, (b, v) -> b.value((Boolean) v)); writers.put(boolean[].class, (b, v) -> b.values((boolean[]) v)); writers.put(Byte.class, (b, v) -> b.value((Byte) v)); writers.put(byte[].class, (b, v) -> b.value((byte[]) v)); - writers.put(Date.class, XContentBuilder::timeValue); + writers.put(Date.class, XContentBuilder::timestampValue); writers.put(Double.class, (b, v) -> b.value((Double) v)); writers.put(double[].class, (b, v) -> b.values((double[]) v)); writers.put(Float.class, (b, v) -> b.value((Float) v)); @@ -128,8 +132,8 @@ public static XContentBuilder builder(XContentType xContentType, Set inc writers.put(Locale.class, (b, v) -> b.value(v.toString())); writers.put(Class.class, (b, v) -> b.value(v.toString())); writers.put(ZonedDateTime.class, (b, v) -> b.value(v.toString())); - writers.put(Calendar.class, XContentBuilder::timeValue); - writers.put(GregorianCalendar.class, XContentBuilder::timeValue); + writers.put(Calendar.class, XContentBuilder::timestampValue); + writers.put(GregorianCalendar.class, XContentBuilder::timestampValue); writers.put(BigInteger.class, (b, v) -> b.value((BigInteger) v)); writers.put(BigDecimal.class, (b, v) -> b.value((BigDecimal) v)); @@ -139,6 +143,8 @@ public static XContentBuilder builder(XContentType xContentType, Set inc // treat strings as already converted dateTransformers.put(String.class, Function.identity()); + LongFunction unixEpochMillisFormatter = Long::toString; + // Load pluggable extensions for (XContentBuilderExtension service : ServiceLoader.load(XContentBuilderExtension.class)) { Map, Writer> addlWriters = service.getXContentWriters(); @@ -156,11 +162,14 @@ public static XContentBuilder builder(XContentType xContentType, Set inc writers.putAll(addlWriters); humanReadableTransformer.putAll(addlTransformers); dateTransformers.putAll(addlDateTransformers); + + unixEpochMillisFormatter = service::formatUnixEpochMillis; } WRITERS = Map.copyOf(writers); HUMAN_READABLE_TRANSFORMERS = Map.copyOf(humanReadableTransformer); DATE_TRANSFORMERS = Map.copyOf(dateTransformers); + UNIX_EPOCH_MILLIS_FORMATTER = unixEpochMillisFormatter; } @FunctionalInterface @@ -796,52 +805,53 @@ public XContentBuilder utf8Value(byte[] bytes, int offset, int length) throws IO } //////////////////////////////////////////////////////////////////////////// - // Date + // Timestamps ////////////////////////////////// /** - * Write a time-based field and value, if the passed timeValue is null a - * null value is written, otherwise a date transformers lookup is performed. - - * @throws IllegalArgumentException if there is no transformers for the type of object + * Write a field with a timestamp value: if the passed timestamp is null then writes null, otherwise looks up the date transformer + * for the type of {@code timestamp} and uses it to format the value. + * + * @throws IllegalArgumentException if there is no transformer for the given value type */ - public XContentBuilder timeField(String name, Object timeValue) throws IOException { - return field(name).timeValue(timeValue); + public XContentBuilder timestampField(String name, Object timestamp) throws IOException { + return field(name).timestampValue(timestamp); } /** - * If the {@code humanReadable} flag is set, writes both a formatted and - * unformatted version of the time value using the date transformer for the - * {@link Long} class. + * Writes a field containing the raw number of milliseconds since the unix epoch, and also if the {@code humanReadable} flag is set, + * writes a formatted representation of this value using the UNIX_EPOCH_MILLIS_FORMATTER. */ - public XContentBuilder timeField(String name, String readableName, long value) throws IOException { - assert name.equals(readableName) == false : "expected raw and readable field names to differ, but they were both: " + name; + public XContentBuilder timestampFieldsFromUnixEpochMillis(String rawFieldName, String humanReadableFieldName, long unixEpochMillis) + throws IOException { + assert rawFieldName.equals(humanReadableFieldName) == false + : "expected raw and readable field names to differ, but they were both: " + rawFieldName; if (humanReadable) { - Function longTransformer = DATE_TRANSFORMERS.get(Long.class); - if (longTransformer == null) { - throw new IllegalArgumentException("cannot write time value xcontent for unknown value of type Long"); - } - field(readableName).value(longTransformer.apply(value)); + field(humanReadableFieldName, UNIX_EPOCH_MILLIS_FORMATTER.apply(unixEpochMillis)); } - field(name, value); + field(rawFieldName, unixEpochMillis); return this; } /** - * Write a time-based value, if the value is null a null value is written, - * otherwise a date transformers lookup is performed. - - * @throws IllegalArgumentException if there is no transformers for the type of object + * Write a timestamp value: if the passed timestamp is null then writes null, otherwise looks up the date transformer for the type of + * {@code timestamp} and uses it to format the value. + * + * @throws IllegalArgumentException if there is no transformer for the given value type */ - public XContentBuilder timeValue(Object timeValue) throws IOException { - if (timeValue == null) { + public XContentBuilder timestampValue(Object timestamp) throws IOException { + if (timestamp == null) { return nullValue(); } else { - Function transformer = DATE_TRANSFORMERS.get(timeValue.getClass()); + Function transformer = DATE_TRANSFORMERS.get(timestamp.getClass()); if (transformer == null) { - throw new IllegalArgumentException("cannot write time value xcontent for unknown value of type " + timeValue.getClass()); + final var exception = new IllegalArgumentException( + "cannot write timestamp value xcontent for value of unknown type " + timestamp.getClass() + ); + assert false : exception; + throw exception; } - return value(transformer.apply(timeValue)); + return value(transformer.apply(timestamp)); } } diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilderExtension.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilderExtension.java index d6e6ba12342a9..4e3b442e7d473 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilderExtension.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilderExtension.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; @@ -67,4 +68,9 @@ public interface XContentBuilderExtension { * */ Map, Function> getDateTransformers(); + + /** + * Used to format a {@code long} representing the number of milliseconds since the Unix Epoch. + */ + String formatUnixEpochMillis(long unixEpochMillis); } diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentEOFException.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentEOFException.java index 01a2407598159..f1faa6d8c61a1 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentEOFException.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentEOFException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentFactory.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentFactory.java index 1907e1a18d99a..18f3a99d32798 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentFactory.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentGenerationException.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentGenerationException.java index 16bc380e011e9..0fbd8d52a2975 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentGenerationException.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentGenerationException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentGenerator.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentGenerator.java index add5a913faf8a..bf8d0ebf12fa7 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentGenerator.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentGenerator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentLocation.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentLocation.java index e821b3893d266..b337e99fc4106 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentLocation.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentLocation.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParseException.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParseException.java index cc2af6b52d77a..cc61d49c1624f 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParseException.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParseException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParser.java index f2cc0b959dcb5..554043c2547cb 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParserConfiguration.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParserConfiguration.java index 365ea2ac731d2..59e5cd5d6485c 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParserConfiguration.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParserConfiguration.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; @@ -48,10 +49,27 @@ public interface XContentParserConfiguration { RestApiVersion restApiVersion(); + // TODO: Remove when serverless uses the new API + XContentParserConfiguration withFiltering( + Set includeStrings, + Set excludeStrings, + boolean filtersMatchFieldNamesWithDots + ); + /** * Replace the configured filtering. + * + * @param prefixPath The path to be prepended to each sub-path before applying the include/exclude rules. + * Specify {@code null} if parsing starts from the root. + * @param includeStrings A set of strings representing paths to include during filtering. + * If specified, only these paths will be included in parsing. + * @param excludeStrings A set of strings representing paths to exclude during filtering. + * If specified, these paths will be excluded from parsing. + * @param filtersMatchFieldNamesWithDots Indicates whether filters should match field names containing dots ('.') + * as part of the field name. */ XContentParserConfiguration withFiltering( + String prefixPath, Set includeStrings, Set excludeStrings, boolean filtersMatchFieldNamesWithDots diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentSubParser.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentSubParser.java index 48bfecb3c338f..21e50266c7ffa 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentSubParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentSubParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentType.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentType.java index 71392aeff542b..4a6547b307a61 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentType.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentType.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentUtils.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentUtils.java index 5d6fecea6c1a0..565bb8aca8d13 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentUtils.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentUtils.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/cbor/CborXContent.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/cbor/CborXContent.java index ed35a3e9b38dc..6e6746a9b89b8 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/cbor/CborXContent.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/cbor/CborXContent.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.cbor; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/json/JsonStringEncoder.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/json/JsonStringEncoder.java index 672b129995c86..217dd1b39a92f 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/json/JsonStringEncoder.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/json/JsonStringEncoder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.json; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/json/JsonXContent.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/json/JsonXContent.java index c9190988075eb..9c3a03094ed60 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/json/JsonXContent.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/json/JsonXContent.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.json; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/smile/SmileXContent.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/smile/SmileXContent.java index f8c438dd5424c..8f499ca50d9bc 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/smile/SmileXContent.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/smile/SmileXContent.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.smile; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/spi/XContentProvider.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/spi/XContentProvider.java index b24e9b08cd78c..ec9d737dca012 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/spi/XContentProvider.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/spi/XContentProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.spi; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/AbstractXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/AbstractXContentParser.java index 9672c73ef56df..a3b495b9c3e38 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/AbstractXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/AbstractXContentParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.support; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/MapXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/MapXContentParser.java index bc072aae5b977..8893ed7450c81 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/MapXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/MapXContentParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.support; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/filtering/FilterPath.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/filtering/FilterPath.java index fe9ca4215de66..0b9aa17ae3e78 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/filtering/FilterPath.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/filtering/FilterPath.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.support.filtering; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/yaml/YamlXContent.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/yaml/YamlXContent.java index 33a1a7e9b9738..d35f7c1aabc51 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/yaml/YamlXContent.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/yaml/YamlXContent.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.yaml; diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/ConstructingObjectParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/ConstructingObjectParserTests.java index f0703c626c583..d385ce32e7d36 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/ConstructingObjectParserTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/ConstructingObjectParserTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/InstantiatingObjectParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/InstantiatingObjectParserTests.java index 874cc28028000..0e25fb839aec2 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/InstantiatingObjectParserTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/InstantiatingObjectParserTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/MapXContentParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/MapXContentParserTests.java index 4cac5c844d773..1c968c2ab4a7c 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/MapXContentParserTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/MapXContentParserTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/ObjectParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/ObjectParserTests.java index 14d330af0400d..e4feaa18365f1 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/ObjectParserTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/ObjectParserTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/ObjectPathTests.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/ObjectPathTests.java index 273ea3fa02978..c7f2bc119b16a 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/ObjectPathTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/ObjectPathTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/ParseFieldTests.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/ParseFieldTests.java index 31e19a76c22d2..3bfaa629074f8 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/ParseFieldTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/ParseFieldTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/ParsedMediaTypeTests.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/ParsedMediaTypeTests.java index eb502c15e26aa..9fafd7c7e7150 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/ParsedMediaTypeTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/ParsedMediaTypeTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/SimpleStruct.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/SimpleStruct.java index c8e5547a45ba7..45138bb24d661 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/SimpleStruct.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/SimpleStruct.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/XContentGeneratorTests.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/XContentGeneratorTests.java index ab141f9af484c..1afb38cdadfaf 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/XContentGeneratorTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/XContentGeneratorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/XContentParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/XContentParserTests.java index 58cb0af79e103..5aff60b1a4c75 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/XContentParserTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/XContentParserTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent; diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java index bc454445231f5..481a62a2cd7b9 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.support.filtering; @@ -21,6 +22,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collection; +import java.util.HashSet; import java.util.Set; import java.util.stream.IntStream; @@ -331,6 +333,24 @@ protected final void testFilter(Builder expected, Builder sample, Collection includes, Set excludes, boolean matchFieldNamesWithDots) throws IOException { assertFilterResult(expected.apply(createBuilder()), filter(sample, includes, excludes, matchFieldNamesWithDots)); + + String rootPrefix = "root.path.random"; + if (includes != null) { + Set rootIncludes = new HashSet<>(); + for (var incl : includes) { + rootIncludes.add(rootPrefix + (randomBoolean() ? "." : "*.") + incl); + } + includes = rootIncludes; + } + + if (excludes != null) { + Set rootExcludes = new HashSet<>(); + for (var excl : excludes) { + rootExcludes.add(rootPrefix + (randomBoolean() ? "." : "*.") + excl); + } + excludes = rootExcludes; + } + assertFilterResult(expected.apply(createBuilder()), filterSub(sample, rootPrefix, includes, excludes, matchFieldNamesWithDots)); } public void testArrayWithEmptyObjectInInclude() throws IOException { @@ -412,21 +432,36 @@ private XContentBuilder filter(Builder sample, Set includes, Set && matchFieldNamesWithDots == false) { return filterOnBuilder(sample, includes, excludes); } - return filterOnParser(sample, includes, excludes, matchFieldNamesWithDots); + return filterOnParser(sample, null, includes, excludes, matchFieldNamesWithDots); + } + + private XContentBuilder filterSub( + Builder sample, + String root, + Set includes, + Set excludes, + boolean matchFieldNamesWithDots + ) throws IOException { + return filterOnParser(sample, root, includes, excludes, matchFieldNamesWithDots); } private XContentBuilder filterOnBuilder(Builder sample, Set includes, Set excludes) throws IOException { return sample.apply(XContentBuilder.builder(getXContentType(), includes, excludes)); } - private XContentBuilder filterOnParser(Builder sample, Set includes, Set excludes, boolean matchFieldNamesWithDots) - throws IOException { + private XContentBuilder filterOnParser( + Builder sample, + String rootPath, + Set includes, + Set excludes, + boolean matchFieldNamesWithDots + ) throws IOException { try (XContentBuilder builtSample = sample.apply(createBuilder())) { BytesReference sampleBytes = BytesReference.bytes(builtSample); try ( XContentParser parser = getXContentType().xContent() .createParser( - XContentParserConfiguration.EMPTY.withFiltering(includes, excludes, matchFieldNamesWithDots), + XContentParserConfiguration.EMPTY.withFiltering(rootPath, includes, excludes, matchFieldNamesWithDots), sampleBytes.streamInput() ) ) { diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/CborXContentFilteringTests.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/CborXContentFilteringTests.java index 5b2dce8e10106..7b8c5c1171ed1 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/CborXContentFilteringTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/CborXContentFilteringTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.support.filtering; diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/FilterPathTests.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/FilterPathTests.java index 465bab256ac3b..abe2a78112fd7 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/FilterPathTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/FilterPathTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.support.filtering; diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/JsonXContentFilteringTests.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/JsonXContentFilteringTests.java index 5a27954754d43..f631735b9c32d 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/JsonXContentFilteringTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/JsonXContentFilteringTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.support.filtering; diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/SmileFilteringGeneratorTests.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/SmileFilteringGeneratorTests.java index 13efcc0738949..f759a8d1fa8c9 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/SmileFilteringGeneratorTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/SmileFilteringGeneratorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.support.filtering; diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/YamlFilteringGeneratorTests.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/YamlFilteringGeneratorTests.java index ada8b696c5d64..6838a616b69b1 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/YamlFilteringGeneratorTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/YamlFilteringGeneratorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.xcontent.support.filtering; diff --git a/licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt b/licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt new file mode 100644 index 0000000000000..4aa3d77f2ea7f --- /dev/null +++ b/licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt @@ -0,0 +1,1276 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + + Server Side Public License + VERSION 1, OCTOBER 16, 2018 + + Copyright © 2018 MongoDB, Inc. + + Everyone is permitted to copy and distribute verbatim copies of this + license document, but changing it is not allowed. + + TERMS AND CONDITIONS + + 0. Definitions. + + “This License” refers to Server Side Public License. + + “Copyright” also means copyright-like laws that apply to other kinds of + works, such as semiconductor masks. + + “The Program” refers to any copyrightable work licensed under this + License. Each licensee is addressed as “you”. “Licensees” and + “recipients” may be individuals or organizations. + + To “modify” a work means to copy from or adapt all or part of the work in + a fashion requiring copyright permission, other than the making of an + exact copy. The resulting work is called a “modified version” of the + earlier work or a work “based on” the earlier work. + + A “covered work” means either the unmodified Program or a work based on + the Program. + + To “propagate” a work means to do anything with it that, without + permission, would make you directly or secondarily liable for + infringement under applicable copyright law, except executing it on a + computer or modifying a private copy. Propagation includes copying, + distribution (with or without modification), making available to the + public, and in some countries other activities as well. + + To “convey” a work means any kind of propagation that enables other + parties to make or receive copies. Mere interaction with a user through a + computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays “Appropriate Legal Notices” to the + extent that it includes a convenient and prominently visible feature that + (1) displays an appropriate copyright notice, and (2) tells the user that + there is no warranty for the work (except to the extent that warranties + are provided), that licensees may convey the work under this License, and + how to view a copy of this License. If the interface presents a list of + user commands or options, such as a menu, a prominent item in the list + meets this criterion. + + 1. Source Code. + + The “source code” for a work means the preferred form of the work for + making modifications to it. “Object code” means any non-source form of a + work. + + A “Standard Interface” means an interface that either is an official + standard defined by a recognized standards body, or, in the case of + interfaces specified for a particular programming language, one that is + widely used among developers working in that language. The “System + Libraries” of an executable work include anything, other than the work as + a whole, that (a) is included in the normal form of packaging a Major + Component, but which is not part of that Major Component, and (b) serves + only to enable use of the work with that Major Component, or to implement + a Standard Interface for which an implementation is available to the + public in source code form. A “Major Component”, in this context, means a + major essential component (kernel, window system, and so on) of the + specific operating system (if any) on which the executable work runs, or + a compiler used to produce the work, or an object code interpreter used + to run it. + + The “Corresponding Source” for a work in object code form means all the + source code needed to generate, install, and (for an executable work) run + the object code and to modify the work, including scripts to control + those activities. However, it does not include the work's System + Libraries, or general-purpose tools or generally available free programs + which are used unmodified in performing those activities but which are + not part of the work. For example, Corresponding Source includes + interface definition files associated with source files for the work, and + the source code for shared libraries and dynamically linked subprograms + that the work is specifically designed to require, such as by intimate + data communication or control flow between those subprograms and other + parts of the work. + + The Corresponding Source need not include anything that users can + regenerate automatically from other parts of the Corresponding Source. + + The Corresponding Source for a work in source code form is that same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of + copyright on the Program, and are irrevocable provided the stated + conditions are met. This License explicitly affirms your unlimited + permission to run the unmodified Program, subject to section 13. The + output from running a covered work is covered by this License only if the + output, given its content, constitutes a covered work. This License + acknowledges your rights of fair use or other equivalent, as provided by + copyright law. Subject to section 13, you may make, run and propagate + covered works that you do not convey, without conditions so long as your + license otherwise remains in force. You may convey covered works to + others for the sole purpose of having them make modifications exclusively + for you, or provide you with facilities for running those works, provided + that you comply with the terms of this License in conveying all + material for which you do not control copyright. Those thus making or + running the covered works for you must do so exclusively on your + behalf, under your direction and control, on terms that prohibit them + from making any copies of your copyrighted material outside their + relationship with you. + + Conveying under any other circumstances is permitted solely under the + conditions stated below. Sublicensing is not allowed; section 10 makes it + unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological + measure under any applicable law fulfilling obligations under article 11 + of the WIPO copyright treaty adopted on 20 December 1996, or similar laws + prohibiting or restricting circumvention of such measures. + + When you convey a covered work, you waive any legal power to forbid + circumvention of technological measures to the extent such circumvention is + effected by exercising rights under this License with respect to the + covered work, and you disclaim any intention to limit operation or + modification of the work as a means of enforcing, against the work's users, + your or third parties' legal rights to forbid circumvention of + technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you + receive it, in any medium, provided that you conspicuously and + appropriately publish on each copy an appropriate copyright notice; keep + intact all notices stating that this License and any non-permissive terms + added in accord with section 7 apply to the code; keep intact all notices + of the absence of any warranty; and give all recipients a copy of this + License along with the Program. You may charge any price or no price for + each copy that you convey, and you may offer support or warranty + protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to + produce it from the Program, in the form of source code under the terms + of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified it, + and giving a relevant date. + + b) The work must carry prominent notices stating that it is released + under this License and any conditions added under section 7. This + requirement modifies the requirement in section 4 to “keep intact all + notices”. + + c) You must license the entire work, as a whole, under this License to + anyone who comes into possession of a copy. This License will therefore + apply, along with any applicable section 7 additional terms, to the + whole of the work, and all its parts, regardless of how they are + packaged. This License gives no permission to license the work in any + other way, but it does not invalidate such permission if you have + separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your work + need not make them do so. + + A compilation of a covered work with other separate and independent + works, which are not by their nature extensions of the covered work, and + which are not combined with it such as to form a larger program, in or on + a volume of a storage or distribution medium, is called an “aggregate” if + the compilation and its resulting copyright are not used to limit the + access or legal rights of the compilation's users beyond what the + individual works permit. Inclusion of a covered work in an aggregate does + not cause this License to apply to the other parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms of + sections 4 and 5, provided that you also convey the machine-readable + Corresponding Source under the terms of this License, in one of these + ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium customarily + used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a written + offer, valid for at least three years and valid for as long as you + offer spare parts or customer support for that product model, to give + anyone who possesses the object code either (1) a copy of the + Corresponding Source for all the software in the product that is + covered by this License, on a durable physical medium customarily used + for software interchange, for a price no more than your reasonable cost + of physically performing this conveying of source, or (2) access to + copy the Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This alternative is + allowed only occasionally and noncommercially, and only if you received + the object code with such an offer, in accord with subsection 6b. + + d) Convey the object code by offering access from a designated place + (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to copy + the object code is a network server, the Corresponding Source may be on + a different server (operated by you or a third party) that supports + equivalent copying facilities, provided you maintain clear directions + next to the object code saying where to find the Corresponding Source. + Regardless of what server hosts the Corresponding Source, you remain + obligated to ensure that it is available for as long as needed to + satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided you + inform other peers where the object code and Corresponding Source of + the work are being offered to the general public at no charge under + subsection 6d. + + A separable portion of the object code, whose source code is excluded + from the Corresponding Source as a System Library, need not be included + in conveying the object code work. + + A “User Product” is either (1) a “consumer product”, which means any + tangible personal property which is normally used for personal, family, + or household purposes, or (2) anything designed or sold for incorporation + into a dwelling. In determining whether a product is a consumer product, + doubtful cases shall be resolved in favor of coverage. For a particular + product received by a particular user, “normally used” refers to a + typical or common use of that class of product, regardless of the status + of the particular user or of the way in which the particular user + actually uses, or expects or is expected to use, the product. A product + is a consumer product regardless of whether the product has substantial + commercial, industrial or non-consumer uses, unless such uses represent + the only significant mode of use of the product. + + “Installation Information” for a User Product means any methods, + procedures, authorization keys, or other information required to install + and execute modified versions of a covered work in that User Product from + a modified version of its Corresponding Source. The information must + suffice to ensure that the continued functioning of the modified object + code is in no case prevented or interfered with solely because + modification has been made. + + If you convey an object code work under this section in, or with, or + specifically for use in, a User Product, and the conveying occurs as part + of a transaction in which the right of possession and use of the User + Product is transferred to the recipient in perpetuity or for a fixed term + (regardless of how the transaction is characterized), the Corresponding + Source conveyed under this section must be accompanied by the + Installation Information. But this requirement does not apply if neither + you nor any third party retains the ability to install modified object + code on the User Product (for example, the work has been installed in + ROM). + + The requirement to provide Installation Information does not include a + requirement to continue to provide support service, warranty, or updates + for a work that has been modified or installed by the recipient, or for + the User Product in which it has been modified or installed. Access + to a network may be denied when the modification itself materially + and adversely affects the operation of the network or violates the + rules and protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, in + accord with this section must be in a format that is publicly documented + (and with an implementation available to the public in source code form), + and must require no special password or key for unpacking, reading or + copying. + + 7. Additional Terms. + + “Additional permissions” are terms that supplement the terms of this + License by making exceptions from one or more of its conditions. + Additional permissions that are applicable to the entire Program shall be + treated as though they were included in this License, to the extent that + they are valid under applicable law. If additional permissions apply only + to part of the Program, that part may be used separately under those + permissions, but the entire Program remains governed by this License + without regard to the additional permissions. When you convey a copy of + a covered work, you may at your option remove any additional permissions + from that copy, or from any part of it. (Additional permissions may be + written to require their own removal in certain cases when you modify the + work.) You may place additional permissions on material, added by you to + a covered work, for which you have or can give appropriate copyright + permission. + + Notwithstanding any other provision of this License, for material you add + to a covered work, you may (if authorized by the copyright holders of + that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some trade + names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that material + by anyone who conveys the material (or modified versions of it) with + contractual assumptions of liability to the recipient, for any + liability that these contractual assumptions directly impose on those + licensors and authors. + + All other non-permissive additional terms are considered “further + restrictions” within the meaning of section 10. If the Program as you + received it, or any part of it, contains a notice stating that it is + governed by this License along with a term that is a further restriction, + you may remove that term. If a license document contains a further + restriction but permits relicensing or conveying under this License, you + may add to a covered work material governed by the terms of that license + document, provided that the further restriction does not survive such + relicensing or conveying. + + If you add terms to a covered work in accord with this section, you must + place, in the relevant source files, a statement of the additional terms + that apply to those files, or a notice indicating where to find the + applicable terms. Additional terms, permissive or non-permissive, may be + stated in the form of a separately written license, or stated as + exceptions; the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly + provided under this License. Any attempt otherwise to propagate or modify + it is void, and will automatically terminate your rights under this + License (including any patent licenses granted under the third paragraph + of section 11). + + However, if you cease all violation of this License, then your license + from a particular copyright holder is reinstated (a) provisionally, + unless and until the copyright holder explicitly and finally terminates + your license, and (b) permanently, if the copyright holder fails to + notify you of the violation by some reasonable means prior to 60 days + after the cessation. + + Moreover, your license from a particular copyright holder is reinstated + permanently if the copyright holder notifies you of the violation by some + reasonable means, this is the first time you have received notice of + violation of this License (for any work) from that copyright holder, and + you cure the violation prior to 30 days after your receipt of the notice. + + Termination of your rights under this section does not terminate the + licenses of parties who have received copies or rights from you under + this License. If your rights have been terminated and not permanently + reinstated, you do not qualify to receive new licenses for the same + material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or run a + copy of the Program. Ancillary propagation of a covered work occurring + solely as a consequence of using peer-to-peer transmission to receive a + copy likewise does not require acceptance. However, nothing other than + this License grants you permission to propagate or modify any covered + work. These actions infringe copyright if you do not accept this License. + Therefore, by modifying or propagating a covered work, you indicate your + acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically receives + a license from the original licensors, to run, modify and propagate that + work, subject to this License. You are not responsible for enforcing + compliance by third parties with this License. + + An “entity transaction” is a transaction transferring control of an + organization, or substantially all assets of one, or subdividing an + organization, or merging organizations. If propagation of a covered work + results from an entity transaction, each party to that transaction who + receives a copy of the work also receives whatever licenses to the work + the party's predecessor in interest had or could give under the previous + paragraph, plus a right to possession of the Corresponding Source of the + work from the predecessor in interest, if the predecessor has it or can + get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the rights + granted or affirmed under this License. For example, you may not impose a + license fee, royalty, or other charge for exercise of rights granted + under this License, and you may not initiate litigation (including a + cross-claim or counterclaim in a lawsuit) alleging that any patent claim + is infringed by making, using, selling, offering for sale, or importing + the Program or any portion of it. + + 11. Patents. + + A “contributor” is a copyright holder who authorizes use under this + License of the Program or a work on which the Program is based. The work + thus licensed is called the contributor's “contributor version”. + + A contributor's “essential patent claims” are all patent claims owned or + controlled by the contributor, whether already acquired or hereafter + acquired, that would be infringed by some manner, permitted by this + License, of making, using, or selling its contributor version, but do not + include claims that would be infringed only as a consequence of further + modification of the contributor version. For purposes of this definition, + “control” includes the right to grant patent sublicenses in a manner + consistent with the requirements of this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free + patent license under the contributor's essential patent claims, to make, + use, sell, offer for sale, import and otherwise run, modify and propagate + the contents of its contributor version. + + In the following three paragraphs, a “patent license” is any express + agreement or commitment, however denominated, not to enforce a patent + (such as an express permission to practice a patent or covenant not to + sue for patent infringement). To “grant” such a patent license to a party + means to make such an agreement or commitment not to enforce a patent + against the party. + + If you convey a covered work, knowingly relying on a patent license, and + the Corresponding Source of the work is not available for anyone to copy, + free of charge and under the terms of this License, through a publicly + available network server or other readily accessible means, then you must + either (1) cause the Corresponding Source to be so available, or (2) + arrange to deprive yourself of the benefit of the patent license for this + particular work, or (3) arrange, in a manner consistent with the + requirements of this License, to extend the patent license to downstream + recipients. “Knowingly relying” means you have actual knowledge that, but + for the patent license, your conveying the covered work in a country, or + your recipient's use of the covered work in a country, would infringe + one or more identifiable patents in that country that you have reason + to believe are valid. + + If, pursuant to or in connection with a single transaction or + arrangement, you convey, or propagate by procuring conveyance of, a + covered work, and grant a patent license to some of the parties receiving + the covered work authorizing them to use, propagate, modify or convey a + specific copy of the covered work, then the patent license you grant is + automatically extended to all recipients of the covered work and works + based on it. + + A patent license is “discriminatory” if it does not include within the + scope of its coverage, prohibits the exercise of, or is conditioned on + the non-exercise of one or more of the rights that are specifically + granted under this License. You may not convey a covered work if you are + a party to an arrangement with a third party that is in the business of + distributing software, under which you make payment to the third party + based on the extent of your activity of conveying the work, and under + which the third party grants, to any of the parties who would receive the + covered work from you, a discriminatory patent license (a) in connection + with copies of the covered work conveyed by you (or copies made from + those copies), or (b) primarily for and in connection with specific + products or compilations that contain the covered work, unless you + entered into that arrangement, or that patent license was granted, prior + to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting any + implied license or other defenses to infringement that may otherwise be + available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or + otherwise) that contradict the conditions of this License, they do not + excuse you from the conditions of this License. If you cannot use, + propagate or convey a covered work so as to satisfy simultaneously your + obligations under this License and any other pertinent obligations, then + as a consequence you may not use, propagate or convey it at all. For + example, if you agree to terms that obligate you to collect a royalty for + further conveying from those to whom you convey the Program, the only way + you could satisfy both those terms and this License would be to refrain + entirely from conveying the Program. + + 13. Offering the Program as a Service. + + If you make the functionality of the Program or a modified version + available to third parties as a service, you must make the Service Source + Code available via network download to everyone at no charge, under the + terms of this License. Making the functionality of the Program or + modified version available to third parties as a service includes, + without limitation, enabling third parties to interact with the + functionality of the Program or modified version remotely through a + computer network, offering a service the value of which entirely or + primarily derives from the value of the Program or modified version, or + offering a service that accomplishes for users the primary purpose of the + Program or modified version. + + “Service Source Code” means the Corresponding Source for the Program or + the modified version, and the Corresponding Source for all programs that + you use to make the Program or modified version available as a service, + including, without limitation, management software, user interfaces, + application program interfaces, automation software, monitoring software, + backup software, storage software and hosting software, all such that a + user could run an instance of the service using the Service Source Code + you make available. + + 14. Revised Versions of this License. + + MongoDB, Inc. may publish revised and/or new versions of the Server Side + Public License from time to time. Such new versions will be similar in + spirit to the present version, but may differ in detail to address new + problems or concerns. + + Each version is given a distinguishing version number. If the Program + specifies that a certain numbered version of the Server Side Public + License “or any later version” applies to it, you have the option of + following the terms and conditions either of that numbered version or of + any later version published by MongoDB, Inc. If the Program does not + specify a version number of the Server Side Public License, you may + choose any version ever published by MongoDB, Inc. + + If the Program specifies that a proxy can decide which future versions of + the Server Side Public License can be used, that proxy's public statement + of acceptance of a version permanently authorizes you to choose that + version for the Program. + + Later license versions may give you additional or different permissions. + However, no additional obligations are imposed on any author or copyright + holder as a result of your choosing to follow a later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY + APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT + HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY + OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, + THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM + IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF + ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING + WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS + THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING + ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF + THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO + LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU + OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER + PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE + POSSIBILITY OF SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided above + cannot be given local legal effect according to their terms, reviewing + courts shall apply local law that most closely approximates an absolute + waiver of all civil liability in connection with the Program, unless a + warranty or assumption of liability accompanies a copy of the Program in + return for a fee. + + END OF TERMS AND CONDITIONS + + + + + +Elastic License 2.0 + +URL: https://www.elastic.co/licensing/elastic-license + +## Acceptance + +By using the software, you agree to all of the terms and conditions below. + +## Copyright License + +The licensor grants you a non-exclusive, royalty-free, worldwide, +non-sublicensable, non-transferable license to use, copy, distribute, make +available, and prepare derivative works of the software, in each case subject to +the limitations and conditions below. + +## Limitations + +You may not provide the software to third parties as a hosted or managed +service, where the service provides users with access to any substantial set of +the features or functionality of the software. + +You may not move, change, disable, or circumvent the license key functionality +in the software, and you may not remove or obscure any functionality in the +software that is protected by the license key. + +You may not alter, remove, or obscure any licensing, copyright, or other notices +of the licensor in the software. Any use of the licensor’s trademarks is subject +to applicable law. + +## Patents + +The licensor grants you a license, under any patent claims the licensor can +license, or becomes able to license, to make, have made, use, sell, offer for +sale, import and have imported the software, in each case subject to the +limitations and conditions in this license. This license does not cover any +patent claims that you cause to be infringed by modifications or additions to +the software. If you or your company make any written claim that the software +infringes or contributes to infringement of any patent, your patent license for +the software granted under these terms ends immediately. If your company makes +such a claim, your patent license ends immediately for work on behalf of your +company. + +## Notices + +You must ensure that anyone who gets a copy of any part of the software from you +also gets a copy of these terms. + +If you modify the software, you must include in any modified copies of the +software prominent notices stating that you have modified the software. + +## No Other Rights + +These terms do not imply any licenses other than those expressly granted in +these terms. + +## Termination + +If you use the software in violation of these terms, such use is not licensed, +and your licenses will automatically terminate. If the licensor provides you +with a notice of your violation, and you cease all violation of this license no +later than 30 days after you receive that notice, your licenses will be +reinstated retroactively. However, if you violate these terms after such +reinstatement, any additional violation of these terms will cause your licenses +to terminate automatically and permanently. + +## No Liability + +*As far as the law allows, the software comes as is, without any warranty or +condition, and the licensor will not be liable to you for any damages arising +out of these terms or the use or nature of the software, under any kind of +legal claim.* + +## Definitions + +The **licensor** is the entity offering these terms, and the **software** is the +software the licensor makes available under these terms, including any portion +of it. + +**you** refers to the individual or entity agreeing to these terms. + +**your company** is any legal entity, sole proprietorship, or other kind of +organization that you work for, plus all organizations that have control over, +are under the control of, or are under common control with that +organization. **control** means ownership of substantially all the assets of an +entity, or the power to direct its management and policies by vote, contract, or +otherwise. Control can be direct or indirect. + +**your licenses** are all the licenses granted to you for the software under +these terms. + +**use** means anything you do with the software requiring one of your licenses. + +**trademark** means trademarks, service marks, and similar rights. diff --git a/licenses/SSPL-1.0+ELASTIC-LICENSE-2.0.txt b/licenses/SSPL-1.0+ELASTIC-LICENSE-2.0.txt deleted file mode 100644 index 5851192fd8d4a..0000000000000 --- a/licenses/SSPL-1.0+ELASTIC-LICENSE-2.0.txt +++ /dev/null @@ -1,655 +0,0 @@ - Server Side Public License - VERSION 1, OCTOBER 16, 2018 - - Copyright © 2018 MongoDB, Inc. - - Everyone is permitted to copy and distribute verbatim copies of this - license document, but changing it is not allowed. - - TERMS AND CONDITIONS - - 0. Definitions. - - “This License” refers to Server Side Public License. - - “Copyright” also means copyright-like laws that apply to other kinds of - works, such as semiconductor masks. - - “The Program” refers to any copyrightable work licensed under this - License. Each licensee is addressed as “you”. “Licensees” and - “recipients” may be individuals or organizations. - - To “modify” a work means to copy from or adapt all or part of the work in - a fashion requiring copyright permission, other than the making of an - exact copy. The resulting work is called a “modified version” of the - earlier work or a work “based on” the earlier work. - - A “covered work” means either the unmodified Program or a work based on - the Program. - - To “propagate” a work means to do anything with it that, without - permission, would make you directly or secondarily liable for - infringement under applicable copyright law, except executing it on a - computer or modifying a private copy. Propagation includes copying, - distribution (with or without modification), making available to the - public, and in some countries other activities as well. - - To “convey” a work means any kind of propagation that enables other - parties to make or receive copies. Mere interaction with a user through a - computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays “Appropriate Legal Notices” to the - extent that it includes a convenient and prominently visible feature that - (1) displays an appropriate copyright notice, and (2) tells the user that - there is no warranty for the work (except to the extent that warranties - are provided), that licensees may convey the work under this License, and - how to view a copy of this License. If the interface presents a list of - user commands or options, such as a menu, a prominent item in the list - meets this criterion. - - 1. Source Code. - - The “source code” for a work means the preferred form of the work for - making modifications to it. “Object code” means any non-source form of a - work. - - A “Standard Interface” means an interface that either is an official - standard defined by a recognized standards body, or, in the case of - interfaces specified for a particular programming language, one that is - widely used among developers working in that language. The “System - Libraries” of an executable work include anything, other than the work as - a whole, that (a) is included in the normal form of packaging a Major - Component, but which is not part of that Major Component, and (b) serves - only to enable use of the work with that Major Component, or to implement - a Standard Interface for which an implementation is available to the - public in source code form. A “Major Component”, in this context, means a - major essential component (kernel, window system, and so on) of the - specific operating system (if any) on which the executable work runs, or - a compiler used to produce the work, or an object code interpreter used - to run it. - - The “Corresponding Source” for a work in object code form means all the - source code needed to generate, install, and (for an executable work) run - the object code and to modify the work, including scripts to control - those activities. However, it does not include the work's System - Libraries, or general-purpose tools or generally available free programs - which are used unmodified in performing those activities but which are - not part of the work. For example, Corresponding Source includes - interface definition files associated with source files for the work, and - the source code for shared libraries and dynamically linked subprograms - that the work is specifically designed to require, such as by intimate - data communication or control flow between those subprograms and other - parts of the work. - - The Corresponding Source need not include anything that users can - regenerate automatically from other parts of the Corresponding Source. - - The Corresponding Source for a work in source code form is that same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of - copyright on the Program, and are irrevocable provided the stated - conditions are met. This License explicitly affirms your unlimited - permission to run the unmodified Program, subject to section 13. The - output from running a covered work is covered by this License only if the - output, given its content, constitutes a covered work. This License - acknowledges your rights of fair use or other equivalent, as provided by - copyright law. Subject to section 13, you may make, run and propagate - covered works that you do not convey, without conditions so long as your - license otherwise remains in force. You may convey covered works to - others for the sole purpose of having them make modifications exclusively - for you, or provide you with facilities for running those works, provided - that you comply with the terms of this License in conveying all - material for which you do not control copyright. Those thus making or - running the covered works for you must do so exclusively on your - behalf, under your direction and control, on terms that prohibit them - from making any copies of your copyrighted material outside their - relationship with you. - - Conveying under any other circumstances is permitted solely under the - conditions stated below. Sublicensing is not allowed; section 10 makes it - unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological - measure under any applicable law fulfilling obligations under article 11 - of the WIPO copyright treaty adopted on 20 December 1996, or similar laws - prohibiting or restricting circumvention of such measures. - - When you convey a covered work, you waive any legal power to forbid - circumvention of technological measures to the extent such circumvention is - effected by exercising rights under this License with respect to the - covered work, and you disclaim any intention to limit operation or - modification of the work as a means of enforcing, against the work's users, - your or third parties' legal rights to forbid circumvention of - technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you - receive it, in any medium, provided that you conspicuously and - appropriately publish on each copy an appropriate copyright notice; keep - intact all notices stating that this License and any non-permissive terms - added in accord with section 7 apply to the code; keep intact all notices - of the absence of any warranty; and give all recipients a copy of this - License along with the Program. You may charge any price or no price for - each copy that you convey, and you may offer support or warranty - protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to - produce it from the Program, in the form of source code under the terms - of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified it, - and giving a relevant date. - - b) The work must carry prominent notices stating that it is released - under this License and any conditions added under section 7. This - requirement modifies the requirement in section 4 to “keep intact all - notices”. - - c) You must license the entire work, as a whole, under this License to - anyone who comes into possession of a copy. This License will therefore - apply, along with any applicable section 7 additional terms, to the - whole of the work, and all its parts, regardless of how they are - packaged. This License gives no permission to license the work in any - other way, but it does not invalidate such permission if you have - separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your work - need not make them do so. - - A compilation of a covered work with other separate and independent - works, which are not by their nature extensions of the covered work, and - which are not combined with it such as to form a larger program, in or on - a volume of a storage or distribution medium, is called an “aggregate” if - the compilation and its resulting copyright are not used to limit the - access or legal rights of the compilation's users beyond what the - individual works permit. Inclusion of a covered work in an aggregate does - not cause this License to apply to the other parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms of - sections 4 and 5, provided that you also convey the machine-readable - Corresponding Source under the terms of this License, in one of these - ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium customarily - used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a written - offer, valid for at least three years and valid for as long as you - offer spare parts or customer support for that product model, to give - anyone who possesses the object code either (1) a copy of the - Corresponding Source for all the software in the product that is - covered by this License, on a durable physical medium customarily used - for software interchange, for a price no more than your reasonable cost - of physically performing this conveying of source, or (2) access to - copy the Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This alternative is - allowed only occasionally and noncommercially, and only if you received - the object code with such an offer, in accord with subsection 6b. - - d) Convey the object code by offering access from a designated place - (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to copy - the object code is a network server, the Corresponding Source may be on - a different server (operated by you or a third party) that supports - equivalent copying facilities, provided you maintain clear directions - next to the object code saying where to find the Corresponding Source. - Regardless of what server hosts the Corresponding Source, you remain - obligated to ensure that it is available for as long as needed to - satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided you - inform other peers where the object code and Corresponding Source of - the work are being offered to the general public at no charge under - subsection 6d. - - A separable portion of the object code, whose source code is excluded - from the Corresponding Source as a System Library, need not be included - in conveying the object code work. - - A “User Product” is either (1) a “consumer product”, which means any - tangible personal property which is normally used for personal, family, - or household purposes, or (2) anything designed or sold for incorporation - into a dwelling. In determining whether a product is a consumer product, - doubtful cases shall be resolved in favor of coverage. For a particular - product received by a particular user, “normally used” refers to a - typical or common use of that class of product, regardless of the status - of the particular user or of the way in which the particular user - actually uses, or expects or is expected to use, the product. A product - is a consumer product regardless of whether the product has substantial - commercial, industrial or non-consumer uses, unless such uses represent - the only significant mode of use of the product. - - “Installation Information” for a User Product means any methods, - procedures, authorization keys, or other information required to install - and execute modified versions of a covered work in that User Product from - a modified version of its Corresponding Source. The information must - suffice to ensure that the continued functioning of the modified object - code is in no case prevented or interfered with solely because - modification has been made. - - If you convey an object code work under this section in, or with, or - specifically for use in, a User Product, and the conveying occurs as part - of a transaction in which the right of possession and use of the User - Product is transferred to the recipient in perpetuity or for a fixed term - (regardless of how the transaction is characterized), the Corresponding - Source conveyed under this section must be accompanied by the - Installation Information. But this requirement does not apply if neither - you nor any third party retains the ability to install modified object - code on the User Product (for example, the work has been installed in - ROM). - - The requirement to provide Installation Information does not include a - requirement to continue to provide support service, warranty, or updates - for a work that has been modified or installed by the recipient, or for - the User Product in which it has been modified or installed. Access - to a network may be denied when the modification itself materially - and adversely affects the operation of the network or violates the - rules and protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, in - accord with this section must be in a format that is publicly documented - (and with an implementation available to the public in source code form), - and must require no special password or key for unpacking, reading or - copying. - - 7. Additional Terms. - - “Additional permissions” are terms that supplement the terms of this - License by making exceptions from one or more of its conditions. - Additional permissions that are applicable to the entire Program shall be - treated as though they were included in this License, to the extent that - they are valid under applicable law. If additional permissions apply only - to part of the Program, that part may be used separately under those - permissions, but the entire Program remains governed by this License - without regard to the additional permissions. When you convey a copy of - a covered work, you may at your option remove any additional permissions - from that copy, or from any part of it. (Additional permissions may be - written to require their own removal in certain cases when you modify the - work.) You may place additional permissions on material, added by you to - a covered work, for which you have or can give appropriate copyright - permission. - - Notwithstanding any other provision of this License, for material you add - to a covered work, you may (if authorized by the copyright holders of - that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some trade - names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that material - by anyone who conveys the material (or modified versions of it) with - contractual assumptions of liability to the recipient, for any - liability that these contractual assumptions directly impose on those - licensors and authors. - - All other non-permissive additional terms are considered “further - restrictions” within the meaning of section 10. If the Program as you - received it, or any part of it, contains a notice stating that it is - governed by this License along with a term that is a further restriction, - you may remove that term. If a license document contains a further - restriction but permits relicensing or conveying under this License, you - may add to a covered work material governed by the terms of that license - document, provided that the further restriction does not survive such - relicensing or conveying. - - If you add terms to a covered work in accord with this section, you must - place, in the relevant source files, a statement of the additional terms - that apply to those files, or a notice indicating where to find the - applicable terms. Additional terms, permissive or non-permissive, may be - stated in the form of a separately written license, or stated as - exceptions; the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly - provided under this License. Any attempt otherwise to propagate or modify - it is void, and will automatically terminate your rights under this - License (including any patent licenses granted under the third paragraph - of section 11). - - However, if you cease all violation of this License, then your license - from a particular copyright holder is reinstated (a) provisionally, - unless and until the copyright holder explicitly and finally terminates - your license, and (b) permanently, if the copyright holder fails to - notify you of the violation by some reasonable means prior to 60 days - after the cessation. - - Moreover, your license from a particular copyright holder is reinstated - permanently if the copyright holder notifies you of the violation by some - reasonable means, this is the first time you have received notice of - violation of this License (for any work) from that copyright holder, and - you cure the violation prior to 30 days after your receipt of the notice. - - Termination of your rights under this section does not terminate the - licenses of parties who have received copies or rights from you under - this License. If your rights have been terminated and not permanently - reinstated, you do not qualify to receive new licenses for the same - material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or run a - copy of the Program. Ancillary propagation of a covered work occurring - solely as a consequence of using peer-to-peer transmission to receive a - copy likewise does not require acceptance. However, nothing other than - this License grants you permission to propagate or modify any covered - work. These actions infringe copyright if you do not accept this License. - Therefore, by modifying or propagating a covered work, you indicate your - acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically receives - a license from the original licensors, to run, modify and propagate that - work, subject to this License. You are not responsible for enforcing - compliance by third parties with this License. - - An “entity transaction” is a transaction transferring control of an - organization, or substantially all assets of one, or subdividing an - organization, or merging organizations. If propagation of a covered work - results from an entity transaction, each party to that transaction who - receives a copy of the work also receives whatever licenses to the work - the party's predecessor in interest had or could give under the previous - paragraph, plus a right to possession of the Corresponding Source of the - work from the predecessor in interest, if the predecessor has it or can - get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the rights - granted or affirmed under this License. For example, you may not impose a - license fee, royalty, or other charge for exercise of rights granted - under this License, and you may not initiate litigation (including a - cross-claim or counterclaim in a lawsuit) alleging that any patent claim - is infringed by making, using, selling, offering for sale, or importing - the Program or any portion of it. - - 11. Patents. - - A “contributor” is a copyright holder who authorizes use under this - License of the Program or a work on which the Program is based. The work - thus licensed is called the contributor's “contributor version”. - - A contributor's “essential patent claims” are all patent claims owned or - controlled by the contributor, whether already acquired or hereafter - acquired, that would be infringed by some manner, permitted by this - License, of making, using, or selling its contributor version, but do not - include claims that would be infringed only as a consequence of further - modification of the contributor version. For purposes of this definition, - “control” includes the right to grant patent sublicenses in a manner - consistent with the requirements of this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free - patent license under the contributor's essential patent claims, to make, - use, sell, offer for sale, import and otherwise run, modify and propagate - the contents of its contributor version. - - In the following three paragraphs, a “patent license” is any express - agreement or commitment, however denominated, not to enforce a patent - (such as an express permission to practice a patent or covenant not to - sue for patent infringement). To “grant” such a patent license to a party - means to make such an agreement or commitment not to enforce a patent - against the party. - - If you convey a covered work, knowingly relying on a patent license, and - the Corresponding Source of the work is not available for anyone to copy, - free of charge and under the terms of this License, through a publicly - available network server or other readily accessible means, then you must - either (1) cause the Corresponding Source to be so available, or (2) - arrange to deprive yourself of the benefit of the patent license for this - particular work, or (3) arrange, in a manner consistent with the - requirements of this License, to extend the patent license to downstream - recipients. “Knowingly relying” means you have actual knowledge that, but - for the patent license, your conveying the covered work in a country, or - your recipient's use of the covered work in a country, would infringe - one or more identifiable patents in that country that you have reason - to believe are valid. - - If, pursuant to or in connection with a single transaction or - arrangement, you convey, or propagate by procuring conveyance of, a - covered work, and grant a patent license to some of the parties receiving - the covered work authorizing them to use, propagate, modify or convey a - specific copy of the covered work, then the patent license you grant is - automatically extended to all recipients of the covered work and works - based on it. - - A patent license is “discriminatory” if it does not include within the - scope of its coverage, prohibits the exercise of, or is conditioned on - the non-exercise of one or more of the rights that are specifically - granted under this License. You may not convey a covered work if you are - a party to an arrangement with a third party that is in the business of - distributing software, under which you make payment to the third party - based on the extent of your activity of conveying the work, and under - which the third party grants, to any of the parties who would receive the - covered work from you, a discriminatory patent license (a) in connection - with copies of the covered work conveyed by you (or copies made from - those copies), or (b) primarily for and in connection with specific - products or compilations that contain the covered work, unless you - entered into that arrangement, or that patent license was granted, prior - to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting any - implied license or other defenses to infringement that may otherwise be - available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or - otherwise) that contradict the conditions of this License, they do not - excuse you from the conditions of this License. If you cannot use, - propagate or convey a covered work so as to satisfy simultaneously your - obligations under this License and any other pertinent obligations, then - as a consequence you may not use, propagate or convey it at all. For - example, if you agree to terms that obligate you to collect a royalty for - further conveying from those to whom you convey the Program, the only way - you could satisfy both those terms and this License would be to refrain - entirely from conveying the Program. - - 13. Offering the Program as a Service. - - If you make the functionality of the Program or a modified version - available to third parties as a service, you must make the Service Source - Code available via network download to everyone at no charge, under the - terms of this License. Making the functionality of the Program or - modified version available to third parties as a service includes, - without limitation, enabling third parties to interact with the - functionality of the Program or modified version remotely through a - computer network, offering a service the value of which entirely or - primarily derives from the value of the Program or modified version, or - offering a service that accomplishes for users the primary purpose of the - Program or modified version. - - “Service Source Code” means the Corresponding Source for the Program or - the modified version, and the Corresponding Source for all programs that - you use to make the Program or modified version available as a service, - including, without limitation, management software, user interfaces, - application program interfaces, automation software, monitoring software, - backup software, storage software and hosting software, all such that a - user could run an instance of the service using the Service Source Code - you make available. - - 14. Revised Versions of this License. - - MongoDB, Inc. may publish revised and/or new versions of the Server Side - Public License from time to time. Such new versions will be similar in - spirit to the present version, but may differ in detail to address new - problems or concerns. - - Each version is given a distinguishing version number. If the Program - specifies that a certain numbered version of the Server Side Public - License “or any later version” applies to it, you have the option of - following the terms and conditions either of that numbered version or of - any later version published by MongoDB, Inc. If the Program does not - specify a version number of the Server Side Public License, you may - choose any version ever published by MongoDB, Inc. - - If the Program specifies that a proxy can decide which future versions of - the Server Side Public License can be used, that proxy's public statement - of acceptance of a version permanently authorizes you to choose that - version for the Program. - - Later license versions may give you additional or different permissions. - However, no additional obligations are imposed on any author or copyright - holder as a result of your choosing to follow a later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY - APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT - HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY - OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, - THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM - IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF - ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING - WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS - THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING - ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF - THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO - LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU - OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER - PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE - POSSIBILITY OF SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided above - cannot be given local legal effect according to their terms, reviewing - courts shall apply local law that most closely approximates an absolute - waiver of all civil liability in connection with the Program, unless a - warranty or assumption of liability accompanies a copy of the Program in - return for a fee. - - END OF TERMS AND CONDITIONS - - - - - -Elastic License 2.0 - -URL: https://www.elastic.co/licensing/elastic-license - -## Acceptance - -By using the software, you agree to all of the terms and conditions below. - -## Copyright License - -The licensor grants you a non-exclusive, royalty-free, worldwide, -non-sublicensable, non-transferable license to use, copy, distribute, make -available, and prepare derivative works of the software, in each case subject to -the limitations and conditions below. - -## Limitations - -You may not provide the software to third parties as a hosted or managed -service, where the service provides users with access to any substantial set of -the features or functionality of the software. - -You may not move, change, disable, or circumvent the license key functionality -in the software, and you may not remove or obscure any functionality in the -software that is protected by the license key. - -You may not alter, remove, or obscure any licensing, copyright, or other notices -of the licensor in the software. Any use of the licensor’s trademarks is subject -to applicable law. - -## Patents - -The licensor grants you a license, under any patent claims the licensor can -license, or becomes able to license, to make, have made, use, sell, offer for -sale, import and have imported the software, in each case subject to the -limitations and conditions in this license. This license does not cover any -patent claims that you cause to be infringed by modifications or additions to -the software. If you or your company make any written claim that the software -infringes or contributes to infringement of any patent, your patent license for -the software granted under these terms ends immediately. If your company makes -such a claim, your patent license ends immediately for work on behalf of your -company. - -## Notices - -You must ensure that anyone who gets a copy of any part of the software from you -also gets a copy of these terms. - -If you modify the software, you must include in any modified copies of the -software prominent notices stating that you have modified the software. - -## No Other Rights - -These terms do not imply any licenses other than those expressly granted in -these terms. - -## Termination - -If you use the software in violation of these terms, such use is not licensed, -and your licenses will automatically terminate. If the licensor provides you -with a notice of your violation, and you cease all violation of this license no -later than 30 days after you receive that notice, your licenses will be -reinstated retroactively. However, if you violate these terms after such -reinstatement, any additional violation of these terms will cause your licenses -to terminate automatically and permanently. - -## No Liability - -*As far as the law allows, the software comes as is, without any warranty or -condition, and the licensor will not be liable to you for any damages arising -out of these terms or the use or nature of the software, under any kind of -legal claim.* - -## Definitions - -The **licensor** is the entity offering these terms, and the **software** is the -software the licensor makes available under these terms, including any portion -of it. - -**you** refers to the individual or entity agreeing to these terms. - -**your company** is any legal entity, sole proprietorship, or other kind of -organization that you work for, plus all organizations that have control over, -are under the control of, or are under common control with that -organization. **control** means ownership of substantially all the assets of an -entity, or the power to direct its management and policies by vote, contract, or -otherwise. Control can be direct or indirect. - -**your licenses** are all the licenses granted to you for the software under -these terms. - -**use** means anything you do with the software requiring one of your licenses. - -**trademark** means trademarks, service marks, and similar rights. diff --git a/modules/aggregations/build.gradle b/modules/aggregations/build.gradle index 91f3303d9d4a8..9faedd67974a9 100644 --- a/modules/aggregations/build.gradle +++ b/modules/aggregations/build.gradle @@ -2,24 +2,25 @@ import org.elasticsearch.gradle.internal.info.BuildParams /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.internal-yaml-rest-test' apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { - description 'Adds "built in" aggregations to Elasticsearch.' - classname 'org.elasticsearch.aggregations.AggregationsPlugin' + description = 'Adds "built in" aggregations to Elasticsearch.' + classname ='org.elasticsearch.aggregations.AggregationsPlugin' extendedPlugins = ['lang-painless'] } restResources { restApi { - include '_common', 'indices', 'cluster', 'index', 'search', 'nodes', 'bulk', 'scripts_painless_execute', 'put_script' + include 'capabilities', '_common', 'indices', 'cluster', 'index', 'search', 'nodes', 'bulk', 'scripts_painless_execute', 'put_script' } restTests { // Pulls in all aggregation tests from core AND the forwards v7's core for forwards compatibility @@ -27,7 +28,7 @@ restResources { } } -if (BuildParams.isSnapshotBuild() == false) { +if (buildParams.isSnapshotBuild() == false) { tasks.named("test").configure { systemProperty 'es.index_mode_feature_flag_registered', 'true' } @@ -56,6 +57,8 @@ tasks.named("yamlRestTestV7CompatTransform").configure { task -> // Something has changed with response codes task.skipTest("search.aggregation/20_terms/IP test", "Hybrid t-digest produces different results.") + // Maths changed + task.skipTest("aggregations/moving_fn/linearWeightedAvg", "math was wrong in previous versions") task.addAllowedWarningRegex("\\[types removal\\].*") } diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/AggregationIntegTestCase.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/AggregationIntegTestCase.java index 302f4c6b4a94e..ffa3ca684a842 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/AggregationIntegTestCase.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/AggregationIntegTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations; diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/AdjacencyMatrixIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/AdjacencyMatrixIT.java index 44e708e00d4d5..cf43080da83db 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/AdjacencyMatrixIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/AdjacencyMatrixIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket; diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/SearchCancellationIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/SearchCancellationIT.java index a6e530a9d66cf..7adf6a09e9a19 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/SearchCancellationIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/SearchCancellationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket; @@ -95,6 +96,8 @@ public void testCancellationDuringTimeSeriesAggregation() throws Exception { } logger.info("Executing search"); + // we have to explicitly set error_trace=true for the later exception check for `TimeSeriesIndexSearcher` + client().threadPool().getThreadContext().putHeader("error_trace", "true"); TimeSeriesAggregationBuilder timeSeriesAggregationBuilder = new TimeSeriesAggregationBuilder("test_agg"); ActionFuture searchResponse = prepareSearch("test").setQuery(matchAllQuery()) .addAggregation( diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java index 3ff2e618f5759..fd7f71176ad39 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket; diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java index 63f58bbb75713..cdf4b131ce749 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket; diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesNestedAggregationsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesNestedAggregationsIT.java index 2967e6f5e322f..aca7514fc70c4 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesNestedAggregationsIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesNestedAggregationsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket; diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java index 97c75689fe5dc..9380610d730bf 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket; @@ -98,17 +99,12 @@ public void setUp() throws Exception { .setSettings( settings.put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersions.NEW_INDEXVERSION_FORMAT).build() ) - .setMapping(mapping) - .get() - ); - - assertAcked( + .setMapping(mapping), indicesAdmin().prepareCreate(afterIndex) .setSettings( settings.put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersions.TIME_SERIES_ID_HASHING).build() ) .setMapping(mapping) - .get() ); final TimeSeriesDataset timeSeriesDataset = new TimeSeriesDataset(); diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/DateDerivativeIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/DateDerivativeIT.java index c306e0fbcba06..e911bf1a41198 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/DateDerivativeIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/DateDerivativeIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.pipeline; @@ -64,17 +65,17 @@ protected Collection> nodePlugins() { } private static IndexRequestBuilder indexDoc(String idx, ZonedDateTime date, int value) throws Exception { - return prepareIndex(idx).setSource(jsonBuilder().startObject().timeField("date", date).field("value", value).endObject()); + return prepareIndex(idx).setSource(jsonBuilder().startObject().timestampField("date", date).field("value", value).endObject()); } private IndexRequestBuilder indexDoc(int month, int day, int value) throws Exception { return prepareIndex("idx").setSource( jsonBuilder().startObject() .field("value", value) - .timeField("date", date(month, day)) + .timestampField("date", date(month, day)) .startArray("dates") - .timeValue(date(month, day)) - .timeValue(date(month + 1, day + 1)) + .timestampValue(date(month, day)) + .timestampValue(date(month + 1, day + 1)) .endArray() .endObject() ); diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/SerialDiffIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/SerialDiffIT.java index 430f71879cb78..b635ba9b386fe 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/SerialDiffIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/pipeline/SerialDiffIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.pipeline; diff --git a/modules/aggregations/src/main/java/module-info.java b/modules/aggregations/src/main/java/module-info.java index c9abc97eae8dc..1fc2f74480160 100644 --- a/modules/aggregations/src/main/java/module-info.java +++ b/modules/aggregations/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.aggs { diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/AggregationsPainlessExtension.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/AggregationsPainlessExtension.java index e181bb0018015..927966be7bb11 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/AggregationsPainlessExtension.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/AggregationsPainlessExtension.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/AggregationsPlugin.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/AggregationsPlugin.java index 86f9c0c5382c5..41ed325e0e947 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/AggregationsPlugin.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/AggregationsPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/adjacency/AdjacencyMatrix.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/adjacency/AdjacencyMatrix.java index d08ac4c28139d..292fdeba59965 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/adjacency/AdjacencyMatrix.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/adjacency/AdjacencyMatrix.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket.adjacency; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilder.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilder.java index dcc51c94a0379..a53671e4ca668 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilder.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket.adjacency; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/adjacency/AdjacencyMatrixAggregator.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/adjacency/AdjacencyMatrixAggregator.java index dfe0a0642ccc3..203105edc5a24 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/adjacency/AdjacencyMatrixAggregator.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/adjacency/AdjacencyMatrixAggregator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket.adjacency; @@ -14,6 +15,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.util.LongArray; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.aggregations.AggregationExecutionContext; import org.elasticsearch.search.aggregations.Aggregator; @@ -176,65 +178,66 @@ public void collect(int doc, long bucket) throws IOException { } @Override - public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws IOException { + public InternalAggregation[] buildAggregations(LongArray owningBucketOrds) throws IOException { // Buckets are ordered into groups - [keyed filters] [key1&key2 intersects] - int maxOrd = owningBucketOrds.length * totalNumKeys; - int totalBucketsToBuild = 0; - for (int ord = 0; ord < maxOrd; ord++) { + long maxOrd = owningBucketOrds.size() * totalNumKeys; + long totalBucketsToBuild = 0; + for (long ord = 0; ord < maxOrd; ord++) { if (bucketDocCount(ord) > 0) { totalBucketsToBuild++; } } - long[] bucketOrdsToBuild = new long[totalBucketsToBuild]; - int builtBucketIndex = 0; - for (int ord = 0; ord < maxOrd; ord++) { - if (bucketDocCount(ord) > 0) { - bucketOrdsToBuild[builtBucketIndex++] = ord; - } - } - assert builtBucketIndex == totalBucketsToBuild; - builtBucketIndex = 0; - var bucketSubAggs = buildSubAggsForBuckets(bucketOrdsToBuild); - InternalAggregation[] results = new InternalAggregation[owningBucketOrds.length]; - for (int owningBucketOrdIdx = 0; owningBucketOrdIdx < owningBucketOrds.length; owningBucketOrdIdx++) { - List buckets = new ArrayList<>(filters.length); - for (int i = 0; i < keys.length; i++) { - long bucketOrd = bucketOrd(owningBucketOrds[owningBucketOrdIdx], i); - long docCount = bucketDocCount(bucketOrd); - // Empty buckets are not returned because this aggregation will commonly be used under a - // a date-histogram where we will look for transactions over time and can expect many - // empty buckets. - if (docCount > 0) { - InternalAdjacencyMatrix.InternalBucket bucket = new InternalAdjacencyMatrix.InternalBucket( - keys[i], - docCount, - bucketSubAggs.apply(builtBucketIndex++) - ); - buckets.add(bucket); + try (LongArray bucketOrdsToBuild = bigArrays().newLongArray(totalBucketsToBuild)) { + int[] builtBucketIndex = new int[] { 0 }; + for (int ord = 0; ord < maxOrd; ord++) { + if (bucketDocCount(ord) > 0) { + bucketOrdsToBuild.set(builtBucketIndex[0]++, ord); } } - int pos = keys.length; - for (int i = 0; i < keys.length; i++) { - for (int j = i + 1; j < keys.length; j++) { - long bucketOrd = bucketOrd(owningBucketOrds[owningBucketOrdIdx], pos); + assert builtBucketIndex[0] == totalBucketsToBuild; + builtBucketIndex[0] = 0; + var bucketSubAggs = buildSubAggsForBuckets(bucketOrdsToBuild); + InternalAggregation[] aggregations = buildAggregations(Math.toIntExact(owningBucketOrds.size()), owningBucketOrdIdx -> { + List buckets = new ArrayList<>(filters.length); + for (int i = 0; i < keys.length; i++) { + long bucketOrd = bucketOrd(owningBucketOrds.get(owningBucketOrdIdx), i); long docCount = bucketDocCount(bucketOrd); - // Empty buckets are not returned due to potential for very sparse matrices + // Empty buckets are not returned because this aggregation will commonly be used under a + // a date-histogram where we will look for transactions over time and can expect many + // empty buckets. if (docCount > 0) { - String intersectKey = keys[i] + separator + keys[j]; + checkRealMemoryCBForInternalBucket(); InternalAdjacencyMatrix.InternalBucket bucket = new InternalAdjacencyMatrix.InternalBucket( - intersectKey, + keys[i], docCount, - bucketSubAggs.apply(builtBucketIndex++) + bucketSubAggs.apply(builtBucketIndex[0]++) ); buckets.add(bucket); } - pos++; } - } - results[owningBucketOrdIdx] = new InternalAdjacencyMatrix(name, buckets, metadata()); + int pos = keys.length; + for (int i = 0; i < keys.length; i++) { + for (int j = i + 1; j < keys.length; j++) { + long bucketOrd = bucketOrd(owningBucketOrds.get(owningBucketOrdIdx), pos); + long docCount = bucketDocCount(bucketOrd); + // Empty buckets are not returned due to potential for very sparse matrices + if (docCount > 0) { + String intersectKey = keys[i] + separator + keys[j]; + InternalAdjacencyMatrix.InternalBucket bucket = new InternalAdjacencyMatrix.InternalBucket( + intersectKey, + docCount, + bucketSubAggs.apply(builtBucketIndex[0]++) + ); + buckets.add(bucket); + } + pos++; + } + } + return new InternalAdjacencyMatrix(name, buckets, metadata()); + }); + assert builtBucketIndex[0] == totalBucketsToBuild; + return aggregations; } - assert builtBucketIndex == totalBucketsToBuild; - return results; } @Override diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/adjacency/AdjacencyMatrixAggregatorFactory.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/adjacency/AdjacencyMatrixAggregatorFactory.java index 1d7353628e2c5..1d2426144957f 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/adjacency/AdjacencyMatrixAggregatorFactory.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/adjacency/AdjacencyMatrixAggregatorFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket.adjacency; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java index 71e989274dd56..999f790ee8117 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket.adjacency; @@ -32,7 +33,7 @@ public class InternalAdjacencyMatrix extends InternalMultiBucketAggregation implements AdjacencyMatrix { - public static class InternalBucket extends InternalMultiBucketAggregation.InternalBucket implements AdjacencyMatrix.Bucket { + public static class InternalBucket extends InternalMultiBucketAggregation.InternalBucketWritable implements AdjacencyMatrix.Bucket { private final String key; private final long docCount; @@ -80,14 +81,12 @@ public InternalAggregations getAggregations() { return aggregations; } - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + private void bucketToXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(CommonFields.KEY.getPreferredName(), key); builder.field(CommonFields.DOC_COUNT.getPreferredName(), docCount); aggregations.toXContentInternal(builder, params); builder.endObject(); - return builder; } @Override @@ -236,7 +235,7 @@ public InternalAggregation finalizeSampling(SamplingContext samplingContext) { public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { builder.startArray(CommonFields.BUCKETS.getPreferredName()); for (InternalBucket bucket : buckets) { - bucket.toXContent(builder, params); + bucket.bucketToXContent(builder, params); } builder.endArray(); return builder; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java index c8e5ae2619e3a..342dc14ec2bf3 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket.histogram; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregator.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregator.java index e30ce79178992..6add1b0ac4a13 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregator.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket.histogram; @@ -140,7 +141,7 @@ public final LeafBucketCollector getLeafCollector(AggregationExecutionContext ag protected final InternalAggregation[] buildAggregations( LongKeyedBucketOrds bucketOrds, LongToIntFunction roundingIndexFor, - long[] owningBucketOrds + LongArray owningBucketOrds ) throws IOException { return buildAggregationsForVariableBuckets( owningBucketOrds, @@ -323,7 +324,7 @@ private void increaseRoundingIfNeeded(long rounded) { } @Override - public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws IOException { + public InternalAggregation[] buildAggregations(LongArray owningBucketOrds) throws IOException { return buildAggregations(bucketOrds, l -> roundingIdx, owningBucketOrds); } @@ -593,7 +594,7 @@ private void rebucket() { } @Override - public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws IOException { + public InternalAggregation[] buildAggregations(LongArray owningBucketOrds) throws IOException { /* * Rebucket before building the aggregation to build as small as result * as possible. diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregatorFactory.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregatorFactory.java index be244a2c62da3..7bfcc9a22c040 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregatorFactory.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregatorFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket.histogram; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregatorSupplier.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregatorSupplier.java index 71d7b987ffae2..5defa49ccbdc0 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregatorSupplier.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregatorSupplier.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket.histogram; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogram.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogram.java index b813c9ec50c83..edb7ec4cffce7 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogram.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogram.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket.histogram; @@ -98,8 +99,7 @@ public Object getKey() { return Instant.ofEpochMilli(key).atZone(ZoneOffset.UTC); } - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + private void bucketToXContent(XContentBuilder builder, Params params, DocValueFormat format) throws IOException { String keyAsString = format.format(key).toString(); builder.startObject(); if (format != DocValueFormat.RAW) { @@ -109,7 +109,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(CommonFields.DOC_COUNT.getPreferredName(), docCount); aggregations.toXContentInternal(builder, params); builder.endObject(); - return builder; } @Override @@ -596,7 +595,7 @@ private BucketReduceResult mergeConsecutiveBuckets( public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { builder.startArray(CommonFields.BUCKETS.getPreferredName()); for (Bucket bucket : buckets) { - bucket.toXContent(builder, params); + bucket.bucketToXContent(builder, params, format); } builder.endArray(); builder.field("interval", getInterval().toString()); diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/package-info.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/package-info.java index 1aa416bdd247d..f410d834816f2 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/package-info.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java index c20d3f3ba612a..c4669b1c25224 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket.timeseries; @@ -12,7 +13,7 @@ import org.apache.lucene.util.PriorityQueue; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; +import org.elasticsearch.index.mapper.RoutingPathFields; import org.elasticsearch.search.aggregations.AggregationReduceContext; import org.elasticsearch.search.aggregations.AggregatorReducer; import org.elasticsearch.search.aggregations.InternalAggregation; @@ -33,26 +34,23 @@ public class InternalTimeSeries extends InternalMultiBucketAggregation getKey() { - return TimeSeriesIdFieldMapper.decodeTsidAsMap(key); + return RoutingPathFields.decodeAsMap(key); } @Override @@ -85,8 +83,7 @@ public InternalAggregations getAggregations() { return aggregations; } - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + private void bucketToXContent(XContentBuilder builder, Params params, boolean keyed) throws IOException { // Use map key in the xcontent response: var key = getKey(); if (keyed) { @@ -98,7 +95,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(CommonFields.DOC_COUNT.getPreferredName(), docCount); aggregations.toXContentInternal(builder, params); builder.endObject(); - return builder; } @Override @@ -111,14 +107,13 @@ public boolean equals(Object other) { } InternalTimeSeries.InternalBucket that = (InternalTimeSeries.InternalBucket) other; return Objects.equals(key, that.key) - && Objects.equals(keyed, that.keyed) && Objects.equals(docCount, that.docCount) && Objects.equals(aggregations, that.aggregations); } @Override public int hashCode() { - return Objects.hash(getClass(), key, keyed, docCount, aggregations); + return Objects.hash(getClass(), key, docCount, aggregations); } } @@ -142,7 +137,7 @@ public InternalTimeSeries(StreamInput in) throws IOException { int size = in.readVInt(); List buckets = new ArrayList<>(size); for (int i = 0; i < size; i++) { - buckets.add(new InternalTimeSeries.InternalBucket(in, keyed)); + buckets.add(new InternalTimeSeries.InternalBucket(in)); } this.buckets = buckets; this.bucketMap = null; @@ -161,7 +156,7 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th builder.startArray(CommonFields.BUCKETS.getPreferredName()); } for (InternalBucket bucket : buckets) { - bucket.toXContent(builder, params); + bucket.bucketToXContent(builder, params, keyed); } if (keyed) { builder.endObject(); @@ -251,14 +246,14 @@ public InternalTimeSeries create(List buckets) { @Override public InternalBucket createBucket(InternalAggregations aggregations, InternalBucket prototype) { - return new InternalBucket(prototype.key, prototype.docCount, aggregations, prototype.keyed); + return new InternalBucket(prototype.key, prototype.docCount, aggregations); } private InternalBucket reduceBucket(List buckets, AggregationReduceContext context) { InternalTimeSeries.InternalBucket reduced = null; for (InternalTimeSeries.InternalBucket bucket : buckets) { if (reduced == null) { - reduced = new InternalTimeSeries.InternalBucket(bucket.key, bucket.docCount, bucket.aggregations, bucket.keyed); + reduced = new InternalTimeSeries.InternalBucket(bucket.key, bucket.docCount, bucket.aggregations); } else { reduced.docCount += bucket.docCount; } diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregationBuilder.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregationBuilder.java index 664e975c5b15f..05418ad92ee6a 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregationBuilder.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregationBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket.timeseries; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregationFactory.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregationFactory.java index 72e390ffd1242..d17c27a035c8e 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregationFactory.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregationFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket.timeseries; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java index f238419687cfc..63472bca1d9ac 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java @@ -1,17 +1,21 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket.timeseries; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.util.LongArray; +import org.elasticsearch.common.util.ObjectArray; import org.elasticsearch.core.Releasables; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.index.mapper.RoutingPathFields; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.search.aggregations.AggregationExecutionContext; import org.elasticsearch.search.aggregations.Aggregator; @@ -28,6 +32,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Comparator; import java.util.List; import java.util.Map; @@ -65,42 +70,39 @@ public TimeSeriesAggregator( } @Override - public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws IOException { + public InternalAggregation[] buildAggregations(LongArray owningBucketOrds) throws IOException { BytesRef spare = new BytesRef(); - InternalTimeSeries.InternalBucket[][] allBucketsPerOrd = new InternalTimeSeries.InternalBucket[owningBucketOrds.length][]; - for (int ordIdx = 0; ordIdx < owningBucketOrds.length; ordIdx++) { - BytesKeyedBucketOrds.BucketOrdsEnum ordsEnum = bucketOrds.ordsEnum(owningBucketOrds[ordIdx]); - List buckets = new ArrayList<>(); - while (ordsEnum.next()) { - long docCount = bucketDocCount(ordsEnum.ord()); - ordsEnum.readValue(spare); - InternalTimeSeries.InternalBucket bucket = new InternalTimeSeries.InternalBucket( - BytesRef.deepCopyOf(spare), // Closing bucketOrds will corrupt the bytes ref, so need to make a deep copy here. - docCount, - null, - keyed - ); - bucket.bucketOrd = ordsEnum.ord(); - buckets.add(bucket); - if (buckets.size() >= size) { - break; + try (ObjectArray allBucketsPerOrd = bigArrays().newObjectArray(owningBucketOrds.size())) { + for (long ordIdx = 0; ordIdx < allBucketsPerOrd.size(); ordIdx++) { + BytesKeyedBucketOrds.BucketOrdsEnum ordsEnum = bucketOrds.ordsEnum(owningBucketOrds.get(ordIdx)); + List buckets = new ArrayList<>(); + while (ordsEnum.next()) { + long docCount = bucketDocCount(ordsEnum.ord()); + ordsEnum.readValue(spare); + checkRealMemoryCBForInternalBucket(); + InternalTimeSeries.InternalBucket bucket = new InternalTimeSeries.InternalBucket( + BytesRef.deepCopyOf(spare), // Closing bucketOrds will corrupt the bytes ref, so need to make a deep copy here. + docCount, + null + ); + bucket.bucketOrd = ordsEnum.ord(); + buckets.add(bucket); + if (buckets.size() >= size) { + break; + } } + // NOTE: after introducing _tsid hashing time series are sorted by (_tsid hash, @timestamp) instead of (_tsid, timestamp). + // _tsid hash and _tsid might sort differently, and out of order data might result in incorrect buckets due to _tsid value + // changes not matching _tsid hash changes. Changes in _tsid hash are handled creating a new bucket as a result of making + // the assumption that sorting data results in new buckets whenever there is a change in _tsid hash. This is no true anymore + // because we collect data sorted on (_tsid hash, timestamp) but build aggregation results sorted by (_tsid, timestamp). + buckets.sort(Comparator.comparing(bucket -> bucket.key)); + allBucketsPerOrd.set(ordIdx, buckets.toArray(new InternalTimeSeries.InternalBucket[0])); } - // NOTE: after introducing _tsid hashing time series are sorted by (_tsid hash, @timestamp) instead of (_tsid, timestamp). - // _tsid hash and _tsid might sort differently, and out of order data might result in incorrect buckets due to _tsid value - // changes not matching _tsid hash changes. Changes in _tsid hash are handled creating a new bucket as a result of making - // the assumption that sorting data results in new buckets whenever there is a change in _tsid hash. This is no true anymore - // because we collect data sorted on (_tsid hash, timestamp) but build aggregation results sorted by (_tsid, timestamp). - buckets.sort(Comparator.comparing(bucket -> bucket.key)); - allBucketsPerOrd[ordIdx] = buckets.toArray(new InternalTimeSeries.InternalBucket[0]); - } - buildSubAggsForAllBuckets(allBucketsPerOrd, b -> b.bucketOrd, (b, a) -> b.aggregations = a); + buildSubAggsForAllBuckets(allBucketsPerOrd, b -> b.bucketOrd, (b, a) -> b.aggregations = a); - InternalAggregation[] result = new InternalAggregation[owningBucketOrds.length]; - for (int ordIdx = 0; ordIdx < owningBucketOrds.length; ordIdx++) { - result[ordIdx] = buildResult(allBucketsPerOrd[ordIdx]); + return buildAggregations(Math.toIntExact(allBucketsPerOrd.size()), ordIdx -> buildResult(allBucketsPerOrd.get(ordIdx))); } - return result; } @Override @@ -160,11 +162,11 @@ public void collect(int doc, long bucket) throws IOException { if (currentTsidOrd == aggCtx.getTsidHashOrd()) { tsid = currentTsid; } else { - TimeSeriesIdFieldMapper.TimeSeriesIdBuilder tsidBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); + RoutingPathFields routingPathFields = new RoutingPathFields(null); for (TsidConsumer consumer : dimensionConsumers.values()) { - consumer.accept(doc, tsidBuilder); + consumer.accept(doc, routingPathFields); } - currentTsid = tsid = tsidBuilder.buildLegacyTsid().toBytesRef(); + currentTsid = tsid = TimeSeriesIdFieldMapper.buildLegacyTsid(routingPathFields).toBytesRef(); } long bucketOrdinal = bucketOrds.add(bucket, tsid); if (bucketOrdinal < 0) { // already seen @@ -183,11 +185,11 @@ public void collect(int doc, long bucket) throws IOException { } InternalTimeSeries buildResult(InternalTimeSeries.InternalBucket[] topBuckets) { - return new InternalTimeSeries(name, List.of(topBuckets), keyed, metadata()); + return new InternalTimeSeries(name, Arrays.asList(topBuckets), keyed, metadata()); } @FunctionalInterface interface TsidConsumer { - void accept(int docId, TimeSeriesIdFieldMapper.TimeSeriesIdBuilder tsidBuilder) throws IOException; + void accept(int docId, RoutingPathFields routingFields) throws IOException; } } diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSource.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSource.java index 05c2e928fd84f..85f59c69bc319 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSource.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSource.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.metric; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSourceAggregationBuilder.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSourceAggregationBuilder.java index 4cfd55a240451..1164a0e0caccb 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSourceAggregationBuilder.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSourceAggregationBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.metric; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSourceAggregatorFactory.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSourceAggregatorFactory.java index 26e2110ecbb87..77e2f8da01243 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSourceAggregatorFactory.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSourceAggregatorFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.metric; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSourceParser.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSourceParser.java index 34a8a44326f46..83d5d8ddc4687 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSourceParser.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSourceParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.metric; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/InternalMatrixStats.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/InternalMatrixStats.java index fa409c0660166..30ef224df9be4 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/InternalMatrixStats.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/InternalMatrixStats.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.metric; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixAggregationInspectionHelper.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixAggregationInspectionHelper.java index 6befc7e1c87e6..62ea9d63a0559 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixAggregationInspectionHelper.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixAggregationInspectionHelper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.metric; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregationBuilder.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregationBuilder.java index ee6157f92a82d..37ad08ef2c757 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregationBuilder.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregationBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.metric; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregator.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregator.java index c6ac2818f8246..f79f5e2feb902 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregator.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.metric; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregatorFactory.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregatorFactory.java index 8802e8d53497d..9e3e3a9d49a6e 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregatorFactory.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregatorFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.metric; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsParser.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsParser.java index edf35947f07c2..336a9f35370e9 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsParser.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.metric; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsResults.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsResults.java index 9aa7358563d79..971be12f73398 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsResults.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsResults.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.metric; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/RunningStats.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/RunningStats.java index 9b8ea7321582b..17fe6c444d25d 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/RunningStats.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/RunningStats.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.metric; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/package-info.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/package-info.java index a9ca464ba4b8c..ee43b7c1a0ed0 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/package-info.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java index b53269608e078..f2eff8bf8a896 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.pipeline; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSelectorPipelineAggregator.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSelectorPipelineAggregator.java index 3b8559c77fddc..515ede05e78a9 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSelectorPipelineAggregator.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSelectorPipelineAggregator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.pipeline; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java index 57b60df785673..8a4eaa41d4b8e 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.pipeline; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSortPipelineAggregator.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSortPipelineAggregator.java index 709744450b361..cc8472702cd56 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSortPipelineAggregator.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSortPipelineAggregator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.pipeline; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/Derivative.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/Derivative.java index 253b416d97f04..a2e7bb8287486 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/Derivative.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/Derivative.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.pipeline; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/DerivativePipelineAggregationBuilder.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/DerivativePipelineAggregationBuilder.java index b4ac7387c6955..7f894d21bab1a 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/DerivativePipelineAggregationBuilder.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/DerivativePipelineAggregationBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.pipeline; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/DerivativePipelineAggregator.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/DerivativePipelineAggregator.java index 91aba020b8856..ed92d6dffd9fb 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/DerivativePipelineAggregator.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/DerivativePipelineAggregator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.pipeline; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/MovFnPipelineAggregationBuilder.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/MovFnPipelineAggregationBuilder.java index eadf3839ed479..551559f80b46e 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/MovFnPipelineAggregationBuilder.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/MovFnPipelineAggregationBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.pipeline; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/MovFnPipelineAggregator.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/MovFnPipelineAggregator.java index c9debf89e8162..626d9c675af3f 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/MovFnPipelineAggregator.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/MovFnPipelineAggregator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.pipeline; diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/MovingFunctionScript.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/MovingFunctionScript.java index 2a6dc1107ad67..b75f889c8ba48 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/MovingFunctionScript.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/MovingFunctionScript.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.pipeline; diff --git a/modules/aggregations/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension b/modules/aggregations/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension index 49fffefd631dc..e4588d7ff9576 100644 --- a/modules/aggregations/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension +++ b/modules/aggregations/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # org.elasticsearch.aggregations.AggregationsPainlessExtension diff --git a/modules/aggregations/src/main/resources/org/elasticsearch/aggregations/moving_function_whitelist.txt b/modules/aggregations/src/main/resources/org/elasticsearch/aggregations/moving_function_whitelist.txt index 0575734988063..ac921a23135fb 100644 --- a/modules/aggregations/src/main/resources/org/elasticsearch/aggregations/moving_function_whitelist.txt +++ b/modules/aggregations/src/main/resources/org/elasticsearch/aggregations/moving_function_whitelist.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # This file contains a allowlist for the Moving Function pipeline aggregator diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/AggregationBuilderTestCase.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/AggregationBuilderTestCase.java index b4fafb0c1395c..91b14ab8437b4 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/AggregationBuilderTestCase.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/AggregationBuilderTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/AggregationMultiBucketAggregationTestCase.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/AggregationMultiBucketAggregationTestCase.java index a5fbfeaa4d537..9c3b48d8e51c0 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/AggregationMultiBucketAggregationTestCase.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/AggregationMultiBucketAggregationTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/AggregationTestCase.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/AggregationTestCase.java index 25c604141742e..964ea97c3f5e3 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/AggregationTestCase.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/AggregationTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/adjacency/AdjacencyMatrixAggregatorTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/adjacency/AdjacencyMatrixAggregatorTests.java index e461a731dc19a..9c4109167fa93 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/adjacency/AdjacencyMatrixAggregatorTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/adjacency/AdjacencyMatrixAggregatorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket.adjacency; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/adjacency/AdjacencyMatrixTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/adjacency/AdjacencyMatrixTests.java index 0a255067f1c97..8a5debbe62c6f 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/adjacency/AdjacencyMatrixTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/adjacency/AdjacencyMatrixTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket.adjacency; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/adjacency/InternalAdjacencyMatrixTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/adjacency/InternalAdjacencyMatrixTests.java index e2f544845a60a..b10f5a9f225a8 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/adjacency/InternalAdjacencyMatrixTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/adjacency/InternalAdjacencyMatrixTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket.adjacency; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilderTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilderTests.java index a1da557562f31..a7dcc56baaf30 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilderTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket.histogram; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java index ab4a008d829c6..6908694c9213d 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket.histogram; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramTests.java index 894516d85adb7..e62f8373bf844 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket.histogram; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java index 9b72f30542c54..227557590731e 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java @@ -1,14 +1,14 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket.histogram; -import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.aggregations.bucket.AggregationMultiBucketAggregationTestCase; import org.elasticsearch.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder.RoundingInfo; @@ -27,7 +27,6 @@ import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.test.InternalAggregationTestCase; -import org.elasticsearch.test.TransportVersionUtils; import java.io.IOException; import java.time.Instant; @@ -458,33 +457,6 @@ public void testCreateWithReplacementBuckets() { assertThat(copy.getInterval(), equalTo(orig.getInterval())); } - public void testSerializationPre830() throws IOException { - // we need to test without sub-aggregations, otherwise we need to also update the interval within the inner aggs - InternalAutoDateHistogram instance = createTestInstance( - randomAlphaOfLengthBetween(3, 7), - createTestMetadata(), - InternalAggregations.EMPTY - ); - TransportVersion version = TransportVersionUtils.randomVersionBetween( - random(), - TransportVersions.MINIMUM_COMPATIBLE, - TransportVersionUtils.getPreviousVersion(TransportVersions.V_8_3_0) - ); - InternalAutoDateHistogram deserialized = copyInstance(instance, version); - assertEquals(1, deserialized.getBucketInnerInterval()); - - InternalAutoDateHistogram modified = new InternalAutoDateHistogram( - deserialized.getName(), - deserialized.getBuckets(), - deserialized.getTargetBuckets(), - deserialized.getBucketInfo(), - deserialized.getFormatter(), - deserialized.getMetadata(), - instance.getBucketInnerInterval() - ); - assertEqualInstances(instance, modified); - } - public void testReadFromPre830() throws IOException { byte[] bytes = Base64.getDecoder() .decode( diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java index a77ea04a2d8a1..3b67d09c0d6a1 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket.timeseries; @@ -14,6 +15,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.MockBigArrays; import org.elasticsearch.common.util.MockPageCacheRecycler; +import org.elasticsearch.index.mapper.RoutingPathFields; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.search.aggregations.AggregationReduceContext; @@ -41,13 +43,13 @@ private List randomBuckets(boolean keyed, InternalAggregations a List> keys = randomKeys(bucketKeys(randomIntBetween(1, 4)), numberOfBuckets); for (int j = 0; j < numberOfBuckets; j++) { long docCount = randomLongBetween(0, Long.MAX_VALUE / (20L * numberOfBuckets)); - var builder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); + var routingPathFields = new RoutingPathFields(null); for (var entry : keys.get(j).entrySet()) { - builder.addString(entry.getKey(), (String) entry.getValue()); + routingPathFields.addString(entry.getKey(), (String) entry.getValue()); } try { - var key = builder.buildLegacyTsid().toBytesRef(); - bucketList.add(new InternalBucket(key, docCount, aggregations, keyed)); + var key = TimeSeriesIdFieldMapper.buildLegacyTsid(routingPathFields).toBytesRef(); + bucketList.add(new InternalBucket(key, docCount, aggregations)); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -106,10 +108,10 @@ public void testReduceSimple() { InternalTimeSeries first = new InternalTimeSeries( "ts", List.of( - new InternalBucket(new BytesRef("1"), 3, InternalAggregations.EMPTY, false), - new InternalBucket(new BytesRef("10"), 6, InternalAggregations.EMPTY, false), - new InternalBucket(new BytesRef("2"), 2, InternalAggregations.EMPTY, false), - new InternalBucket(new BytesRef("9"), 5, InternalAggregations.EMPTY, false) + new InternalBucket(new BytesRef("1"), 3, InternalAggregations.EMPTY), + new InternalBucket(new BytesRef("10"), 6, InternalAggregations.EMPTY), + new InternalBucket(new BytesRef("2"), 2, InternalAggregations.EMPTY), + new InternalBucket(new BytesRef("9"), 5, InternalAggregations.EMPTY) ), false, Map.of() @@ -117,8 +119,8 @@ public void testReduceSimple() { InternalTimeSeries second = new InternalTimeSeries( "ts", List.of( - new InternalBucket(new BytesRef("2"), 1, InternalAggregations.EMPTY, false), - new InternalBucket(new BytesRef("3"), 3, InternalAggregations.EMPTY, false) + new InternalBucket(new BytesRef("2"), 1, InternalAggregations.EMPTY), + new InternalBucket(new BytesRef("3"), 3, InternalAggregations.EMPTY) ), false, Map.of() @@ -126,9 +128,9 @@ public void testReduceSimple() { InternalTimeSeries third = new InternalTimeSeries( "ts", List.of( - new InternalBucket(new BytesRef("1"), 2, InternalAggregations.EMPTY, false), - new InternalBucket(new BytesRef("3"), 4, InternalAggregations.EMPTY, false), - new InternalBucket(new BytesRef("9"), 4, InternalAggregations.EMPTY, false) + new InternalBucket(new BytesRef("1"), 2, InternalAggregations.EMPTY), + new InternalBucket(new BytesRef("3"), 4, InternalAggregations.EMPTY), + new InternalBucket(new BytesRef("9"), 4, InternalAggregations.EMPTY) ), false, Map.of() diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregationBuilderTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregationBuilderTests.java index eb738dc3fe828..2c46dd66b3d12 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregationBuilderTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregationBuilderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket.timeseries; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java index 54d1c14931d92..493b4bdc81860 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.bucket.timeseries; @@ -29,8 +30,8 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.mapper.RoutingPathFields; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; -import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper.TimeSeriesIdBuilder; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; @@ -92,10 +93,10 @@ public static void writeTS(RandomIndexWriter iw, long timestamp, Object[] dimens final List fields = new ArrayList<>(); fields.add(new SortedNumericDocValuesField(DataStreamTimestampFieldMapper.DEFAULT_PATH, timestamp)); fields.add(new LongPoint(DataStreamTimestampFieldMapper.DEFAULT_PATH, timestamp)); - final TimeSeriesIdBuilder builder = new TimeSeriesIdBuilder(null); + RoutingPathFields routingPathFields = new RoutingPathFields(null); for (int i = 0; i < dimensions.length; i += 2) { if (dimensions[i + 1] instanceof Number n) { - builder.addLong(dimensions[i].toString(), n.longValue()); + routingPathFields.addLong(dimensions[i].toString(), n.longValue()); if (dimensions[i + 1] instanceof Integer || dimensions[i + 1] instanceof Long) { fields.add(new NumericDocValuesField(dimensions[i].toString(), ((Number) dimensions[i + 1]).longValue())); } else if (dimensions[i + 1] instanceof Float) { @@ -104,7 +105,7 @@ public static void writeTS(RandomIndexWriter iw, long timestamp, Object[] dimens fields.add(new DoubleDocValuesField(dimensions[i].toString(), (double) dimensions[i + 1])); } } else { - builder.addString(dimensions[i].toString(), dimensions[i + 1].toString()); + routingPathFields.addString(dimensions[i].toString(), dimensions[i + 1].toString()); fields.add(new SortedSetDocValuesField(dimensions[i].toString(), new BytesRef(dimensions[i + 1].toString()))); } } @@ -117,7 +118,9 @@ public static void writeTS(RandomIndexWriter iw, long timestamp, Object[] dimens fields.add(new DoubleDocValuesField(metrics[i].toString(), (double) metrics[i + 1])); } } - fields.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, builder.buildLegacyTsid().toBytesRef())); + fields.add( + new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, TimeSeriesIdFieldMapper.buildLegacyTsid(routingPathFields).toBytesRef()) + ); iw.addDocument(fields); } @@ -173,19 +176,19 @@ public void testMultiBucketAggregationAsSubAggregation() throws IOException { InternalDateHistogram byTimeStampBucket = ts.getBucketByKey("{dim1=aaa, dim2=xxx}").getAggregations().get("by_timestamp"); assertThat( byTimeStampBucket.getBuckets(), - contains(new InternalDateHistogram.Bucket(startTime, 2, false, null, InternalAggregations.EMPTY)) + contains(new InternalDateHistogram.Bucket(startTime, 2, null, InternalAggregations.EMPTY)) ); assertThat(ts.getBucketByKey("{dim1=aaa, dim2=yyy}").docCount, equalTo(2L)); byTimeStampBucket = ts.getBucketByKey("{dim1=aaa, dim2=yyy}").getAggregations().get("by_timestamp"); assertThat( byTimeStampBucket.getBuckets(), - contains(new InternalDateHistogram.Bucket(startTime, 2, false, null, InternalAggregations.EMPTY)) + contains(new InternalDateHistogram.Bucket(startTime, 2, null, InternalAggregations.EMPTY)) ); assertThat(ts.getBucketByKey("{dim1=bbb, dim2=zzz}").docCount, equalTo(4L)); byTimeStampBucket = ts.getBucketByKey("{dim1=bbb, dim2=zzz}").getAggregations().get("by_timestamp"); assertThat( byTimeStampBucket.getBuckets(), - contains(new InternalDateHistogram.Bucket(startTime, 4, false, null, InternalAggregations.EMPTY)) + contains(new InternalDateHistogram.Bucket(startTime, 4, null, InternalAggregations.EMPTY)) ); }; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/BaseMatrixStatsTestCase.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/BaseMatrixStatsTestCase.java index 9cbebb4dab1eb..5587c4ae7b5d0 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/BaseMatrixStatsTestCase.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/BaseMatrixStatsTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.metric; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/InternalMatrixStatsTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/InternalMatrixStatsTests.java index 82bee40e13ca1..bd8c294d8cd55 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/InternalMatrixStatsTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/InternalMatrixStatsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.metric; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregatorTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregatorTests.java index 1de3c513bef9c..74c1f3c16278f 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregatorTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregatorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.metric; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/MultiPassStats.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/MultiPassStats.java index 7275cd26cae65..2bff81a2d0aab 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/MultiPassStats.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/MultiPassStats.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.metric; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/RunningStatsTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/RunningStatsTests.java index 76eda2a8dca19..85e48d292bb9c 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/RunningStatsTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/RunningStatsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.metric; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/BucketSelectorAggregationBuilderTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/BucketSelectorAggregationBuilderTests.java index 59bdd52193bd6..06ce185c7143c 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/BucketSelectorAggregationBuilderTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/BucketSelectorAggregationBuilderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.pipeline; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/BucketSortPipelineAggregationBuilderTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/BucketSortPipelineAggregationBuilderTests.java index a025f03d0eafc..9716df6720783 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/BucketSortPipelineAggregationBuilderTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/BucketSortPipelineAggregationBuilderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.pipeline; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/CumulativeSumAggregatorTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/CumulativeSumAggregatorTests.java index ce9a47796fc03..f2a36d192bd0c 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/CumulativeSumAggregatorTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/CumulativeSumAggregatorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.pipeline; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/CumulativeSumTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/CumulativeSumTests.java index 7cab1eb5cfd0e..b9edda03fb1b0 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/CumulativeSumTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/CumulativeSumTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.pipeline; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/DerivativeAggregatorTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/DerivativeAggregatorTests.java index 68245d31c14b6..2cb19a90955c4 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/DerivativeAggregatorTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/DerivativeAggregatorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.pipeline; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/DerivativePipelinesAggregationBuilderTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/DerivativePipelinesAggregationBuilderTests.java index 81c65ed1fceac..50804de1ad688 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/DerivativePipelinesAggregationBuilderTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/DerivativePipelinesAggregationBuilderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.pipeline; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/DerivativeResultTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/DerivativeResultTests.java index bbbf302b08147..825d93e8af71c 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/DerivativeResultTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/DerivativeResultTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.pipeline; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/MovFnAggregatorTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/MovFnAggregatorTests.java index 8fec4d4d460a2..1d85d80558d7f 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/MovFnAggregatorTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/MovFnAggregatorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.pipeline; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/MovFnPipelineAggregationBuilderSerializationTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/MovFnPipelineAggregationBuilderSerializationTests.java index 88c8a54232310..2448142f0fcec 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/MovFnPipelineAggregationBuilderSerializationTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/MovFnPipelineAggregationBuilderSerializationTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.pipeline; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/PipelineAggregationHelperTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/PipelineAggregationHelperTests.java index 94ad4e4bc657a..626a5c10979af 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/PipelineAggregationHelperTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/PipelineAggregationHelperTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.pipeline; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/SerialDifferenceTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/SerialDifferenceTests.java index 28378b8cd8708..f06fdad1385e8 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/SerialDifferenceTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/pipeline/SerialDifferenceTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations.pipeline; diff --git a/modules/aggregations/src/yamlRestTest/java/org/elasticsearch/aggregations/AggregationsClientYamlTestSuiteIT.java b/modules/aggregations/src/yamlRestTest/java/org/elasticsearch/aggregations/AggregationsClientYamlTestSuiteIT.java index 33cb223569b9b..c4f44bc3e4b3e 100644 --- a/modules/aggregations/src/yamlRestTest/java/org/elasticsearch/aggregations/AggregationsClientYamlTestSuiteIT.java +++ b/modules/aggregations/src/yamlRestTest/java/org/elasticsearch/aggregations/AggregationsClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aggregations; diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/date_agg_per_day_of_week.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/date_agg_per_day_of_week.yml index dc349171e3a27..5157d563540b1 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/date_agg_per_day_of_week.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/date_agg_per_day_of_week.yml @@ -1,8 +1,8 @@ --- setup: - requires: - cluster_features: ["gte_v7.7.0"] - reason: "Start of the week Monday was enabled in a backport to 7.7 PR#50916" + cluster_features: ["gte_v8.16.0"] + reason: "Start of the week Sunday was changed in 8.16 as part of the locale changes" - do: indices.create: @@ -25,7 +25,7 @@ setup: --- # The inserted document has a field date=2009-11-15T14:12:12 which is Sunday. -# When aggregating per day of the week this should be considered as last day of the week (7) +# When aggregating per day of the week this should be considered as first day of the week (1) # and this value should be used in 'key_as_string' "Date aggregartion per day of week": - do: @@ -44,4 +44,4 @@ setup: - match: {hits.total: 1} - length: { aggregations.test.buckets: 1 } - - match: { aggregations.test.buckets.0.key_as_string: "7" } + - match: { aggregations.test.buckets.0.key_as_string: "1" } diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/moving_fn.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/moving_fn.yml index cd6feb601b1df..3abad87d57907 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/moving_fn.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/moving_fn.yml @@ -255,6 +255,17 @@ linearWeightedAvg: - skip: features: close_to + - requires: + test_runner_features: [capabilities] + + - requires: + capabilities: + - method: POST + path: /_search + parameters: [method, path, parameters, capabilities] + capabilities: [moving_fn_right_math] + reason: "math not fixed yet" + - do: search: index: no_gaps @@ -275,11 +286,11 @@ linearWeightedAvg: - match: { hits.total.value: 6 } - length: { aggregations.@timestamp.buckets: 6 } - is_false: aggregations.@timestamp.buckets.0.d.value - - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 0.500, error: 0.0005 } } - - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 1.250, error: 0.0005 } } - - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 1.000, error: 0.0005 } } - - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 2.250, error: 0.0005 } } - - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 3.500, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 1.667, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 1.333, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 3.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 4.667, error: 0.0005 } } - do: search: @@ -301,11 +312,11 @@ linearWeightedAvg: - match: { hits.total.value: 6 } - length: { aggregations.@timestamp.buckets: 6 } - is_false: aggregations.@timestamp.buckets.0.d.value - - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 0.500, error: 0.0005 } } - - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 1.250, error: 0.0005 } } - - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 1.143, error: 0.0005 } } - - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 2.286, error: 0.0005 } } - - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 3.429, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 1.667, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 1.333, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 2.667, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 4.000, error: 0.0005 } } --- ewma: diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/random_sampler.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/random_sampler.yml index 5b2c2dc379cb9..4d8efe2a6f9d8 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/random_sampler.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/random_sampler.yml @@ -142,6 +142,66 @@ setup: } - match: { aggregations.sampled.mean.value: 1.0 } --- +"Test random_sampler aggregation with scored subagg": + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ random_sampler_with_scored_subaggs ] + test_runner_features: capabilities + reason: "Support for random sampler with scored subaggs capability required" + - do: + search: + index: data + size: 0 + body: > + { + "query": { + "function_score": { + "random_score": {} + } + }, + "aggs": { + "sampled": { + "random_sampler": { + "probability": 0.5 + }, + "aggs": { + "top": { + "top_hits": {} + } + } + } + } + } + - is_true: aggregations.sampled.top.hits + - do: + search: + index: data + size: 0 + body: > + { + "query": { + "function_score": { + "random_score": {} + } + }, + "aggs": { + "sampled": { + "random_sampler": { + "probability": 1.0 + }, + "aggs": { + "top": { + "top_hits": {} + } + } + } + } + } + - match: { aggregations.sampled.top.hits.total.value: 6 } + - is_true: aggregations.sampled.top.hits.hits.0._score +--- "Test random_sampler aggregation with poor settings": - requires: cluster_features: ["gte_v8.2.0"] diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/stats_metric_fail_formatting.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/stats_metric_fail_formatting.yml index d9298a832e650..1ff376eac61d1 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/stats_metric_fail_formatting.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/stats_metric_fail_formatting.yml @@ -3,6 +3,8 @@ setup: indices.create: index: test_date body: + settings: + number_of_shards: 1 mappings: properties: date_field: @@ -30,7 +32,7 @@ setup: cluster_features: "gte_v8.15.0" reason: fixed in 8.15.0 - do: - catch: /Cannot format stat \[sum\] with format \[DocValueFormat.DateTime\(format\[date_hour_minute_second_millis\] locale\[\], Z, MILLISECONDS\)\]/ + catch: /Cannot format stat \[sum\] with format \[DocValueFormat.DateTime\(format\[date_hour_minute_second_millis\] locale\[(en)?\], Z, MILLISECONDS\)\]/ search: index: test_date body: diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml index 1703d4908a753..acab855e17df6 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml @@ -291,23 +291,6 @@ setup: sum: sum: field: val ---- -"Configure with no synthetic source": - - requires: - cluster_features: ["gte_v8.15.0"] - reason: "Error message changed in 8.15.0" - - - do: - catch: '/Indices with with index mode \[time_series\] only support synthetic source/' - indices.create: - index: tsdb_error - body: - settings: - mode: time_series - routing_path: [key] - mappings: - _source: - enabled: false --- "Number for keyword routing field": diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/top_hits.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/top_hits.yml index ed24e1cc8404c..48fe6a5f4dbbf 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/top_hits.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/top_hits.yml @@ -349,16 +349,17 @@ sequence number and primary term: --- synthetic _source: - requires: - cluster_features: ["gte_v8.4.0"] - reason: introduced in 8.4.0 + cluster_features: ["mapper.source.mode_from_index_setting"] + reason: "Source mode configured through index setting" - do: indices.create: index: test_synthetic body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: page: type: keyword diff --git a/modules/analysis-common/build.gradle b/modules/analysis-common/build.gradle index 1fc42a1b294fe..ac339b5c89c46 100644 --- a/modules/analysis-common/build.gradle +++ b/modules/analysis-common/build.gradle @@ -1,19 +1,20 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ -import org.elasticsearch.gradle.Version +import org.elasticsearch.gradle.internal.info.BuildParams apply plugin: 'elasticsearch.internal-yaml-rest-test' apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { - description 'Adds "built in" analyzers to Elasticsearch.' - classname 'org.elasticsearch.analysis.common.CommonAnalysisPlugin' + description = 'Adds "built in" analyzers to Elasticsearch.' + classname = 'org.elasticsearch.analysis.common.CommonAnalysisPlugin' extendedPlugins = ['lang-painless'] } @@ -29,6 +30,15 @@ dependencies { clusterModules project(':modules:mapper-extras') } +tasks.named("yamlRestTest").configure { task-> + if (buildParams.getRuntimeJavaVersion().get().majorVersion.toInteger() < 21) { + // Requires at least Java 21 + systemProperty 'tests.rest.blacklist', [ + "analysis-common/50_char_filters/pattern_replace error handling (too complex pattern)" + ].join(',') + } +} + tasks.named("yamlRestTestV7CompatTransform").configure { task -> task.skipTest("indices.analyze/10_analyze/htmlStrip_deprecated", "Cleanup versioned deprecations in analysis #41560") task.skipTest("analysis-common/40_token_filters/delimited_payload_filter_error", "Remove preconfigured delimited_payload_filter #43686") diff --git a/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersIT.java b/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersIT.java index bb450f1cc43ee..2aac5d0e6d8a4 100644 --- a/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersIT.java +++ b/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/ReloadAnalyzerTests.java b/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/ReloadAnalyzerTests.java index 1014406745e1c..40a7b64bc91e7 100644 --- a/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/ReloadAnalyzerTests.java +++ b/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/ReloadAnalyzerTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/ReloadSynonymAnalyzerIT.java b/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/ReloadSynonymAnalyzerIT.java index 4be28483e0d30..8209d9f543a31 100644 --- a/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/ReloadSynonymAnalyzerIT.java +++ b/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/ReloadSynonymAnalyzerIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/module-info.java b/modules/analysis-common/src/main/java/module-info.java index c2cc2b5f2ada7..d33c3a6da4de4 100644 --- a/modules/analysis-common/src/main/java/module-info.java +++ b/modules/analysis-common/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.analysis.common { diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactory.java index c7aff01a1ba14..ba4333bf1be01 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AbstractCompoundWordTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AbstractCompoundWordTokenFilterFactory.java index 6ec3f9fdcb087..1b68b2a6ebaf9 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AbstractCompoundWordTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AbstractCompoundWordTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AnalysisPainlessExtension.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AnalysisPainlessExtension.java index 4bbe159820a56..fb5391cc556f0 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AnalysisPainlessExtension.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AnalysisPainlessExtension.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AnalysisPredicateScript.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AnalysisPredicateScript.java index 3ce6021bfb63d..03ad4a56e32f3 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AnalysisPredicateScript.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AnalysisPredicateScript.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ApostropheFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ApostropheFilterFactory.java index f9a50b5918b69..97a5de3a1408a 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ApostropheFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ApostropheFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ArabicAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ArabicAnalyzerProvider.java index acf12c8153817..4e021f8138e76 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ArabicAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ArabicAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ArabicNormalizationFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ArabicNormalizationFilterFactory.java index bfa3c05a27bea..fb55dcaa67961 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ArabicNormalizationFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ArabicNormalizationFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ArabicStemTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ArabicStemTokenFilterFactory.java index d23d3fd6129bb..a8f8fa6f1bcab 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ArabicStemTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ArabicStemTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ArmenianAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ArmenianAnalyzerProvider.java index 56c4d8b166780..a3a5638ea31f9 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ArmenianAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ArmenianAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/BasqueAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/BasqueAnalyzerProvider.java index 1b49ee8a90bce..cda85c71e5b8e 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/BasqueAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/BasqueAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/BengaliAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/BengaliAnalyzerProvider.java index 7431557ccb70c..32979b5841c83 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/BengaliAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/BengaliAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/BengaliNormalizationFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/BengaliNormalizationFilterFactory.java index 7fc1a6f3bee8a..858287cb12f56 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/BengaliNormalizationFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/BengaliNormalizationFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/BrazilianAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/BrazilianAnalyzerProvider.java index e5d4fdf3a7242..94e0dd627ee85 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/BrazilianAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/BrazilianAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/BrazilianStemTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/BrazilianStemTokenFilterFactory.java index 392245a9e6198..c3ccdc517df06 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/BrazilianStemTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/BrazilianStemTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/BulgarianAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/BulgarianAnalyzerProvider.java index 0e9359c81a625..cf6411ed29c8b 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/BulgarianAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/BulgarianAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CJKBigramFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CJKBigramFilterFactory.java index a5db0c4bcb6a6..c34847258109b 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CJKBigramFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CJKBigramFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CJKWidthFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CJKWidthFilterFactory.java index a85247279747e..07e28c5a4924f 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CJKWidthFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CJKWidthFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CatalanAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CatalanAnalyzerProvider.java index e902f84833e68..2be6f220e4441 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CatalanAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CatalanAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CharGroupTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CharGroupTokenizerFactory.java index 3d0f9a64876e1..b2f0b55a749d7 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CharGroupTokenizerFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CharGroupTokenizerFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CharMatcher.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CharMatcher.java index 33d5378f5bcb0..ad9ea667e16ff 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CharMatcher.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CharMatcher.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ChineseAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ChineseAnalyzerProvider.java index bcad4330b196b..3adec292d7650 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ChineseAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ChineseAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CjkAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CjkAnalyzerProvider.java index 1fd9281cad6a3..96f6718127399 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CjkAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CjkAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicFilterFactory.java index bc86b8f56d09f..94bfe5eed3378 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicTokenizerFactory.java index 580597bbc2442..6f54c10146a01 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicTokenizerFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicTokenizerFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java index 35face57b8294..4f8ebae6693fd 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -365,7 +366,7 @@ public Map> getTokenizers() { if (indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.V_8_0_0)) { throw new IllegalArgumentException( "The [edgeNGram] tokenizer name was deprecated in 7.6. " - + "Please use the tokenizer name to [edge_nGram] for indices created in versions 8 or higher instead." + + "Please use the tokenizer name to [edge_ngram] for indices created in versions 8 or higher instead." ); } else if (indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.V_7_6_0)) { deprecationLogger.warn( diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonGramsTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonGramsTokenFilterFactory.java index a93c57c0f1c37..959da9d368189 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonGramsTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonGramsTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CzechAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CzechAnalyzerProvider.java index bd4f99e8094f1..e65fb0da548f1 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CzechAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CzechAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CzechStemTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CzechStemTokenFilterFactory.java index 7a7a09a9b0912..ed9fb66d8aa54 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CzechStemTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CzechStemTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DanishAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DanishAnalyzerProvider.java index a1fce7bff36a9..6605937f23fd2 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DanishAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DanishAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DecimalDigitFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DecimalDigitFilterFactory.java index 35f1803b79421..9ae6375596f6b 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DecimalDigitFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DecimalDigitFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DelimitedPayloadTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DelimitedPayloadTokenFilterFactory.java index 5f266d1517893..53f7667d226d2 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DelimitedPayloadTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DelimitedPayloadTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DictionaryCompoundWordTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DictionaryCompoundWordTokenFilterFactory.java index 22f6762286f11..7c2bb6ba1c116 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DictionaryCompoundWordTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DictionaryCompoundWordTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DutchAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DutchAnalyzerProvider.java index 80a11a3a2a143..2f73b1d982a08 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DutchAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DutchAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DutchStemTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DutchStemTokenFilterFactory.java index 9fdbeea7e7d16..04cddb2ebd6c5 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DutchStemTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DutchStemTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ESSolrSynonymParser.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ESSolrSynonymParser.java index 486b83166e45b..867edaa98d9dd 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ESSolrSynonymParser.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ESSolrSynonymParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ESWordnetSynonymParser.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ESWordnetSynonymParser.java index 6466335449e1b..dd9c3083aa486 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ESWordnetSynonymParser.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ESWordnetSynonymParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/EdgeNGramTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/EdgeNGramTokenFilterFactory.java index 07f2dd83ec94b..eda1c1ca19eac 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/EdgeNGramTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/EdgeNGramTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerFactory.java index fe956dfe41464..a8ae783711f87 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ElisionTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ElisionTokenFilterFactory.java index 52d9898934c37..ff454849196be 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ElisionTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ElisionTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/EnglishAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/EnglishAnalyzerProvider.java index 5fd2578ffbd92..e613dd0f1aa15 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/EnglishAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/EnglishAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/EstonianAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/EstonianAnalyzerProvider.java index 255c23149fdbb..d6bdf293021fc 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/EstonianAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/EstonianAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FingerprintAnalyzer.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FingerprintAnalyzer.java index 9e2fd9057dde4..119a3f862e343 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FingerprintAnalyzer.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FingerprintAnalyzer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FingerprintAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FingerprintAnalyzerProvider.java index 32a90bb7a8f64..452d57e9a8dbf 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FingerprintAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FingerprintAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FingerprintTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FingerprintTokenFilterFactory.java index daded81ea56b5..86ab5f7d3a5b8 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FingerprintTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FingerprintTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FinnishAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FinnishAnalyzerProvider.java index f1a5012e7208c..2caf30c30fef0 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FinnishAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FinnishAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FlattenGraphTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FlattenGraphTokenFilterFactory.java index 6a65eadcd9118..28724b3360e19 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FlattenGraphTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FlattenGraphTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FrenchAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FrenchAnalyzerProvider.java index 6ff88037163f9..6bc447e12987d 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FrenchAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FrenchAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FrenchStemTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FrenchStemTokenFilterFactory.java index 65ff7a6488c3b..3ac9afd0e34a4 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FrenchStemTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/FrenchStemTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/GalicianAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/GalicianAnalyzerProvider.java index 0ec8c668fb90d..d898cf3ee000d 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/GalicianAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/GalicianAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/GermanAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/GermanAnalyzerProvider.java index 2c895a31b0177..870324012b2e6 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/GermanAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/GermanAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/GermanNormalizationFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/GermanNormalizationFilterFactory.java index 3d0473a9ad134..9722140c82e7a 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/GermanNormalizationFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/GermanNormalizationFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/GermanStemTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/GermanStemTokenFilterFactory.java index 4474833d748bf..17f072ccc647f 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/GermanStemTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/GermanStemTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/GreekAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/GreekAnalyzerProvider.java index c0cc5e0c96a94..d9c394942d6d2 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/GreekAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/GreekAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HindiAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HindiAnalyzerProvider.java index f410ec651b14a..b28b7e5a3c8a1 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HindiAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HindiAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HindiNormalizationFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HindiNormalizationFilterFactory.java index 35fa6e9f3c92c..0be557154acbc 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HindiNormalizationFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HindiNormalizationFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HtmlStripCharFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HtmlStripCharFilterFactory.java index b2b43bf0cde1a..9e1ab6d6149fe 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HtmlStripCharFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HtmlStripCharFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HungarianAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HungarianAnalyzerProvider.java index f658bdec6c8d7..e7040479fdce6 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HungarianAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HungarianAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HyphenationCompoundWordTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HyphenationCompoundWordTokenFilterFactory.java index 79d2662491b6e..e091f0175009e 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HyphenationCompoundWordTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HyphenationCompoundWordTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -27,6 +28,8 @@ */ public class HyphenationCompoundWordTokenFilterFactory extends AbstractCompoundWordTokenFilterFactory { + private final boolean noSubMatches; + private final boolean noOverlappingMatches; private final HyphenationTree hyphenationTree; HyphenationCompoundWordTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { @@ -45,6 +48,9 @@ public class HyphenationCompoundWordTokenFilterFactory extends AbstractCompoundW } catch (Exception e) { throw new IllegalArgumentException("Exception while reading hyphenation_patterns_path.", e); } + + noSubMatches = settings.getAsBoolean("no_sub_matches", false); + noOverlappingMatches = settings.getAsBoolean("no_overlapping_matches", false); } @Override @@ -56,7 +62,9 @@ public TokenStream create(TokenStream tokenStream) { minWordSize, minSubwordSize, maxSubwordSize, - onlyLongestMatch + onlyLongestMatch, + noSubMatches, + noOverlappingMatches ); } } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/IndicNormalizationFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/IndicNormalizationFilterFactory.java index cd52345a12e25..2d40a93568bde 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/IndicNormalizationFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/IndicNormalizationFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/IndonesianAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/IndonesianAnalyzerProvider.java index 39038d49f3cc3..a822061cd7f4a 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/IndonesianAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/IndonesianAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/IrishAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/IrishAnalyzerProvider.java index 19a7e82aca676..293a3d87e48fc 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/IrishAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/IrishAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ItalianAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ItalianAnalyzerProvider.java index 887b6e0798c57..ae11c94f595fb 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ItalianAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ItalianAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KStemTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KStemTokenFilterFactory.java index eca004dd45bb9..7f741077fa7ad 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KStemTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KStemTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeepTypesFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeepTypesFilterFactory.java index bc1587a4fc66f..5b65450a2782a 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeepTypesFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeepTypesFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeepWordFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeepWordFilterFactory.java index 070f0f038f29e..0fa763d627a7a 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeepWordFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeepWordFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeywordAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeywordAnalyzerProvider.java index a4bc2dbac1d33..c43327cf508cc 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeywordAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeywordAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeywordMarkerTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeywordMarkerTokenFilterFactory.java index de44dbff88d27..357568eb42076 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeywordMarkerTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeywordMarkerTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeywordTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeywordTokenizerFactory.java index 9ec69fc006a6d..a20cfd80a05ee 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeywordTokenizerFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeywordTokenizerFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LatvianAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LatvianAnalyzerProvider.java index 900c28b2b1bf3..36b05ca9b8975 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LatvianAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LatvianAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LengthTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LengthTokenFilterFactory.java index f38fa6b4b2982..0a9ae48d8472b 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LengthTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LengthTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LetterTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LetterTokenizerFactory.java index 7dda7cb3cf596..f0651e928fc2b 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LetterTokenizerFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LetterTokenizerFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LimitTokenCountFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LimitTokenCountFilterFactory.java index b83be77243f79..7cdc08941d043 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LimitTokenCountFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LimitTokenCountFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LithuanianAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LithuanianAnalyzerProvider.java index 97854a40dcb61..ce407de1d053d 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LithuanianAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LithuanianAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LowerCaseTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LowerCaseTokenFilterFactory.java index 9d6bb37cdf540..bb5538fc6e174 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LowerCaseTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LowerCaseTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MappingCharFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MappingCharFilterFactory.java index 05b7f6713ed69..ab05185d28b16 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MappingCharFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MappingCharFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MinHashTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MinHashTokenFilterFactory.java index 2d4da603b851c..cc2526f36003a 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MinHashTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MinHashTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MultiplexerTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MultiplexerTokenFilterFactory.java index fba6b78a23cff..d78feeafd66b8 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MultiplexerTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MultiplexerTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/NGramTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/NGramTokenFilterFactory.java index 31afa780f27e3..12277bfba5a57 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/NGramTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/NGramTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/NGramTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/NGramTokenizerFactory.java index a8b5489f81d0b..a3533d5952c64 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/NGramTokenizerFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/NGramTokenizerFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/NorwegianAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/NorwegianAnalyzerProvider.java index 395cfdd8c1745..5d54ad5ea50fd 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/NorwegianAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/NorwegianAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PathHierarchyTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PathHierarchyTokenizerFactory.java index 969ef2fd86e8f..30e60d58cc9ca 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PathHierarchyTokenizerFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PathHierarchyTokenizerFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternAnalyzer.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternAnalyzer.java index c0582be9c9be5..7a2a25b147cbf 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternAnalyzer.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternAnalyzer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternAnalyzerProvider.java index 6b27f96e3a0de..ff59210319ab8 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternCaptureGroupTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternCaptureGroupTokenFilterFactory.java index 092a80b86dfd0..94102ac2b6747 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternCaptureGroupTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternCaptureGroupTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternReplaceCharFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternReplaceCharFilterFactory.java index 0e5b1a3a8a8e7..2e10c25276046 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternReplaceCharFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternReplaceCharFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternReplaceTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternReplaceTokenFilterFactory.java index b037aee147fb9..cd15b05aabfbe 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternReplaceTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternReplaceTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternTokenizerFactory.java index ba4cf17d20a3c..b8f2e194c2ca0 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternTokenizerFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternTokenizerFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PersianAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PersianAnalyzerProvider.java index e0233c3d7922d..9ea3a9fa4eee9 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PersianAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PersianAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PersianNormalizationFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PersianNormalizationFilterFactory.java index 813ab6e237e8a..6915a9e992fa5 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PersianNormalizationFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PersianNormalizationFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PersianStemTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PersianStemTokenFilterFactory.java index 4fcf3fe896fbd..a3a5035cbe871 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PersianStemTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PersianStemTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PorterStemTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PorterStemTokenFilterFactory.java index 84c0534d43000..8171605c9732e 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PorterStemTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PorterStemTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PortugueseAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PortugueseAnalyzerProvider.java index 45ed643b55b6d..a08dafd26e1fc 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PortugueseAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PortugueseAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PredicateTokenFilterScriptFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PredicateTokenFilterScriptFactory.java index 8ffc6a6cc83c2..245793092b894 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PredicateTokenFilterScriptFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PredicateTokenFilterScriptFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/RemoveDuplicatesTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/RemoveDuplicatesTokenFilterFactory.java index c7389338af662..7c7bb8b249e70 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/RemoveDuplicatesTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/RemoveDuplicatesTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ReverseTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ReverseTokenFilterFactory.java index d1fb6caa1abf2..eee23f0a775d4 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ReverseTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ReverseTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/RomanianAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/RomanianAnalyzerProvider.java index ea194b174489d..cf33a38abd634 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/RomanianAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/RomanianAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/RussianAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/RussianAnalyzerProvider.java index 0fa2a4178a943..b3deb700aef34 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/RussianAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/RussianAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/RussianStemTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/RussianStemTokenFilterFactory.java index 24074d29a33e2..55405ba09c33d 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/RussianStemTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/RussianStemTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScandinavianFoldingFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScandinavianFoldingFilterFactory.java index c429ae1a576f8..0e713c34ffbf5 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScandinavianFoldingFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScandinavianFoldingFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScandinavianNormalizationFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScandinavianNormalizationFilterFactory.java index 508e2b6f27155..5ba612dd2495a 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScandinavianNormalizationFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScandinavianNormalizationFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterFactory.java index b41e52835ebe8..c5c35ebe207fe 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SerbianAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SerbianAnalyzerProvider.java index 567502b75bced..6dc899be95875 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SerbianAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SerbianAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SerbianNormalizationFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SerbianNormalizationFilterFactory.java index 4da6c2f16d965..bb46039fb7e71 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SerbianNormalizationFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SerbianNormalizationFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SimpleAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SimpleAnalyzerProvider.java index fb63e4e18b2b1..60eb17dcdba2f 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SimpleAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SimpleAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SimplePatternSplitTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SimplePatternSplitTokenizerFactory.java index ab508fcbfe34d..8a6cf514fda3f 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SimplePatternSplitTokenizerFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SimplePatternSplitTokenizerFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SimplePatternTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SimplePatternTokenizerFactory.java index ff741f297f76b..42f5bb0e80927 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SimplePatternTokenizerFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SimplePatternTokenizerFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SnowballAnalyzer.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SnowballAnalyzer.java index 9a10cc2493523..4ab47a3b55ff8 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SnowballAnalyzer.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SnowballAnalyzer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SnowballAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SnowballAnalyzerProvider.java index 8a51176c856fa..b9c0ceb49f4e8 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SnowballAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SnowballAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SnowballTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SnowballTokenFilterFactory.java index 57b19f8348c8e..c480da6edb5b4 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SnowballTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SnowballTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SoraniAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SoraniAnalyzerProvider.java index f53e9eb1aeab1..f5d58c2cae0ca 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SoraniAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SoraniAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SoraniNormalizationFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SoraniNormalizationFilterFactory.java index a5e3128f4ddc6..43faa49fdeb9f 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SoraniNormalizationFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SoraniNormalizationFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SpanishAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SpanishAnalyzerProvider.java index d45e50ebb93cd..a6785a0d45514 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SpanishAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SpanishAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StemmerOverrideTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StemmerOverrideTokenFilterFactory.java index 105267eeb14d4..81fb5f86db23e 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StemmerOverrideTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StemmerOverrideTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactory.java index 69b0921a4144f..1c71c64311517 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -46,6 +47,8 @@ import org.apache.lucene.analysis.snowball.SnowballFilter; import org.apache.lucene.analysis.sv.SwedishLightStemFilter; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.logging.DeprecationCategory; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; @@ -80,6 +83,8 @@ public class StemmerTokenFilterFactory extends AbstractTokenFilterFactory { + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(StemmerTokenFilterFactory.class); + private static final TokenStream EMPTY_TOKEN_STREAM = new EmptyTokenStream(); private String language; @@ -89,6 +94,20 @@ public class StemmerTokenFilterFactory extends AbstractTokenFilterFactory { this.language = Strings.capitalize(settings.get("language", settings.get("name", "porter"))); // check that we have a valid language by trying to create a TokenStream create(EMPTY_TOKEN_STREAM).close(); + if ("lovins".equalsIgnoreCase(language)) { + deprecationLogger.critical( + DeprecationCategory.ANALYSIS, + "lovins_deprecation", + "The [lovins] stemmer is deprecated and will be removed in a future version." + ); + } + if ("dutch_kp".equalsIgnoreCase(language) || "dutchKp".equalsIgnoreCase(language) || "kp".equalsIgnoreCase(language)) { + deprecationLogger.critical( + DeprecationCategory.ANALYSIS, + "dutch_kp_deprecation", + "The [dutch_kp] stemmer is deprecated and will be removed in a future version." + ); + } } @Override diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StopAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StopAnalyzerProvider.java index 82165369b7a33..0977d08d0fd48 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StopAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StopAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SwedishAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SwedishAnalyzerProvider.java index c75773792dcf0..4c0ad0ebc4678 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SwedishAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SwedishAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymGraphTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymGraphTokenFilterFactory.java index 7691253313d50..d6d4e644b06b2 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymGraphTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymGraphTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java index 762de01b93d51..9e31fdde4330b 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ThaiAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ThaiAnalyzerProvider.java index b461aca6ddc35..45f8ef180d686 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ThaiAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ThaiAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ThaiTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ThaiTokenizerFactory.java index 4b85a64636296..ba4562673dc57 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ThaiTokenizerFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ThaiTokenizerFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TrimTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TrimTokenFilterFactory.java index 433f9f2ca380f..4a754de207fba 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TrimTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TrimTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TruncateTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TruncateTokenFilterFactory.java index 6c37ac70b89e8..9f35609b675e8 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TruncateTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TruncateTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TurkishAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TurkishAnalyzerProvider.java index 29dd5b725ed32..4621a821b9df4 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TurkishAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TurkishAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UAX29URLEmailTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UAX29URLEmailTokenizerFactory.java index 2391c36b90a29..2c5f3faf5114a 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UAX29URLEmailTokenizerFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UAX29URLEmailTokenizerFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UniqueTokenFilter.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UniqueTokenFilter.java index 01a1e05ff40f2..5b43d31651d87 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UniqueTokenFilter.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UniqueTokenFilter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UniqueTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UniqueTokenFilterFactory.java index e9f29f590f991..73166b36478f5 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UniqueTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UniqueTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UpperCaseTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UpperCaseTokenFilterFactory.java index 6e7ce922328db..0afebbbacb7e9 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UpperCaseTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UpperCaseTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WhitespaceAnalyzerProvider.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WhitespaceAnalyzerProvider.java index 5656a99069822..995f24381de0f 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WhitespaceAnalyzerProvider.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WhitespaceAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WhitespaceTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WhitespaceTokenizerFactory.java index 10878b21ae7c1..0a5813a19d2ce 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WhitespaceTokenizerFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WhitespaceTokenizerFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactory.java index fe4f62ef3f2cb..f55e7b255d53c 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactory.java index e766ce9eba511..083594f6ab02e 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/XLowerCaseTokenizer.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/XLowerCaseTokenizer.java index 392af4cd567b0..ed867be04c45a 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/XLowerCaseTokenizer.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/XLowerCaseTokenizer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/XLowerCaseTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/XLowerCaseTokenizerFactory.java index 1ee7d586332df..7fc1bf6882354 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/XLowerCaseTokenizerFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/XLowerCaseTokenizerFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/XUniqueTokenFilter.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/XUniqueTokenFilter.java index 35ac6ed5398c2..c4d90ad61b958 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/XUniqueTokenFilter.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/XUniqueTokenFilter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/main/resources/org/elasticsearch/analysis/common/painless_whitelist.txt b/modules/analysis-common/src/main/resources/org/elasticsearch/analysis/common/painless_whitelist.txt index f99f26fc3467f..ae5934f838f75 100644 --- a/modules/analysis-common/src/main/resources/org/elasticsearch/analysis/common/painless_whitelist.txt +++ b/modules/analysis-common/src/main/resources/org/elasticsearch/analysis/common/painless_whitelist.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # class org.elasticsearch.analysis.common.AnalysisPredicateScript$Token { CharSequence getTerm() diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactoryTests.java index 32c16d01333bf..c3017bd3ea237 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -20,6 +21,8 @@ import java.io.IOException; import java.io.StringReader; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; + public class ASCIIFoldingTokenFilterFactoryTests extends ESTokenStreamTestCase { public void testDefault() throws IOException { ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings( diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/BaseWordDelimiterTokenFilterFactoryTestCase.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/BaseWordDelimiterTokenFilterFactoryTestCase.java index 51522b33ecdde..ab26112005bd6 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/BaseWordDelimiterTokenFilterFactoryTestCase.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/BaseWordDelimiterTokenFilterFactoryTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -19,6 +20,8 @@ import java.io.IOException; import java.io.StringReader; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; + /** * Base class to test {@link WordDelimiterTokenFilterFactory} and * {@link WordDelimiterGraphTokenFilterFactory}. diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CJKFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CJKFilterFactoryTests.java index af9c6b79882e1..95b093b03f9a7 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CJKFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CJKFilterFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -21,6 +22,8 @@ import java.io.IOException; import java.io.StringReader; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; + public class CJKFilterFactoryTests extends ESTokenStreamTestCase { private static final String RESOURCE = "/org/elasticsearch/analysis/common/cjk_analysis.json"; diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CharGroupTokenizerFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CharGroupTokenizerFactoryTests.java index 31c086cfe4ef8..ee9701b89c127 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CharGroupTokenizerFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CharGroupTokenizerFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -23,6 +24,8 @@ import java.io.StringReader; import java.util.Arrays; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; + public class CharGroupTokenizerFactoryTests extends ESTokenStreamTestCase { public void testParseTokenChars() { diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CharMatcherTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CharMatcherTests.java index 8018c392cf716..24843053bfce0 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CharMatcherTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CharMatcherTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java index f147cb47a2c01..02565305cc494 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisPluginTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisPluginTests.java index c18cb3dddf0ae..9972d58b2dcc1 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisPluginTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisPluginTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonGramsTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonGramsTokenFilterFactoryTests.java index e62910d4d1bbd..77902a2ab982f 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonGramsTokenFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonGramsTokenFilterFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -26,6 +27,8 @@ import java.nio.file.Files; import java.nio.file.Path; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; + public class CommonGramsTokenFilterFactoryTests extends ESTokenStreamTestCase { public void testDefault() throws IOException { Settings settings = Settings.builder() diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CompoundAnalysisTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CompoundAnalysisTests.java index 6ac62246e3da5..69dd8e91b52b2 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CompoundAnalysisTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CompoundAnalysisTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -30,6 +31,9 @@ import org.hamcrest.MatcherAssert; import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -41,6 +45,7 @@ import static org.hamcrest.Matchers.instanceOf; public class CompoundAnalysisTests extends ESTestCase { + public void testDefaultsCompoundAnalysis() throws Exception { Settings settings = getJsonSettings(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings); @@ -62,6 +67,44 @@ public void testDictionaryDecompounder() throws Exception { assertWarnings("Setting [version] on analysis component [custom7] has no effect and is deprecated"); } + public void testHyphenationDecompoundingAnalyzerOnlyLongestMatch() throws Exception { + Settings[] settingsArr = new Settings[] { getJsonSettings(), getYamlSettings() }; + for (Settings settings : settingsArr) { + List terms = analyze(settings, "hyphenationDecompoundingAnalyzerOnlyLongestMatch", "kaffeemaschine fussballpumpe"); + MatcherAssert.assertThat( + terms, + hasItems("kaffeemaschine", "kaffee", "fee", "maschine", "fussballpumpe", "fussball", "ballpumpe", "pumpe") + ); + } + assertWarnings("Setting [version] on analysis component [custom7] has no effect and is deprecated"); + } + + /** + * For example given a word list of: ["kaffee", "fee", "maschine"] + * no_sub_matches should prevent the token "fee" as a token in "kaffeemaschine". + */ + public void testHyphenationDecompoundingAnalyzerNoSubMatches() throws Exception { + Settings[] settingsArr = new Settings[] { getJsonSettings(), getYamlSettings() }; + for (Settings settings : settingsArr) { + List terms = analyze(settings, "hyphenationDecompoundingAnalyzerNoSubMatches", "kaffeemaschine fussballpumpe"); + MatcherAssert.assertThat(terms, hasItems("kaffeemaschine", "kaffee", "maschine", "fussballpumpe", "fussball", "ballpumpe")); + } + assertWarnings("Setting [version] on analysis component [custom7] has no effect and is deprecated"); + } + + /** + * For example given a word list of: ["fuss", "fussball", "ballpumpe", "ball", "pumpe"] + * no_overlapping_matches should prevent the token "ballpumpe" as a token in "fussballpumpe. + */ + public void testHyphenationDecompoundingAnalyzerNoOverlappingMatches() throws Exception { + Settings[] settingsArr = new Settings[] { getJsonSettings(), getYamlSettings() }; + for (Settings settings : settingsArr) { + List terms = analyze(settings, "hyphenationDecompoundingAnalyzerNoOverlappingMatches", "kaffeemaschine fussballpumpe"); + MatcherAssert.assertThat(terms, hasItems("kaffeemaschine", "kaffee", "maschine", "fussballpumpe", "fussball", "pumpe")); + } + assertWarnings("Setting [version] on analysis component [custom7] has no effect and is deprecated"); + } + private List analyze(Settings settings, String analyzerName, String text) throws IOException { IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings); AnalysisModule analysisModule = createAnalysisModule(settings); @@ -91,20 +134,25 @@ public Map> getTokenFilters() { } private Settings getJsonSettings() throws IOException { - String json = "/org/elasticsearch/analysis/common/test1.json"; - return Settings.builder() - .loadFromStream(json, getClass().getResourceAsStream(json), false) - .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) - .build(); + return getSettings("/org/elasticsearch/analysis/common/test1.json"); } private Settings getYamlSettings() throws IOException { - String yaml = "/org/elasticsearch/analysis/common/test1.yml"; + return getSettings("/org/elasticsearch/analysis/common/test1.yml"); + } + + private Settings getSettings(String filePath) throws IOException { + String hypenationRulesFileName = "de_DR.xml"; + InputStream hypenationRules = getClass().getResourceAsStream(hypenationRulesFileName); + Path home = createTempDir(); + Path config = home.resolve("config"); + Files.createDirectory(config); + Files.copy(hypenationRules, config.resolve(hypenationRulesFileName)); + return Settings.builder() - .loadFromStream(yaml, getClass().getResourceAsStream(yaml), false) + .loadFromStream(filePath, getClass().getResourceAsStream(filePath), false) .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put(Environment.PATH_HOME_SETTING.getKey(), home.toString()) .build(); } } diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java index 8da4729db1367..e9f3d628bfead 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ESSolrSynonymParserTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ESSolrSynonymParserTests.java index 5b3fb137b9be6..b0cda5620c3e5 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ESSolrSynonymParserTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ESSolrSynonymParserTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -22,6 +23,7 @@ import java.io.StringReader; import java.text.ParseException; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; import static org.hamcrest.Matchers.containsString; public class ESSolrSynonymParserTests extends ESTokenStreamTestCase { diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ESWordnetSynonymParserTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ESWordnetSynonymParserTests.java index c30d0c85703a2..17455c431992f 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ESWordnetSynonymParserTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ESWordnetSynonymParserTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -22,6 +23,7 @@ import java.io.StringReader; import java.text.ParseException; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; import static org.hamcrest.Matchers.containsString; public class ESWordnetSynonymParserTests extends ESTokenStreamTestCase { diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenFilterFactoryTests.java index 5bb7b4d38c1ff..446cee8f48379 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenFilterFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -20,6 +21,8 @@ import java.io.IOException; import java.io.StringReader; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; + public class EdgeNGramTokenFilterFactoryTests extends ESTokenStreamTestCase { public void testDefault() throws IOException { diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerTests.java index 412e3ba3e380a..28f73b1bbca4c 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -30,6 +31,9 @@ import java.io.StringReader; import java.util.Collections; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertAnalyzesTo; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; + public class EdgeNGramTokenizerTests extends ESTokenStreamTestCase { private IndexAnalyzers buildAnalyzers(IndexVersion version, String tokenizer) throws IOException { @@ -87,20 +91,21 @@ public void testPreConfiguredTokenizer() throws IOException { // Check deprecated name as well, needs version before 8.0 because throws IAE after that { - try ( - IndexAnalyzers indexAnalyzers = buildAnalyzers( - IndexVersionUtils.randomVersionBetween( - random(), - IndexVersions.V_7_3_0, - IndexVersionUtils.getPreviousVersion(IndexVersions.V_8_0_0) - ), - "edgeNGram" - ) - ) { + IndexVersion version = IndexVersionUtils.randomVersionBetween( + random(), + IndexVersions.V_7_3_0, + IndexVersionUtils.getPreviousVersion(IndexVersions.V_8_0_0) + ); + try (IndexAnalyzers indexAnalyzers = buildAnalyzers(version, "edgeNGram")) { NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer"); assertNotNull(analyzer); assertAnalyzesTo(analyzer, "test", new String[] { "t", "te" }); - + } + if (version.onOrAfter(IndexVersions.V_7_6_0)) { + assertWarnings( + "The [edgeNGram] tokenizer name is deprecated and will be removed in a future version. " + + "Please change the tokenizer name to [edge_ngram] instead." + ); } } diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ElisionFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ElisionFilterFactoryTests.java index de2d0b550f500..41133c1677495 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ElisionFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ElisionFilterFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/FingerprintAnalyzerTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/FingerprintAnalyzerTests.java index a85fac1d86067..8783860b8e02e 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/FingerprintAnalyzerTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/FingerprintAnalyzerTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -12,6 +13,8 @@ import org.apache.lucene.analysis.CharArraySet; import org.elasticsearch.test.ESTokenStreamTestCase; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertAnalyzesTo; + public class FingerprintAnalyzerTests extends ESTokenStreamTestCase { public void testFingerprint() throws Exception { diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/FlattenGraphTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/FlattenGraphTokenFilterFactoryTests.java index f01742da8d16b..2f3dd1917ebe2 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/FlattenGraphTokenFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/FlattenGraphTokenFilterFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -19,6 +20,8 @@ import java.io.IOException; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; + public class FlattenGraphTokenFilterFactoryTests extends ESTokenStreamTestCase { public void testBasic() throws IOException { diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java index 2693245ac2757..b406fa8335779 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/KeepFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/KeepFilterFactoryTests.java index 82f8d0b5420dc..a3c9eb2cf3aae 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/KeepFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/KeepFilterFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -21,6 +22,7 @@ import java.io.IOException; import java.io.StringReader; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; import static org.hamcrest.Matchers.instanceOf; public class KeepFilterFactoryTests extends ESTokenStreamTestCase { diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/KeepTypesFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/KeepTypesFilterFactoryTests.java index 06dd33d2b1740..e499f6f7eebdc 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/KeepTypesFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/KeepTypesFilterFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -20,6 +21,7 @@ import java.io.IOException; import java.io.StringReader; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; import static org.hamcrest.Matchers.instanceOf; public class KeepTypesFilterFactoryTests extends ESTokenStreamTestCase { diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/KeywordMarkerFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/KeywordMarkerFilterFactoryTests.java index 0242b4dcbd3a7..8ede58ae2082b 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/KeywordMarkerFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/KeywordMarkerFilterFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -17,11 +18,11 @@ import org.elasticsearch.index.analysis.AnalysisTestsHelper; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.analysis.TokenFilterFactory; -import org.elasticsearch.test.ESTestCase.TestAnalysis; import org.elasticsearch.test.ESTokenStreamTestCase; import java.io.IOException; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertAnalyzesTo; import static org.hamcrest.Matchers.instanceOf; /** diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/LimitTokenCountFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/LimitTokenCountFilterFactoryTests.java index f8cd8aea9c5e5..ee117de653d95 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/LimitTokenCountFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/LimitTokenCountFilterFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -20,6 +21,8 @@ import java.io.IOException; import java.io.StringReader; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; + public class LimitTokenCountFilterFactoryTests extends ESTokenStreamTestCase { public void testDefault() throws IOException { Settings settings = Settings.builder() diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/MassiveWordListTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/MassiveWordListTests.java index 20c988998bdf0..ad2cbd0ffa2d7 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/MassiveWordListTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/MassiveWordListTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/MinHashFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/MinHashFilterFactoryTests.java index 46ea7cf033f79..020b78a50b213 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/MinHashFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/MinHashFilterFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -20,6 +21,8 @@ import java.io.IOException; import java.io.StringReader; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertStreamHasNumberOfTokens; + public class MinHashFilterFactoryTests extends ESTokenStreamTestCase { public void testDefault() throws IOException { int default_hash_count = 1; diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/MultiplexerTokenFilterTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/MultiplexerTokenFilterTests.java index 3d05bbb6cc54c..eb9032061d134 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/MultiplexerTokenFilterTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/MultiplexerTokenFilterTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -25,6 +26,8 @@ import java.io.IOException; import java.util.Collections; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertAnalyzesTo; + public class MultiplexerTokenFilterTests extends ESTokenStreamTestCase { public void testMultiplexingFilter() throws IOException { diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/NGramTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/NGramTokenFilterFactoryTests.java index ab37cada806b7..ef02f91c30a40 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/NGramTokenFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/NGramTokenFilterFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -20,6 +21,8 @@ import java.io.IOException; import java.io.StringReader; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; + public class NGramTokenFilterFactoryTests extends ESTokenStreamTestCase { public void testDefault() throws IOException { diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/NGramTokenizerFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/NGramTokenizerFactoryTests.java index f90b1fecd58a9..8c365a1362f85 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/NGramTokenizerFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/NGramTokenizerFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -27,7 +28,7 @@ import java.io.StringReader; import java.util.Arrays; -import static com.carrotsearch.randomizedtesting.RandomizedTest.scaledRandomIntBetween; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; import static org.hamcrest.Matchers.instanceOf; public class NGramTokenizerFactoryTests extends ESTokenStreamTestCase { @@ -182,6 +183,9 @@ public void testBackwardsCompatibilityEdgeNgramTokenFilter() throws Exception { assertThat(edgeNGramTokenFilter, instanceOf(EdgeNGramTokenFilter.class)); } } + assertWarnings( + "The [side] parameter is deprecated and will be removed. Use a [reverse] before and after the [edge_ngram] instead." + ); } /*` diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PathHierarchyTokenizerFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PathHierarchyTokenizerFactoryTests.java index 64b54e4166134..5121c6390ceb0 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PathHierarchyTokenizerFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PathHierarchyTokenizerFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -19,6 +20,8 @@ import java.io.IOException; import java.io.StringReader; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; + public class PathHierarchyTokenizerFactoryTests extends ESTokenStreamTestCase { public void testDefaults() throws IOException { diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PatternAnalyzerTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PatternAnalyzerTests.java index 353b4c7de1084..91fd74bca9c93 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PatternAnalyzerTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PatternAnalyzerTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -17,6 +18,9 @@ import java.util.Arrays; import java.util.regex.Pattern; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertAnalyzesTo; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.checkRandomData; + /** * Verifies the behavior of PatternAnalyzer. */ diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PatternCaptureTokenFilterTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PatternCaptureTokenFilterTests.java index 2db769127e3c0..80a270b033678 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PatternCaptureTokenFilterTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PatternCaptureTokenFilterTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -18,7 +19,7 @@ import org.elasticsearch.test.ESTokenStreamTestCase; import org.elasticsearch.test.IndexSettingsModule; -import static org.elasticsearch.test.ESTestCase.createTestAnalysis; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; import static org.hamcrest.Matchers.containsString; public class PatternCaptureTokenFilterTests extends ESTokenStreamTestCase { diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PatternReplaceTokenFilterTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PatternReplaceTokenFilterTests.java index 3033d73e58b7f..91637f1cb9449 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PatternReplaceTokenFilterTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PatternReplaceTokenFilterTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -18,6 +19,8 @@ import java.io.IOException; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; + public class PatternReplaceTokenFilterTests extends ESTokenStreamTestCase { public void testNormalizer() throws IOException { diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java index b333c8534d19b..40ba9acbc257a 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -36,6 +37,7 @@ import java.io.IOException; import java.util.Collections; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertAnalyzesTo; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/RemoveDuplicatesFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/RemoveDuplicatesFilterFactoryTests.java index 7e056ff96fa25..bb37b9bb7f4ef 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/RemoveDuplicatesFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/RemoveDuplicatesFilterFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -19,6 +20,7 @@ import java.io.IOException; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; import static org.hamcrest.Matchers.instanceOf; public class RemoveDuplicatesFilterFactoryTests extends ESTokenStreamTestCase { diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterTests.java index 98fdb551c27f1..fb5eee96acffb 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -35,6 +36,7 @@ import java.util.Collections; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertAnalyzesTo; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ShingleTokenFilterTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ShingleTokenFilterTests.java index 7b3c90dc7510f..43084ff36ef16 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ShingleTokenFilterTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ShingleTokenFilterTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SnowballAnalyzerTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SnowballAnalyzerTests.java index 00d0c6c983335..8fd8b86047488 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SnowballAnalyzerTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SnowballAnalyzerTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -12,6 +13,8 @@ import org.apache.lucene.analysis.en.EnglishAnalyzer; import org.elasticsearch.test.ESTokenStreamTestCase; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertAnalyzesTo; + public class SnowballAnalyzerTests extends ESTokenStreamTestCase { public void testEnglish() throws Exception { diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/StemmerOverrideTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/StemmerOverrideTokenFilterFactoryTests.java index 795d6a27882e9..bbe22ff43d52e 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/StemmerOverrideTokenFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/StemmerOverrideTokenFilterFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -24,6 +25,8 @@ import java.util.List; import java.util.Locale; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; + public class StemmerOverrideTokenFilterFactoryTests extends ESTokenStreamTestCase { @Rule public ExpectedException expectedException = ExpectedException.none(); diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactoryTests.java index c668a89e53b95..8f3d52f0174c6 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/StemmerTokenFilterFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -26,12 +27,11 @@ import java.io.IOException; import java.io.StringReader; -import static com.carrotsearch.randomizedtesting.RandomizedTest.scaledRandomIntBetween; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertAnalyzesTo; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED; import static org.hamcrest.Matchers.instanceOf; public class StemmerTokenFilterFactoryTests extends ESTokenStreamTestCase { - private static final CommonAnalysisPlugin PLUGIN = new CommonAnalysisPlugin(); public void testEnglishFilterFactory() throws IOException { @@ -102,4 +102,30 @@ public void testMultipleLanguagesThrowsException() throws IOException { ); assertEquals("Invalid stemmer class specified: [english, light_english]", e.getMessage()); } + + public void testKpDeprecation() throws IOException { + IndexVersion v = IndexVersionUtils.randomVersion(random()); + Settings settings = Settings.builder() + .put("index.analysis.filter.my_kp.type", "stemmer") + .put("index.analysis.filter.my_kp.language", "kp") + .put(SETTING_VERSION_CREATED, v) + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .build(); + + AnalysisTestsHelper.createTestAnalysisFromSettings(settings, PLUGIN); + assertCriticalWarnings("The [dutch_kp] stemmer is deprecated and will be removed in a future version."); + } + + public void testLovinsDeprecation() throws IOException { + IndexVersion v = IndexVersionUtils.randomVersion(random()); + Settings settings = Settings.builder() + .put("index.analysis.filter.my_lovins.type", "stemmer") + .put("index.analysis.filter.my_lovins.language", "lovins") + .put(SETTING_VERSION_CREATED, v) + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .build(); + + AnalysisTestsHelper.createTestAnalysisFromSettings(settings, PLUGIN); + assertCriticalWarnings("The [lovins] stemmer is deprecated and will be removed in a future version."); + } } diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/StopAnalyzerTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/StopAnalyzerTests.java index 58c2f30fd6029..da84c4814514f 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/StopAnalyzerTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/StopAnalyzerTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -18,7 +19,7 @@ import org.elasticsearch.test.ESTokenStreamTestCase; import org.elasticsearch.test.IndexSettingsModule; -import static org.elasticsearch.test.ESTestCase.createTestAnalysis; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; public class StopAnalyzerTests extends ESTokenStreamTestCase { public void testDefaultsCompoundAnalysis() throws Exception { diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymsAnalysisTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymsAnalysisTests.java index 7a2bd2a822988..bd187948cd0f1 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymsAnalysisTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymsAnalysisTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TrimTokenFilterTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TrimTokenFilterTests.java index 3c4387a4f30c7..63e9732f99a8a 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TrimTokenFilterTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TrimTokenFilterTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -18,6 +19,8 @@ import java.io.IOException; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; + public class TrimTokenFilterTests extends ESTokenStreamTestCase { public void testNormalizer() throws IOException { diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/UniqueTokenFilterTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/UniqueTokenFilterTests.java index a2a5180b55ad6..6bec8dc1ebc62 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/UniqueTokenFilterTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/UniqueTokenFilterTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WhitespaceTokenizerFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WhitespaceTokenizerFactoryTests.java index 87323f3a2bb89..5a71853b36d5b 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WhitespaceTokenizerFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WhitespaceTokenizerFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java index 68e6d6661f944..7948afbb499c8 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -31,6 +32,9 @@ import java.io.StringReader; import java.util.Collections; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertAnalyzesTo; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; + public class WordDelimiterGraphTokenFilterFactoryTests extends BaseWordDelimiterTokenFilterFactoryTestCase { public WordDelimiterGraphTokenFilterFactoryTests() { super("word_delimiter_graph"); diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactoryTests.java index 7ee34c42f32ed..636174f5c79cc 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; @@ -18,6 +19,8 @@ import java.io.IOException; import java.io.StringReader; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; + public class WordDelimiterTokenFilterFactoryTests extends BaseWordDelimiterTokenFilterFactoryTestCase { public WordDelimiterTokenFilterFactoryTests() { super("word_delimiter"); diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/XUniqueTokenFilterTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/XUniqueTokenFilterTests.java index 3db13ad793838..da0826effd2ae 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/XUniqueTokenFilterTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/XUniqueTokenFilterTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/analysis-common/src/test/resources/org/elasticsearch/analysis/common/de_DR.xml b/modules/analysis-common/src/test/resources/org/elasticsearch/analysis/common/de_DR.xml new file mode 100644 index 0000000000000..37bcde1246a81 --- /dev/null +++ b/modules/analysis-common/src/test/resources/org/elasticsearch/analysis/common/de_DR.xml @@ -0,0 +1,1130 @@ + + + + + + + + + + + + aA + bB + cC + dD + eE + fF + gG + hH + iI + jJ + kK + lL + mM + nN + oO + pP + qQ + rR + sS + tT + uU + vV + wW + xX + yY + zZ + �� + �� + �� + �� + � + + + + .aa6l .ab3a4s .ab3ei .abi2 .ab3it .ab1l .ab1r .ab3u .ad3o4r .alti6 + .ana3c .an5alg .an1e + .ang8s2t1 + .an1s .ap1p .ar6sc .ar6ta .ar6tei .as2z + .au2f1 .au2s3 .be5erb .be3na .ber6t5r .bie6r5 .bim6s5t .brot3 .bru6s + .ch6 .che6f5 .da8c .da2r .dar5in .dar5u .den6ka .de5r6en .des6pe + .de8spo .de3sz .dia3s4 .dien4 .dy2s1 .ehren5 .eine6 .ei6n5eh .ei8nen + .ein5sa .en6der .en6d5r .en3k4 .en8ta8 .en8tei .en4t3r .epo1 .er6ban + .er6b5ei .er6bla .er6d5um .er3ei .er5er .er3in .er3o4b .erwi5s .es1p + .es8t1l .es8t1n + .ex1a2 .ex3em .fal6sc .fe6st5a .flu4g3 .furch8 .ga6ner .ge3n4a + .ge5r� + .ges6 + .halb5 .halbe6 .hal6br .haup4 .hau4t .heima6 .he4r3e + .her6za .he5x .hin3 .hir8sc .ho4c .hu3sa .hy5o .ibe5 .ima6ge .in1 + .ini6 .is5chi .jagd5 .kal6k5o .ka6ph .ki4e .kop6f3 .kraf6 .k�5ra + .lab6br .liie6 .lo6s5k .l�4s3t .ma5d .mi2t1 .no6th .no6top + .obe8ri .ob1l .obs2 .ob6st5e .or3c .ort6s5e .ost3a .oste8r .pe4re + .pe3ts .ph6 .po8str .rau4m3 .re5an .ro8q .ru5the .r�5be + + .sch8 .se6e .se5n6h .se5ra .si2e .spi6ke .st4 .sy2n + .tages5 .tan6kl .ta8th .te6e .te8str .to6der .to8nin .to6we .um1 + .umpf4 .un1 .une6 .unge5n .ur1c .ur5en .ve6rin .vora8 .wah6l5 .we8ges + .we8s2t .wes3te + .wo6r .wor3a .wun4s .zi4e .zuch8 .�nde8re .�ch8 aa1c aa2gr + aal5e aa6r5a a5arti aa2s1t aat2s 6aba ab3art 1abdr 6abel aben6dr + ab5erk ab5err ab5esse 1abf 1abg 1abh� ab1ir 1abko a1bl ab1la + 5ablag a6bla� ab4ler ab1lu a8bl� 5a6bl� abma5c + 1abn ab1ra ab1re 5a6brec ab1ro + ab1s + ab8sk abs2z 3abtei ab1ur 1abw + 5abze 5abzu ab1�n ab�u8 a4ce. a5chal ach5art ach5au a1che + a8chent ach6er. a6ch5erf a1chi ach1l ach3m ach5n a1cho ach3re a1chu + ach1w a1chy ach5�f ack1o acks6t ack5sta a1d 8ad. a6d5ac ad3ant + ad8ar 5addi a8dein ade5o8 adi5en 1adj 1adle ad1op a2dre 3adres adt1 + 1adv a6d� a1e2d ae1r a1er. 1aero 8afa a3fal af1an a5far a5fat + af1au a6fentl a2f1ex af1fr af5rau af1re 1afri af6tent af6tra aft5re + a6f5um 8af� ag5abe 5a4gent ag8er ages5e 1aggr ag5las ag1lo a1gn + ag2ne 1agog a6g5und a1ha a1he ah5ein a4h3erh a1hi ahl1a ah1le ah4m3ar + ahn1a a5ho ahra6 ahr5ab ah1re ah8rei ahren8s ahre4s3 ahr8ti ah1ru a1hu + ah8� ai3d2s ai1e aif6 a3inse ai4re. a5isch. ais8e a3ismu ais6n + aiso6 a1j 1akad a4kade a1ke a1ki 1akko 5akro1 a5lal al5ans 3al8arm + al8beb al8berw alb5la 3album al1c a1le a6l5e6be a4l3ein a8lel a8lerb + a8lerh a6lert 5a6l5eth 1algi al4gli al3int al4lab al8lan al4l3ar + alle3g a1lo a4l5ob al6schm al4the + + al4t3re 8a1lu alu5i a6lur + alu3ta a1l� a6mate 8ame. 5a6meise am6m5ei am6mum am2n ampf3a + am6schw am2ta a1mu a1m� a3nac a1nad anadi5e an3ako an3alp 3analy + an3ame an3ara a1nas an5asti a1nat anat5s an8dent ande4s3 an1ec an5eis + an1e2k 4aner. a6n5erd a8nerf a6n5erke 1anfa 5anfert 1anf� 3angab + 5angebo an3gli ang6lis an2gn 3angri ang5t6 5anh� ani5g ani4ka + an5i8on an1kl an6kno an4kro 1anl anma5c anmar4 3annah anne4s3 a1no + 5a6n1o2d 5a6n3oma 5a6nord 1anr an1sa 5anschl an4soz an1st 5anstal + an1s2z 5antenn an1th 5anw� a5ny an4z3ed 5anzeig 5anzieh 3anzug + an1� 5an�s a1n� an�8d a1os a1pa 3apfel a2ph1t + aph5�6 a1pi 8apl apo1c apo1s + a6pos2t + a6poth 1appa ap1pr a1pr + a5p� a3p� a1ra a4r3af ar3all 3arbei 2arbt ar1c 2a1re ar3ein + ar2gl 2a1ri ari5es ar8kers ar6les ar4nan ar5o6ch ar1o2d a1rol ar3ony + a8ror a3ros ar5ox ar6schl 8artei ar6t5ri a1ru a1ry 1arzt arz1w + ar8z� ar�8m ar�6 ar5�m ar1�2 a1sa a6schec + asch5l asch3m a6schn a3s4hi as1pa asp5l + + as5tev 1asth + + a1str ast3re 8a1ta ata5c ata3la a6tapf ata5pl a1te a6teli aten5a + ate5ran 6atf 6atg a1th at3hal 1athl 2a1ti 5atlant 3atlas 8atmus 6atn + a1to a6t5ops ato6ra a6t5ort. 4a1tr a6t5ru at2t1h at5t6h� 6a1tu + atz1w a1t� a1t� au1a au6bre auch3a au1e aue4l 5aufent + 3auff� 3aufga 1aufn auf1t 3auftr 1aufw 3auge. au4kle aule8s 6aum + au8mar aum5p 1ausb 3ausd 1ausf 1ausg au8sin + + au4sta 1ausw 1ausz + aut5eng au1th 1auto au�e8 a1v ave5r6a aver6i a1w a6wes a1x + a2xia a6xio a1ya a1z azi5er. 8a� 1ba 8ba8del ba1la ba1na + ban6k5r ba5ot bardi6n ba1ro basten6 bau3sp 2b1b bb6le b2bli 2b1c 2b1d + 1be be1a be8at. be1ch 8becht 8becke. be5el be1en bee8rei be5eta bef2 + 8beff be1g2 beh�8 bei1s 6b5eisen bei3tr b8el bel8o belu3t be3nac + bend6o be6ners be6nerw be4nor ben4se6 bens5el be1n� be1n� + be1o2 b8er. be1ra be8rac ber8gab. ber1r be1r� bes8c bes5erh + bes2p be5tha bet5sc be1un be1ur 8bex be6zwec 2b1f8 + + 2b1g2 + bga2s5 bge1 2b1h bhole6 1bi bi1bl b6ie bi1el bi1la bil�5 bi1na + bi4nok + + bi6stu bi5tr bit4t5r b1j 2b1k2 bk�6 bl8 b6la. + 6b1lad 6blag 8blam 1blat b8latt 3blau. b6lav 3ble. b1leb b1led + 8b1leg 8b1leh 8bleid 8bleih 6b3lein + + ble4m3o 4blich b4lind + 8bling b2lio 5blit b4litz b1loh 8b1los 1blu 5blum 2blun blut3a blut5sc + 3bl� bl�s5c 5bl� 3bl� bl�8sc 2b1m 2b1n 1bo + bo1ch bo5d6s boe5 8boff 8bonk bo1ra b1ort 2b1p2 b1q 1br brail6 brast8 + bre4a b5red 8bref 8b5riem b6riga bro1s b1rup b2ruz 8br�h + br�s5c 8bs b1sa b8sang b2s1ar b1sc bs3erl bs3erz b8sof b1s2p + bst1h b3stru b5st� b6sun 2b1t b2t1h 1bu bu1ie bul6k b8ure bu6sin + 6b1v 2b1w 1by1 by6te. 8b1z + + 1b� b5�6s5 1b� + b6�5bere b�ge6 b�gel5e b�r6sc 1ca cag6 ca5la ca6re + ca5y c1c 1ce celi4c celich5 ce1ro c8h 2ch. 1chae ch1ah ch3akt cha6mer + 8chanz 5chara 3chari 5chato 6chb 1chef 6chei ch3eil ch3eis 6cherkl + 6chf 4chh 5chiad 5chias 6chins 8chj chl6 5chlor 6ch2m 2chn6 ch8nie + 5cho. 8chob choi8d 6chp ch3ren ch6res ch3r� 2chs 2cht cht5ha + cht3hi 5chthon ch6tin 6chuh chu4la 6ch3unt chut6t 8chw 1ci ci5tr c2k + 2ck. ck1ei 4ckh ck3l ck3n ck5o8f ck1r 2cks ck5stra ck6s5u c2l 1c8o + con6ne 8corb cos6t c3q 1c6r 8c1t 1cu 1cy 5c�1 c�5 1da. + 8daas 2dabg 8dabr 6dabt 6dabw 1dac da2gr 6d5alk 8d5amt dan6ce. + dani5er dan8ker 2danl danla6 6dans 8danzi 6danzu d1ap da2r1a8 2d1arb + d3arc dar6men 4d3art 8darz 1dat 8datm 2d1auf 2d1aus 2d1b 2d1c 2d1d + d5de d3d2h dd�mme8 1de 2deal de5an de3cha de1e defe6 6deff 2d1ehr + 5d4eic de5isc de8lar del6s5e del6spr de4mag de8mun de8nep dene6r + 8denge. 8dengen de5o6d 2deol de5ram 8derdb der5ein de1ro der1r d8ers + der5um de4s3am de4s3an de4sau de6sil de4sin de8sor de4spr de2su 8deul + de5us. 2d1f df2l 2d1g 2d1h 1di dia5c di5ara dice5 di3chr di5ena di1gn + di1la dil8s di1na 8dind 6dinf 4d3inh 2d1ins di5o6d di3p4t di8sen dis1p + di5s8per di6s5to + dis3tr + di8tan di8tin d1j 6dje 2dju 2d1k 2d1l 2d1m + 2d1n6 dni6 dnje6 1do 6d5obe do6berf 6d5ony do3ran 6dord 2d1org dor4t3h + + 6doth dott8e 2d1p d5q dr4 1drah 8drak d5rand 6dre. 4drech + d6reck 4d3reg 8d3reic d5reife 8drem 8d1ren 2drer 8dres. 6d5rh 1dria + d1ric 8drind droi6 dro5x 1dru 8drut dr�s5c 1dr� dr�5b + dr�8sc 2ds d1sa d6san dsat6 d1sc 5d6scha. 5dschik dse8e d8serg + 8dsl d1sp d4spak ds2po d8sp� d1st d1s� 2dt d1ta d1te d1ti + d1to dt1s6 d1tu d5t� 1du du5als du1b6 du1e duf4t3r 4d3uh du5ie + 8duml 8dumw 2d1und du8ni 6d5unt dur2c durch3 6durl 6dursa 8durt + dus1t + du8schr 2d1v 2d1w dwa8l 2d1z 1d� 6d�h 8d�nd d�6r + d�8bl d5�l d�r6fl d�8sc d5�4st + + 1d� ea4ben e1ac e1ah e1akt e1al. e5alf e1alg e5a8lin e1alk e1all + e5alp e1alt e5alw e1am e1and ea6nim e1ar. e5arf e1ark e5arm e3art + e5at. e6ate e6a5t6l e8ats e5att e6au. e1aus e1b e6b5am ebens5e + eb4lie eb4ser eb4s3in e1che e8cherz e1chi ech3m 8ech3n ech1r ech8send + ech4su e1chu eck5an e5cl e1d ee5a ee3e ee5g e1ei ee5isc eei4s3t + ee6lend e1ell ee5l� e1erd ee3r4e ee8reng eere6s5 ee5r� + ee6tat e1ex e1f e6fau e8fe8b 3effek ef3rom ege6ra eglo6si 1egy e1ha + e6h5ach eh5ans e6hap eh5auf e1he e1hi ehl3a eh1le ehl5ein eh1mu ehn5ec + e1ho ehr1a eh1re ehre6n eh1ri eh1ru ehr5um e1hu eh1w e1hy e1h� + e1h� e3h�t ei1a eia6s ei6bar eich3a eich5r ei4dar ei6d5ei + ei8derf ei3d4sc ei1e 8eifen 3eifri 1eign eil1d ei6mab ei8mag ein1a4 + ei8nat ei8nerh ei8ness ei6nete ein1g e8ini ein1k ei6n5od ei8nok ei4nor + e3ins� ei1o e1irr ei5ru ei8sab ei5schn ei6s5ent ei8sol ei4t3al + eit3ar eit1h ei6thi ei8tho eit8samt ei6t5um e1j 1ekd e1ke e1ki e1k2l + e1kn ekni4 e1la e2l1al 6elan e6lanf e8lanl e6l5ans el3arb el3arm + e6l3art 5e6lasti e6lauge elbst5a e1le 6elef ele6h e6l5ehe e8leif + e6l5einh 1elek e8lel 3eleme e6lemen e6lente el5epi e4l3err e6l5ersc + elf2l elg2 e6l5ins ell8er 4e1lo e4l3ofe el8soh el8tent 5eltern e1lu + elut2 e1l� e1l� em8dei em8meis 4emo emo5s 1emp1f 1empt 1emto + e1mu emurk4 emurks5 e1m� en5a6ben en5achs en5ack e1nad en5af + en5all en3alt en1am en3an. en3ant en3anz en1a6p en1ar en1a6s 6e1nat + en3auf en3aus en2ce enda6l end5erf end5erg en8dess 4ene. en5eck + e8neff e6n5ehr e6n5eim en3eis 6enem. 6enen e4nent 4ener. e8nerd + e6n3erf e4nerg 5energi e6n5erla en5ers e6nerst en5erw 6enes e6n5ess + e2nex en3glo 2eni enni6s5 ennos4 enns8 e1no e6nober eno8f en5opf + e4n3ord en8sers ens8kl en1sp ens6por en5t6ag enta5go en8terbu en6tid + 3entla ent5ric 5entwic 5entwu 1entz enu5i e3ny en8zan en1�f + e1n�s e1n�g eo1c e5o6fe e5okk e1on. e3onf e5onk e5onl e5onr + e5opf e5ops e5or. e1ord e1org eo5r6h eo1t e1pa e8pee e6p5e6g ep5ent + e1p2f e1pi 5epid e6pidem e1pl 5epos e6pos. ep4p3a e1pr e1p� e1q + e1ra. er5aal 8eraba e5rabel er5a6ben e5rabi er3abs er3ach era5e + era5k6l er3all er3amt e3rand e3rane er3ans e5ranz. e1rap er3arc + e3rari er3a6si e1rat erat3s er3auf e3raum 3erbse er1c e1re 4e5re. + er3eck er5egg er5e2h 2erei e3rei. e8reine er5einr 6eren. e4r3enm + 4erer. e6r5erm er5ero er5erst e4r3erz er3ess 5erf�l er8gan. + 5ergebn er2g5h 5erg�nz 5erh�hu 2e1ri eri5ak e6r5iat e4r3ind + e6r5i6n5i6 er5ins e6r5int er5itio er1kl 3erkl� 5erl�s. + ermen6s er6nab 3ernst 6e1ro. e1rod er1o2f e1rog 6e3roi ero8ide e3rol + e1rom e1ron e3rop8 e2r1or e1ros e1rot er5ox ersch4 5erstat er6t5ein + er2t1h er5t6her 2e1ru eruf4s3 e4r3uhr er3ums e5rus 5erwerb e1ry er5zwa + er3zwu er�8m er5�s er�8 e3r�s. e6r1�2b e1sa + esa8b e8sap e6s5a6v e1sc esch4l ese1a es5ebe eserve5 e8sh es5ill + es3int es4kop e2sl eso8b e1sp espei6s5 es2po es2pu 5essenz e6stabs + e6staf e6st5ak est3ar e8stob e1str est5res es3ur e2sz e1s� e1ta + et8ag etari5e eta8ta e1te eten6te et5hal e5thel e1ti 1etn e1to e1tr + et3rec e8tscha et8se et6tei et2th et2t1r e1tu etu1s et8zent et8zw + e1t� e1t� e1t� eu1a2 eu1e eue8rei eu5fe euin5 euk2 + e1um. eu6nio e5unter eu1o6 eu5p 3europ eu1sp eu5str eu8zo e1v eval6s + eve5r6en ever4i e1w e2wig ex1or 1exp 1extr ey3er. e1z e1�2 + e5�8 e1� e8�es fa6ch5i fade8 fa6del fa5el. + fal6lo falt8e fa1na fan4gr 6fanl 6fap far6ba far4bl far6r5a 2f1art + fa1sc fau8str fa3y 2f1b2 6f1c 2f1d 1fe 2f1eck fe6dr feh6lei f6eim + 8feins f5eis fel5en 8feltern 8femp fe5rant 4ferd. ferri8 fe8stof + fe6str fe6stum fe8tag fet6ta fex1 2ff f1fa f6f5arm f5fe ffe5in ffe6la + ffe8ler ff1f f1fla ff3lei ff4lie ff8sa ff6s5ta 2f1g2 fgewen6 4f1h 1fi + fid4 fi3ds fieb4 fi1la fi8lei fil4m5a f8in. fi1na 8finf fi8scho fi6u + 6f1j 2f1k2 f8lanz fl8e 4f3lein 8flib 4fling f2lix 6f3lon 5flop 1flor + 5f8l�c 3fl�t 2f1m 2f1n 1fo foh1 f2on fo6na 2f1op fo5ra + for8mei for8str for8th for6t5r fo5ru 6f5otte 2f1p8 f1q fr6 f5ram + 1f8ran f8ra� f8re. frei1 5frei. f3reic f3rest f1rib + 8f1ric 6frig 1fris fro8na fr�s5t 2fs f1sc f2s1er f5str + fs3t�t 2ft f1tak f1te ft5e6h ftere6 ft1h f1ti f5to f1tr ft5rad + ft1sc ft2so f1tu ftwi3d4 ft1z 1fu 6f5ums 6funf fun4ka fu8�end + 6f1v 2f1w 2f1z 1f� f�1c 8f�rm 6f�ug + f�8� f�de3 8f�f 3f�r 1f� + f�n4f3u 1ga ga6bl 6gabw 8gabz g3a4der ga8ho ga5isc 4gak ga1la + 6g5amt ga1na gan5erb gan6g5a ga5nj 6ganl 8gansc 6garb 2g1arc 2g1arm + ga5ro 6g3arti ga8sa ga8sc ga6stre 2g1atm 6g5auf gau5fr g5aus 2g1b g5c + 6gd g1da 1ge ge1a2 ge6an ge8at. ge1e2 ge6es gef2 8geff ge1g2l ge1im + 4g3eise geist5r gel8bra gelt8s ge5l� ge8nin gen3k 6g5entf + ge3n� ge1or ge1ra ge6rab ger8au 8gerh� ger8ins ge1ro 6g5erz. + ge1r� ge1r� ge1s ges2p + ge2s7te. ge2s7ten ge2s7ter ge2s7tik + ge5unt 4g3ex3 2g1f8 2g1g g1ha 6g1hei + 5ghel. g5henn 6g1hi g1ho 1ghr g1h� 1gi gi5la gi8me. gi1na + 4g3ins + gis1tr + g1j 2g1k 8gl. 1glad g5lag glan4z3 1glas 6glass 5glaub + g3lauf 1gle. g5leb 3gleic g3lein 5gleis 1glem 2gler 8g3leu gli8a + g2lie 3glied 1g2lik 1g2lim g6lio 1gloa 5glom 1glon 1glop g1los g4loss + g5luf 1g2ly 1gl� 2g1m gn8 6gn. 1gna 8gnach 2gnah g1nas g8neu + g2nie g3nis 1gno 8gnot 1go goe1 8gof 2gog 5gogr 6g5oh goni5e 6gonist + go1ra 8gord 2g1p2 g1q 1gr4 g5rahm gra8m gra4s3t 6g1rec gre6ge 4g3reic + g5reit 8grenn gri4e g5riem 5grif 2grig g5ring 6groh 2grot gro6� + 4grut 2gs gs1ab g5sah gs1ak gs1an gs8and gs1ar gs1au g1sc + gs1ef g5seil gs5ein g2s1er gs1in g2s1o gso2r gs1pr g2s1u 2g1t g3te + g2t1h 1gu gu5as gu2e 2gue. 6gued 4g3uh 8gums 6g5unt + + gut3h gu2tu + 4g1v 2g1w gy1n g1z 1g� 8g�8m 6g�rm 1g� 1g� + 6g�b 1haa hab8r ha8del hade4n 8hae ha5el. haf6tr 2hal. ha1la + hal4b5a 6hale 8han. ha1na han6dr han6ge. 2hani h5anth 6hanz 6harb + h3arbe h3arme ha5ro ha2t1h h1atm hau6san ha8� h1b2 h1c h1d + he2bl he3cho h3echt he5d6s 5heft h5e6he. hei8ds h1eif 2hein he3ism + he5ist. heit8s3 hek6ta hel8lau 8helt he6mer 1hemm 6h1emp hen5end + hen5klo hen6tri he2nu 8heo he8q her3ab he5rak her3an 4herap her3au + h3erbi he1ro he8ro8b he4r3um her6z5er he4spe he1st heta6 het5am he5th + heu3sc he1xa hey5e h1f2 h1g hgol8 h1h h1iat hie6r5i hi5kt hil1a2 + hil4fr hi5nak hin4ta hi2nu hi5ob hirn5e hir6ner hi1sp hi1th hi5tr + 5hitz h1j h6jo h1k2 hlabb4 hla4ga hla6gr h5lai hl8am h1las h1la� + hl1c h1led h3lein h5ler. h2lif h2lim h8linf hl5int h2lip + h2lit h4lor h3lose h1l�s hme5e h2nee h2nei hn3eig h2nel hne8n + hne4p3f hn8erz h6netz h2nip h2nit h1nol hn5sp h2nuc h2nud h2nul hoch1 + 1hoh hoh8lei 2hoi ho4l3ar 1holz h2on ho1ra 6horg 5horn. ho3sl hos1p + ho4spi h1p hpi6 h1q 6hr h1rai h8rank h5raum hr1c hrcre8 h1red h3reg + h8rei. h4r3erb h8rert hrg2 h1ric hr5ins h2rom hr6t5erl hr2t1h hr6t5ra + hr8tri h6rum hr1z hs3ach h6s5amt h1sc h6s5ec h6s5erl hs8erle h4sob + h1sp h8spa� h8spel hs6po h4spun h1str h4s3tum hs3und + h1s� h5ta. h5tab ht3ac ht1ak ht3ang h5tanz ht1ar ht1at h5taub + h1te h2t1ec ht3eff ht3ehe h4t3eif h8teim h4t3ein ht3eis h6temp h8tentf + hte8ren h6terf� h8tergr h4t3erh h6t5ersc h8terst h8tese h8tess + h2t1eu h4t3ex ht1he ht5hu h1ti ht5rak hts3ah ht1sc ht6sex ht8sk ht8so + h1tu htz8 h5t�m hub5l hu6b5r huh1l h5uhr. huld5a6 hu8lent + hu8l� h5up. h1v h5weib h3weis h1z h�8kl h�l8s + h�ma8tu8 h�8sche. h�t1s h�u4s3c 2h�. + 2h�e 8h�i h�6s h�s5c h�hne6 h�l4s3t + h�tte8re i5adn i1af i5ak. i1al. i1al1a i1alb i1ald i5alei i1alf + i1alg i3alh i1alk i1all i1alp i1alr i1als i1alt i1alv i5alw i3alz + i1an. ia5na i3and ian8e ia8ne8b i1ang i3ank i5ann i1ant i1anz i6apo + i1ar. ia6rab i5arr i1as. i1asm i1ass i5ast. i1at. i5ats i1au i5azz + i6b5eig i6b5eis ib2le i4blis i6brig i6b5unt i6b�b i1che ich5ei + i6cherb i1chi ich5ins ich1l ich3m ich1n i1cho icht5an icht3r i1chu + ich1w ick6s5te ic5l i1d id3arm 3ideal ide8na 3ideol ide5r� i6diot + id5rec id1t ie1a ie6b5ar iebe4s3 ie2bl ieb1r ie8bra ie4bre ie8b� + ie2dr ie1e8 ie6f5ad ief5f ie2f1l ie4fro ief1t i1ei ie4l3ec ie8lei + ie4lek i3ell i1en. i1end ien6e i3enf i5enn ien6ne. i1enp i1enr + i5ensa ien8stal i5env i1enz ie5o ier3a4b ie4rap i2ere ie4rec ie6r5ein + ie6r5eis ier8er i3ern. ie8rum ie8rund ie6s5che ie6tau ie8tert ie5the + ie6t5ri i1ett ie5un iex5 2if i1fa if5ang i6fau if1fr if5lac i5f6lie + i1fre ift5a if6t5r ig3art 2ige i8gess ig5he i5gla ig2ni i5go ig3rot + ig3s2p i1ha i8ham i8hans i1he i1hi ih1n ih1r i1hu i8hum ih1w 8i1i ii2s + ii2t i1j i1k i6kak i8kerz i6kes ik4ler i6k5unt 2il i5lac i1lag il3ans + i5las i1lau il6auf i1le ile8h i8lel il2fl il3ipp il6l5enn i1lo ilt8e + i1lu i1l� i8mart imb2 i8mele i8mid imme6l5a i1mu i1m� + i5m� ina5he i1nat in1au inau8s 8ind. in4d3an 5index ind2r 3indus + i5nec i2n1ei i8nerw 3infek 1info 5ingeni ing5s6o 5inhab ini5er. 5inj + in8k�t in8nan i1no inoi8d in3o4ku in5sau in1sp 5inspe 5instit + 5instru ins4ze 5intere 5interv in3the in5t2r i5ny in�2 i1n�r + in1�s in�8 in5�d i1n�s 2io io1a8 io1c iode4 io2di + ioi8 i1ol. i1om. i1on. i5onb ion2s1 i1ont i5ops i5o8pt i1or. + i3oral io3rat i5orc i1os. i1ot. i1o8x 2ip i1pa i1pi i1p2l i1pr i1q + i1ra ir6bl i1re i1ri ir8me8d ir2m1o2 ir8nak i1ro ir5rho ir6schl + ir6sch5r i5rus i5ry i5r� i1sa i8samt i6sar i2s1au i8scheh i8schei + isch5m isch3r isch�8 is8ele ise3ra i4s3erh is3err isi6de i8sind + is4kop ison5e is6por i8s5tum i5sty i5s� i1ta it5ab. i2t1a2m + i8tax i1te i8tersc i1thi i1tho i5thr it8h� i1ti i8ti8d iti6kl + itmen4 i1to i8tof it3ran it3rau i1tri itri5o it1sc it2se it5spa it8tru + i1tu it6z5erg it6z1w i1t� it�6r5e it�t2 it�ts5 + i1t� i1u iu6r 2i1v i6vad iva8tin i8vei i6v5ene i8verh i2vob i8vur + i1w iwi2 i5xa i1xe i1z ize8n i8zir i6z5w i�8m i1�6r + i5�t. i5�v i1�8 i�8 i6�5ers ja5la + je2t3r 6jm 5jo jo5as jo1ra jou6l ju5cha jugen4 jugend5 jung5s6 + + 3j� 1ka 8kachs 8kakz ka1la kal5d kam5t ka1na 2kanl 8kapf ka6pl + ka5r6a 6k3arbe ka1ro kar6p5f 4k3arti 8karz ka1r� kasi5e ka6teb + kat8ta kauf6s kau3t2 2k1b 2k1c 4k1d kehr6s kehrs5a 8keic 2k1eig 6k5ein + 6k5eis ke6lar ke8leis ke8lo 8kemp k5ente. k3entf 8k5ents 6kentz ke1ra + k5erlau 2k1f8 2k1g 2k1h ki5fl 8kik king6s5 6kinh ki5os ki5sp ki5th + 8ki8� 2k1k2 kl8 1kla 8klac k5lager kle4br k3leib 3kleid kle5isc + 4k3leit k3lek 6k5ler. 5klet 2klic 8klig k2lim k2lin 5klip 5klop k3lor + 1kl� 2k1m kmani5e kn8 6kner k2ni kn�8 1k2o ko1a2 ko6de. + ko1i koi8t ko6min ko1op ko1or ko6pht ko3ra kor6d5er ko5ru ko5t6sc k3ou + 3kow 6k5ox 2k1p2 k1q 1kr8 4k3rad 2k1rec 4k3reic kre5ie 2krib 6krig + 2krip 6kroba 2ks k1sa k6sab ksal8s k8samt k6san k1sc k2s1ex k5spat + k5spe k8spil ks6por k1spr kst8 k2s1uf 2k1t kta8l kt5a6re k8tein kte8re + k2t1h k8tinf kt3rec kt1s 1ku ku1ch kuck8 k3uhr ku5ie kum2s1 kunfts5 + kun2s kunst3 ku8rau ku4ro kurz1 + + 4kusti ku1ta ku8� + 6k1v 2k1w ky5n 2k1z 1k� k�4m 4k3�mi k�se5 1k� + k�1c k�1s 1k� k�1c k�r6sc + + 1la. + 8labf 8labh lab2r 2l1abs lach3r la8dr 5ladu 8ladv 6laff laf5t la2gn + 5laken 8lamb la6mer 5lampe. 2l1amt la1na 1land lan4d3a lan4d3r lan4gr + 8lanme 6lann 8lanw 6lan� 8lappa lap8pl lap6pr l8ar. la5ra lar4af + la8rag la8ran la6r5a6s l3arbe la8rei 6larm. la8sa la1sc la8sta lat8i + 6l5atm 4lauss 4lauto 1law 2lb l8bab l8bauf l8bede l4b3ins l5blo + lbst5an lbst3e 8lc l1che l8chert l1chi lch3m l5cho lch5w 6ld l4d3ei + ld1re l6d�b le2bl le8bre lecht6s5 led2r 6leff le4gas 1lehr lei6br + le8inf 8leinn 5leistu 4lektr le6l5ers lemo2 8lemp l8en. 8lends + 6lendun le8nend len8erw 6l5ents 4l3entw 4lentz 8lenzy 8leoz 6lepi + le6pip 8lepo 1ler l6er. 8lerbs 6l5erde le8reis le8rend le4r3er 4l3erg + l8ergr 6lerkl 6l5erzie 8ler� 8lesel lesi5e le3sko le3tha let1s + 5leuc 4leuro leu4s3t le5xe 6lexp l1f 2l1g lgend8 l8gh lglie3 lglied6 + 6l1h 1li li1ar li1as 2lick li8dr li1en lien6n li8ers li8ert 2lie� + 3lig li8ga8b li1g6n li1l8a 8limb li1na 4l3indu lings5 + 4l3inh 6linj link4s3 4linkt 2lint 8linv + + 4lipp 5lipt 4lisam + livi5e 6l1j 6l1k l8keim l8kj lk2l lko8f lkor8 lk2sa lk2se 6ll l1la + ll3a4be l8labt ll8anl ll1b ll1c ll1d6 l1le l4l3eim l6l5eise ller3a + l4leti l5lip l1lo ll3ort ll5ov ll6spr llte8 l1lu ll3urg l1l� + l5l� l6l�b 2l1m l6m5o6d 6ln l1na l1no 8lobl lo6br 3loch. + l5o4fen 5loge. 5lohn 4l3ohr 1lok l2on 4l3o4per lo1ra 2l1ord 6lorg + 4lort lo1ru 1los. lo8sei 3losig lo6ve lowi5 6l1p lp2f l8pho l8pn + lp4s3te l2pt l1q 8l1r 2ls l1sa l6sarm l1sc l8sec l6s5erg l4s3ers l8sh + l5s6la l1sp ls4por ls2pu l1str l8suni l1s� 2l1t lt5amp l4t3ein + l5ten l6t5eng l6t5erp l4t3hei lt3her l2t1ho l6t5i6b lti1l l8tr� + lt1sc lt6ser lt4s3o lt5ums lu8br lu2dr lu1en8 8lu8fe luft3a luf8tr + lu6g5r 2luh l1uhr lu5it 5luk 2l1umf 2l1umw 1lun 6l5u6nio 4l3unte lu5ol + 4lurg 6lurs l3urt lu4sto + lus1tr + lu6st5re lu8su lu6tal lu6t5e6g lu8terg + lu3the lu6t5or lu2t1r lu6�5 l1v lve5r6u 2l1w 1ly lya6 + 6lymp ly1no l8zess l8zo8f l3zwei lz5wu 3l�nd l�5on + l�6sc l�t1s 5l�uf 2l�ug l�u6s5c l�5v + l1�l 1l�s l�1�6t 6l1�be 1ma + 8mabg ma5chan mad2 ma5el 4magg mag8n ma1la ma8lau mal5d 8malde mali5e + malu8 ma8lut 2m1amp 3man mand2 man3ds 8mangr mani5o 8m5anst 6mappa + 4m3arbe mar8kr ma1r4o mar8schm 3mas ma1sc ma1t� 4m5auf ma5yo 2m1b + mb6r 2m1c 2m1d md6s� 1me me1ch me5isc 5meld mel8sa 8memp me5nal + men4dr men8schl men8schw 8mentsp me1ra mer4gl me1ro 3mes me6s5ei me1th + me8� 2m1f6 2m1g 2m1h 1mi mi1a mi6ale mi1la 2m1imm mi1na + mi5n� mi4s3an mit1h mi5t6ra 3mitt mitta8 mi6�5 6mj + 2m1k8 2m1l 2m1m m6mad m6m5ak m8menth m8mentw mme6ra m2mn mm5sp mm5ums + mmut5s m8m�n m1n8 m5ni 1mo mo5ar mo4dr 8mof mo8gal mo4kla mol5d + m2on mon8do mo4n3od + mon2s1tr + mont8a 6m5ony mopa6 mo1ra mor8d5a mo1sc mo1sp 5mot + moy5 2mp m1pa mpfa6 mpf3l mphe6 m1pi mpin6 m1pl mp2li m2plu mpo8ste + m1pr mpr�5 mp8th mput6 mpu5ts m1p� 8m1q 2m1r 2ms ms5au m1sc + msch4l ms6po m3spri m1str 2m1t mt1ar m8tein m2t1h mt6se mt8s� + mu5e 6m5uh mumi1 1mun mun6dr muse5e mu1ta 2m1v mvol2 mvoll3 2m1w 1my + 2m1z m�6kl 1m�n m�1s m�5tr m�u4s3c 3m�� + m�b2 6m�l 1m� 5m�n 3m�t 1na. + n5ab. 8nabn n1abs n1abz na6b� na2c nach3e 3nacht 1nae na5el + n1afr 1nag 1n2ah na8ha na8ho 1nai 6nair na4kol n1akt nal1a 8naly 1nama + na4mer na1mn n1amp 8n1amt 5nanc nan6ce n1and n6and. 2n1ang 1nani + 1nann n1ans 8nanw 5napf. 1n2ar. na2ra 2n1arc n8ard 1nari n8ark + 6n1arm 5n6ars 2n1art n8arv 6natm nat6s5e 1naue 4nauf n3aug 5naui n5auk + na5um 6nausb 6nauto 1nav 2nax 3naz 1na� n1b2 nbau5s n1c + nche5e nch5m 2n1d nda8d n2d1ak nd5ans n2d1ei nde8lac ndel6sa n8derhi + nde4se nde8stal n2dj ndnis5 n6d5or6t nd3rec nd3rot nd8samt nd6sau + ndt1h n8dumd 1ne ne5as ne2bl 6n5ebn 2nec 5neei ne5en ne1g4l 2negy + 4n1ein 8neis 4n3e4lem 8nemb 2n1emp nen1a 6n5energ nen3k 8nentb + 4n3en3th 8nentl 8n5entn 8n5ents ne1ra ne5r8al ne8ras 8nerbi 6n5erde. + nere5i6d nerfor6 6n5erh� 8nerl� 2n1err n8ers. 6n5ertra + 2n1erz nesi3e net1h neu4ra neu5sc 8neu� n1f nf5f nf2l + nflei8 nf5lin nft8st n8g5ac ng5d ng8en nge8ram ngg2 ng1h n6glic ng3rip + ng8ru ng2se4 ng2si n2g1um n1gy n8g�l n1h nhe6r5e 1ni ni1bl + ni5ch� ni8dee n6ie ni1en nie6s5te niet5h ni8etn 4n3i6gel n6ik + ni1la 2n1imp ni5na 2n1ind 8ninf 6n5inh ni8nit 6n5inn 2n1ins 4n1int + n6is + nis1tr + ni1th ni1tr n1j n6ji n8kad nk5ans n1ke n8kerla n1ki nk5inh + n5kl� n1k2n n8k5not nk3rot n8kr� nk5spo nk6t5r n8kuh + n6k�b n5l6 nli4mi n1m nmen4s n1na n8nerg nni5o n1no nn4t3ak nnt1h + nnu1e n1ny n1n� n1n� n1n� no5a no4b3la 4n3obs 2nobt + noche8 no6die no4dis no8ia no5isc 6n5o6leu no4mal noni6er 2n1onk n1ony + 4n3o4per 6nopf 6nopti no3ra no4ram nor6da 4n1org 2n1ort n6os no1st + 8nost. no8tan no8ter noty6pe 6n5ox n1p2 n1q n1r nr�s3 6ns n1sac + ns3ang n1sc n8self n8s5erf n8serg n6serk ns5erw n8sint n1s2pe n1spr + n6s5tat. + + n6stob n1str n1ta n4t3a4go nt5anh nt3ark nt3art + n1te nt3eis nte5n6ar nte8nei nter3a nte6rei nt1ha nt6har n3ther nt5hie + n3thus n1ti nti1c n8tinh nti1t ntlo6b ntmen8 n1to nt3o4ti n1tr ntra5f + ntra5ut nt8rea nt3rec nt8rep n4t3rin nt8rop n4t3rot n4tr� nt1s + nts6an nt2sk n1tu nt1z n1t� n1t� n8t�l n1t� 1nu + nu1a nu5el nu5en 4n1uhr nu5ie 8numl 6n5ums 6n5umw 2n1und 6nuni 6n5unr + 2n1unt 2nup 2nu6r n5uri nu3skr nu5ta n1v 8n1w 1nys n1za n6zab n2z1ar + n6zaus nzi4ga n8zof n6z5unt n1zw n6zwir 1n�c 5n�e 5n�i + n8�l n�6m n�6re n5�rz 5n�us n1�l + 1n�t n5�z 5n�. 6n1�2b 5n�� + o5ab. oa2l o8ala o1a2m o1an ob1ac obe4ra o6berh 5o4bers o4beru + obe6ser 1obj o1bl o2bli ob5sk 3obst. ob8sta obst5re ob5sz o1che + oche8b o8chec o3chi och1l och3m ocho8f o3chro och3to o3chu och1w o1d + o2d1ag od2dr ode5i ode6n5e od1tr o5e6b o5e6der. oe8du o1ef o1e2l + o1e2p o1er. o5e8x o1fa of8fan 1offi of8fin of6f5la o5fla o1fr 8o1g + og2n o1ha o1he o6h5eis o1hi ohl1a oh1le oh4l3er 5ohm. oh2ni o1ho + oh1re oh1ru o1hu oh1w o1hy o1h� o5ia o1id. o8idi oi8dr o5ids + o5isch. oiset6 o1ism o3ist. o5i6tu o1j o1k ok2l ok3lau o8kl� + 1okta o1la old5am old5r o1le ole5in ole1r ole3u ol6gl ol2kl olk4s1 + ol8lak ol8lauf. ol6lel ol8less o1lo + ol1s ol2ster + ol6sk o1lu oly1e2 5olym + o2mab om6an o8mau ombe4 o8merz om5sp o1mu o8munt o1m� o1m� + o1na ona8m on1ax on8ent o6n5erb 8oni oni5er. on1k on6n5a6b o1no ono1c + o4nokt 1ons onts8 o1n� oo8f 1oog oo2pe oo2sa o1pa 3o4pera o3pfli + opf3lo opf3r o1pi o1pl o2pli o5p6n op8pa op6pl o1pr o3p4ter 1opti + o1p� o5p� o1q o1ra. o3rad o8radd 1oram o6rang o5ras o8rauf + or5cha or4d3a4m or8dei or8deu 1ordn or4dos o1re o5re. ore2h o8r5ein + ore5isc or6enn or8fla or8fli 1orga 5orgel. or2gl o1ri 5o6rient or8nan + or8n� o1ro or1r2h or6t5an or8tau or8tere o1rus o1ry o1r� + or1�2 o1sa osa3i 6ose o8serk o1sk o6ske o6ski os2kl os2ko os2kr + osni5e o2s1o2d o3s4per o4stam o6stau o3stra ost3re osu6 o6s5ur o5s6ze + o1ta ot3auf o6taus o1te o6terw o1th othe5u o2th1r o1ti o1to oto1a + ot1re o1tri o1tro ot1sc o3tsu ot6t5erg ot2t3h ot2t5r ot8t� o1tu + ou3e ouf1 ou5f6l o5u6gr ou5ie ou6rar ou1t6a o1v o1wa o1we o6wer. o1wi + owid6 o1wo o5wu o1xe oy5al. oy1e oy1i o5yo o1z oza2r 1o2zea ozo3is + o�8 o�5elt o�1t 3paa pa6ce 5pad pag2 1pak + pa1la pa8na8t pani5el pa4nor pan1s2 1pap pap8s pa8rei par8kr paro8n + par5o6ti part8e 5partei 3partn pas6sep pa4tha 1pau 6paug pau3sc p1b + 8p5c 4p1d 1pe 4peic pe5isc 2pek pen3k pen8to8 p8er pe1ra pere6 per5ea + per5eb pe4rem 2perr per8ran 3pers 4persi pe3r� pe4sta pet2s + p2f1ec p4fei pf1f pf2l 5pflanz pf8leg pf3lei 2pft pf3ta p1g 1ph 2ph. + 2p1haf 6phb 8phd 6p5heit ph5eme 6phg phi6e 8phk 6phn p5holl pht2 + ph3tha 4ph3the phu6 6phz pi1en pi5err pi1la pi1na 5pinse pioni8e 1pis + pi1s2k pi1th p1k pl8 5pla p2lau 4plei p3lein 2pler 6p5les 2plig p6lik + 6p5ling p2liz plo8min 6p1m p1n 1p2o 8poh 5pol po8lan poly1 po3ny po1ra + 2porn por4t3h po5r� 5poti p1pa p6p5ei ppe6la pp5f p2p1h p1pi pp1l + ppp6 pp5ren + pp1s pp2ste + p5p� pr6 3preis 1pres 2p3rig 5prinz 1prob 1prod + 5prog pro8pt pro6t5a prote5i 8pro� pr�3l 1pr�s + pr�te4 1pr�f p5schl 2pst 1p2sy p1t p8to8d pt1s 5p6ty 1pu + pu1b2 2puc pu2dr puf8fr 6p5uh pun8s pu8rei pu5s6h pu1ta p1v p3w 5py + py5l p1z p�6der p5�6m p�8nu 8p�r p�t5h + p�t1s qu6 1qui 8rabk ra6bla 3rable ra2br r1abt 6rabz ra4dan ra2dr + 5rafal ra4f3er ra5gla ra2g3n 6raha ral5am 5rald 4ralg ra8lins 2rall + ral5t 8ramei r3anal r6and ran8der ran4dr 8ranf 6ranga 5rangi ran8gli + r3angr rans5pa 8ranw r8anz. ra5or 6rapf ra5pl rap6s5er 2r1arb 1rarh + r1arm ra5ro 2r1art 6r1arz ra8tei ra6t5he 6ratl ra4t3ro r5atta raue4n + 6raus. r5austa rau8tel raut5s ray1 r1b rb5lass r6bler rb4lie rbon6n + r8brecht rb6s5t� r8ces r1che rch1l rch3m rch3re rch3tr rch1w 8rd + r1da r8dachs r8dap rda5ro rde5ins rdio5 r8dir rd3ost r1dr r8drau 1re. + re1ak 3reakt re3als re6am. re1as 4reben re6bl rech5a r8edi re3er + 8reff 3refl 2reh 5reha r4ei. reich6s5 8reier 6reign re5imp 4r3eina + 6r3einb 6reing 6r5einn 6reinr 4r3eins r3eint reli3e 8r5elt 6rempf + 2remt ren5a6b ren8gl r3enni 1reno 5rente 4r3enth 8rentl 4r3entw 8rentz + ren4zw re1on requi5 1rer rer4bl 6rerbs 4r3erd 8rerh� 8rerkl + 4r3erla 8rerl� 4r3erns 6r5ern� rer5o 6r5erreg r5ertr r5erwec + r5er� re2sa re8schm 2ress re5u8ni 6rewo 2r1ex r1f r8ferd rf4lie + 8r1g r8gah rge4bl rge5na rgest4 rg6ne r2gni2 r8gob r4g3ret rg8sel r1h8 + r2hy 5rhyt ri1ar ri5cha rid2g r2ie rieg4s5 ri8ei ri1el ri6ele ri1en + ri3er. ri5ers. ri6fan ri8fer ri8fr 1r2ig ri8kn ri5la rim�8 + ri1na r8inde rin4ga rin6gr 1rinn 6rinner rino1 r8insp 4rinst + ri1n� ri5o6ch ri1o2d ri3o6st 2r1ir r2is ri3sko ri8spr + + ri5sv r2it 6r5i6tal ri5tr ri6ve. 8r1j 6rk r1ke rkehrs5 r1ki r3klin + r1k2n rk3str rk4t3an rk6to r6kuh rk�4s3t r1l r5li rline5a 6r1m + r6manl rma4p r4m3aph r8minf r8mob rm5sa 2rn r1na rna8be r5ne rn2ei + r6neif r6nex r6nh rn1k r1no r6n5oc rn1sp r1n� r1n� ro6bern + 6robs ro1ch 3rock. ro5de ro1e 4rofe ro8hert 1rohr ro5id ro1in ro5isc + 6rolym r2on 6roog ro6phan r3ort ro1s2p ro5s6w ro4tau ro1tr ro6ts 5rout + r1p rpe8re rp2f r2ps r2pt r1q 2rr r1ra r1re rrer6 + rr6hos r5rh� + r1ri r1ro rro8f rr8or rror5a r1ru r3ry r1r� r1r� r1r� + 2r1s + r2ste r2sti + r6sab r4sanf rse6e rse5na r2sh r6ska r6ski rs2kl r8sko r2sl rs2p + r6stauf r8sterw r8stran rswi3d4 r2sz 2r1t rt3art r8taut r5tei rt5eige + r8tepe r4t3erh r8terla r4t3hei r5t6hu r4t3int rt5reif rt1sc rt6ser + rt6s5o rt6s5u rt5und r8turt rube6 ru1en 1r4uf ruf4st ru1ie 2r1umg + 2r1uml 2rums run8der run4d5r 6rundz 6runf 8runs 2r1unt 2r1ur r6us + ru6sta + rus1tr + ru6tr 1ruts r1v rven1 rvi2c r1w r1x r1za rz5ac r6z5al + r8z1ar r8zerd r6z5erf rz8erh rz4t3h r8zum r�4ste r�u8sc + r1�f 5r�hr r�5le 3r�ll 5r�mis r1�r + r�2sc 3r�mp 1sa. 1saa s3a4ben sa2bl 2s1abs 6s1abt 6sabw + 3sack. 6s3a4der 1saf sa1fa 4s1aff sa5fr 1sag 1sai sa1i2k1 4s1akt 1sal + sa1la 4s3alpi 6salter salz3a 1sam s5anb san2c 1sand s5angeh 6sanl + 2s1ans 6s3antr 8s1anw s1ap s6aph 8sapo sap5p6 s8ar. 2s1arb 3sarg + s1arm sa5ro 2s1art 6s1arz 1sas 1sat sat8a 2s1atl sa8tom 3s8aue s5auff + sau5i s6aur 2s1aus 5s6ause 2s1b2 2sca s4ce 8sch. 3scha. 5schade + 3schaf 3schal sch5ame 8schanc 8schb 1sche 6schef 8schex 2schf 2schg + 2schh 1schi 2schk 5schlag 5schlu 6schm�� + 6schna� 1scho 6schord 6schp 3schri 8schric 8schrig + 8schrou 6schs 2scht sch3ta sch3tr 1schu 8schunt 6schv 2schz 5sch� + 5sch� 2sco scre6 6scu 2s1d 1se se5an se1ap se6ben se5ec see5i6g + se3erl 8seff se6han se8hi se8h� 6s5eid. 2s1eig s8eil 5sein. + sei5n6e 6s5einh 3s8eit 3sel. se4lar selb4 6s3e4lem se8lerl 2s1emp + sen3ac se5nec 6s5ents 4sentz s8er. se8reim ser5inn 8serm� + 8s5erzi 6ser�f se1um 8sexa 6sexp 2s1f2 sfal8ler 2s3g2 sge5b2 s1h + s8hew 5s6hip 5s4hop 1si 2siat si1b sicht6s 6s5i6dee siege6s5 si1en + si5err si1f2 si1g2n si6g5r si8kau sik1i si4kin si2kl si8k� si1la + sil6br si1na 2s1inf sin5gh 2s1inh sinne6s5 2s1ins si5ru si5str 4s1j + s1k2 6sk. 2skau skel6c skelch5 s6kele 1s2ki. 3s4kin. s6kiz s8kj + 6skn 2skow 3skrib 3skrip 2sku 8sk� s1l s8lal slei3t s4low 2s1m + s1n 6sna 6snot 1so so1ch 2s1odo so4dor 6s5o4fen solo3 s2on so5of 4sope + so1ra 2s1ord 4sorga sou5c so3un 4s3ox sp2 8spaa 5spal 1span 2spap + s2pec s4peis 1spek s6perg 4spers s6pes 2s1pf 8sphi 1s2ph� 1spi + spi4e 6s5pig 6spinse 2spis 2spla 2spol 5s6pom 6s5pos 6spoti 1spra + 3s8prec 6spreis 5spring 6sprob 1spru s2pul 1s2pur 6spy 5sp�n + 1sp� s1q 2s1r + + + 2ssa 2sse 2ssi 2sso 2ss� 2ss� 2ss� 2s1sch + sse8nu ssini6s ssoi6r 2st. + 1sta 4stafe 2stag + sta3la 6stale + 4s2talg + 8stalk 8stamt 6st5anf 4stans 6stanw 6starb sta4te + 6staus 2stb 6stc 6std + s1te + 4steil + + 6steppi + + 8stesse 6stf 2stg 2sth st1ha st3hei s8t1hi st1ho st5hu + s1ti + s2ti4el + 4s2tigm + + 6s2tind + 4s2tinf + s2ti8r + 2stk 2stl 2stm + 1sto 6stoll. 4st3ope + 6stopf. 6stord 6stp + + 4strai + s3tral + 6s5traum 3stra� + 3strec 6s3tref 8streib 5streif 6streno 6stres 6strev + + 2st5rig + + 8s2t1ris + + s8troma st5rose 4struf 3strum + 6str�g 2st1s6 2stt + 1stu stu5a 4stuc 2stue 8stun. 2stv 2stw s2tyl + 6stz 1st� 8st�g + 1st� + 1st� 8st�ch 4st�r. + 1su su2b1 3suc su1e su2fe su8mar 6sumfa 8sumk 2s1unt sup1p2 6s5u6ran + 6surte 2s1v 2s1w 1sy 8syl. sy5la syn1 sy2na syne4 s1z s4zend 5s6zene. + 8szu 1s� 6s5�nd 6s�ugi 6s�u� + 5s�m 2s1�2b 1s�c s�8di 1s�n 5s�� + taats3 4tab. taba6k ta8ban tab2l ta6bre 4tabs t3absc + 8tabz 6t3acht ta6der 6tadr tad6s tad2t 1tafe4 1tag ta6ga6 ta8gei + tage4s tag6s5t tah8 tahl3 tai6ne. ta5ir. tak8ta tal3au 1tale ta8leng + tal5ert 6t5a6mer 6tamp tampe6 2t1amt tan5d6a tan8dr tands5a tani5e + 6tanl 2tanr t3ans 8t5antr tanu6 t5anw 8tanwa tan8zw ta8rau 6tarbe + 1tari 2tark 2t1arm ta1ro 2tart t3arti 6tarz ta1sc ta6sien ta8stem + ta8sto t5aufb 4taufn 8taus. 5tause 8tausf 6tausg t5ausl 2t1b2 2t1c + t6chu 2t1d te2am tea4s te8ben 5techn 4teff te4g3re te6hau 2tehe te4hel + 2t1ehr te5id. teig5l 6teign tei8gr 1teil 4teinh t5einhe 4teis t5eisen + 8teiw te8lam te4lar 4telek 8telem te6man te6n5ag ten8erw ten5k tens4p + ten8tro 4t3entw 8tentz te6pli 5teppi ter5a6b te3ral ter5au 8terbar + t5erbe. 6terben 8terbs 4t3erbt t5erde. ter5ebe ter5ein te8rers terf4 + 8terh� 6terkl� ter8nor ter6re. t8erscha t5e6sel te8stau + t3euro te1xa tex3e 8texp tex6ta 2t1f2 2t1g2 2th. th6a 5tha. 2thaa + 6t1hab 6t5haf t5hah 8thak 3thal. 6thals 6t3hand 2t1hau 1the. 3t4hea + t1heb t5heil t3heit t3helf 1theo 5therap 5therf 6t5herz 1thes 1thet + 5thi. 2t1hil t3him 8thir 3this t5hj 2th1l 2th1m th1n t5hob t5hof + 4tholz 6thopti 1thr6 4ths t1hum 1thy 4t1h� 2t1h� t1h� + ti1a2m ti1b tie6fer ti1en ti8gerz tig3l ti8kin ti5lat 1tilg t1ind + tin4k3l ti3spa ti5str 5tite ti5tr ti8vel ti8vr 2t1j 2t1k2 2t1l tl8a + 2t1m8 2t1n 3tobe 8tobj to3cha 5tocht 8tock tode4 to8del to8du to1e + 6t5o6fen to1in toi6r 5toll. to8mene t2ons 2t1ony to4per 5topf. 6topt + to1ra + to1s to2ste + to6ska tos2l 2toti to1tr t8ou 2t1p2 6t1q tr6 tra5cha + tra8far traf5t 1trag tra6gl tra6gr t3rahm 1trai t6rans tra3sc tra6st + 3traue t4re. 2trec t3rech t8reck 6t1red t8ree 4t1reg 3treib 4treif + 8t3reis 8trepo tre6t5r t3rev 4t3rez 1trib t6rick tri6er 2trig t8rink + tri6o5d trizi5 tro1a 3troc trocke6 troi8d tro8man. tro3ny 5tropf + 6t5rosa t5ro� 5trub 5trup trut5 1tr�g 6t1r�h + 5tr�b tr�3bu t1r�c t1r�s 2ts ts1ab t1sac tsa8d + ts1ak t6s5alt ts1an ts1ar ts3auf t3schr t5sch� tse6e tsee5i + tsein6s ts3ent ts1er t8serf t4serk t8sh 5t6sik t4s3int ts5ort. + t5s6por t6sprei + t1st t2ste + t6s5tanz ts1th t6stit t4s3tor 1t2sua t2s1uf + t8sum. t2s1u8n t2s1ur 2t1t tt5eif tte6sa tt1ha tt8ret tt1sc tt8ser + tt5s6z 1tuc tuch5a 1tu1e 6tuh t5uhr tu1i tu6it 1tumh 6t5umr 1tums + 8tumt 6tund 6tunf 2t1unt tu5ra tu6rau tu6re. tu4r3er 2t1v 2t1w 1ty1 + ty6a ty8la 8tym 6ty6o 2tz tz5al tz1an tz1ar t8zec tzeh6 tzehn5 t6z5ei. + t6zor t4z3um t6z�u 5t�g 6t�h t5�lt t8�n + t�re8 8t�8st 6t�u� t5�ffen + 8t�8k 1t�n 4t�b t6�5ber. 5t�ch 1t�r. + u3al. u5alb u5alf u3alh u5alk u3alp u3an. ua5na u3and u5ans u5ar. + ua6th u1au ua1y u2bab ubi5er. u6b5rit ubs2k u5b� u8b�b 2uc + u1che u6ch5ec u1chi uch1l uch3m uch5n uch1r uch5to ucht5re u1chu uch1w + uck1a uck5in u1d ud4a u1ei u6ela uene8 u6ep u1er uer1a ue8rerl uer5o + u8esc u2est u8ev u1fa u2f1ei u4f3ent u8ferh uf1fr uf1l uf1ra uf1re + uf1r� uf1r� uf1s2p uf1st uft1s u8gabt u8gad u6gap ugeb8 u8gn + ugo3s4 u1ha u1he u1hi uh1le u1ho uh1re u1hu uh1w u1h� u1h� + 6ui ui5en u1ig u3ins uin8tes u5isch. u1j 6uk u1ke u1ki u1kl u8klu + u1k6n u5ky u1la uld8se u1le ul8lac ul6lau ul6le6l ul6lo ulni8 u1lo + ulo6i ult6a ult8e u1lu ul2vr u1l� u1l� 3umfan 5umlau umo8f + um8pho u1mu umu8s u5m� u1n1a un2al un6at unau2 6und. 5undein + un4d3um 3undzw und�8 un8d�b une2b un1ec une2h un3eis 3unfal + 1unf� 5ungea 3ungl� ung2s1 un8g� 1u2nif un4it un8kro + unk5s u1no unpa2 uns2p unvol4 unvoll5 u5os. u1pa u1pi u1p2l u1pr + up4s3t up2t1a u1q u1ra ur5abs ura8d ur5ah u6rak ur3alt u6rana u6r5ans + u8rap ur5a6ri u8ratt u1re ur3eig ur8gri u1ri ur5ins 3urlau urmen6 + ur8nan u1ro 3ursac ur8sau ur8sei ur4sk 3urtei u1ru uru5i6 uru6r u1ry + ur2za ur6z� ur5�6m u5r� u1r� ur�ck3 u1sa + usa4gi u2s1ar u2s1au u8schec usch5wi u2s1ei use8kel u8sl u4st3a4b + us3tau + + u2s1uf u8surn ut1ac u1tal uta8m u1tan ut1ar u1tas ut1au + u1te u8teic u4tent u8terf u6terin u4t3hei ut5ho ut1hu u1ti utine5 + uti6q u1to uto5c u1tr ut1sa ut1s6p ut6stro u1tu utz5w u1u u1v uve5n + uve3r4� u1w u1xe u5ya uy5e6 u1yi u2z1eh u8zerh u5� u�e6n + u�en5e 8vanb 6vang 6varb var8d va6t5a va8tei + va2t1r 2v1b 6v5c 6vd 1ve 6ve5g6 ver1 ver5b verb8l ve2re2 verg8 ve2ru8 + ve1s ve2s3p ve3xe 2v1f 2v1g 6v5h vi6el vie6w5 vi1g4 vi8leh vil6le. + 8vint vi1ru vi1tr 2v1k 2v1l 2v1m 4v5n 8vo8f voi6le vol8lend vol8li + v2or1 vo2re vo8rin vo2ro 2v1p 8vra v6re + 2v2s + 2v1t 2v1v 4v3w 2v1z + waffe8 wa6g5n 1wah wah8n wa5la wal8din wal6ta wan4dr 5ware wa8ru + war4za 1was w5c w1d 5wech we6fl 1weg we8geng weg5h weg3l we2g1r + weh6r5er 5weise weit3r wel2t welt3r we6rat 8werc 5werdu wer4fl 5werk. + wer4ka wer8ku wer4ta wer8term we2sp + we8s4tend + + we8str + we8st� wet8ta wich6s5t 1wid wi2dr wiede4 wieder5 wik6 wim6ma + win4d3r 5wirt wisch5l 1wj 6wk 2w1l 8w1n wo1c woche6 wol6f wor6t5r 6ws2 + w1sk 6w5t 5wunde. wun6gr wu1sc wu2t1 6w5w wy5a w�rme5 w�1sc + 1xag x1ak x3a4men 8xamt x1an 8x1b x1c 1xe. x3e4g 1xen xe1ro x1erz + 1xes 8xf x1g 8x1h 1xi 8xid xi8so 4xiste x1k 6x1l x1m 8xn 1xo 8x5o6d + 8x3p2 x1r x1s6 8x1t x6tak x8terf x2t1h 1xu xu1e x5ul 6x3w x1z 5ya. + y5an. y5ank y1b y1c y6cha y4chia y1d yen6n y5ern y1g y5h y5in y1j + y1k2 y1lak yl1al yla8m y5lax y1le y1lo y5lu y8mn ym1p2 y3mu y1na yno2d + yn1t y1on. y1o4p y5ou ypo1 y1pr y8ps y1r yri3e yr1r2 + + ys5iat ys8ty + y1t y3w y1z y�8m z5a6b zab5l 8za6d 1zah za5is 4z3ak 6z1am 5zange. + 8zanl 2z1ara 6z5as z5auf 3zaun 2z1b 6z1c 6z1d 1ze ze4dik 4z3eff 8zein + zei4ta zei8ters ze6la ze8lec zel8th 4zemp 6z5engel zen8zin 8zerg� + zer8i ze1ro zers8 zerta8 zer8tab zer8tag 8zerz ze8ste zeu6gr 2z1ex + 2z1f8 z1g 4z1h 1zi zi1en zi5es. 4z3imp zi1na 6z5inf 6z5inni zin6s5er + 8zinsuf zist5r zi5th zi1tr 6z1j 2z1k 2z1l 2z1m 6z1n 1zo zo6gl 4z3oh + zo1on zor6na8 4z1p z5q 6z1r 2z1s8 2z1t z4t3end z4t3hei z8thi 1zu zu3al + zu1b4 zu1f2 6z5uhr zun2a 8zunem zunf8 8zungl zu1o zup8fi zu1s8 zu1z + 2z1v zw8 z1wal 5zweck zwei3s z1wel z1wer z6werg 8z5wes 1zwi zwi1s + 6z1wo 1zy 2z1z zz8a zzi1s 1z� 1z� 6z�l. z�1le + 1z� 2z1�2b �1a6 �b1l �1che �3chi + �ch8sc �ch8sp �5chu �ck5a �d1a �d5era + �6d5ia �1e �5fa �f1l �ft6s �g1h + �g3le �6g5nan �g5str �1he �1hi �h1le + �h5ne 1�hnl �h1re �h5ri �h1ru �1hu + �h1w 6�i �1isc �6ische �5ism �5j + �1k �l1c �1le �8lei �l6schl �mi1e + �m8n �m8s �5na 5�nderu �ne5i8 �ng3l + �nk5l �1no �n6s5c �1pa �p6s5c 3�q + �r1c �1re �re8m 5�rgern �r6gl �1ri + 3�rmel �1ro �rt6s5 �1ru 3�rztl �5r� + �6s5chen �sen8s �s1th �ta8b �1te �teri4 + �ter5it �6thy �1ti 3�tk �1to �t8schl + �ts1p �5tu �ub1l �u1e 1�ug �u8ga + �u5i �1um. �1us. 1�u� �1z + �1b �1che �5chi + �ch8s2tei + �ch8str �cht6 + 5�6dem 5�ffn �1he �h1l8 �h1re �1hu + �1is �1ke 1�2ko 1�l. �l6k5l �l8pl + �1mu �5na �nig6s3 �1no �5o6t �pf3l + �p6s5c �1re �r8gli �1ri �r8tr �1ru + 5�sterr �1te �5th �1ti �1tu �1v �1w + �we8 �2z �b6e2 3�4ber1 �b1l �b1r + 5�2bu �1che �1chi �8ch3l �ch6s5c �8ck + �ck1a �ck5ers �d1a2 �6deu �di8t �2d1o4 + �d5s6 �ge4l5a �g1l �h5a �1he �8heh + �6h5erk �h1le �h1re �h1ru �1hu �h1w + �3k �1le �l4l5a �l8lo �l4ps �l6s5c + �1lu �n8da �n8fei �nk5l �n8za �n6zw + �5pi �1re �8rei �r8fl �r8fr �r8geng + �1ri �1ro �r8sta + + �1ru �se8n + �8sta �8stes + + �3ta �1te �1ti + �t8tr �1tu �t8zei �1v �1a8 5�a. + �8as �1b8 �1c �1d + 1�e �5ec 8�e8g 8�e8h + 2�1ei 8�em �1f8 �1g �1h + 1�i �1k �1l �1m + + �1n �1o �1p8 �5q + �1r �1s2 �st8 �1ta + �1te �t3hei �1ti �5to + �1tr 1�u8 6�5um �1v �1w + �1z + + 2s1ta. + i2s1tal + 2s1tani 2s1tan. + fe2s1ta + te2s1ta + + nd2ste + ve2ste + 3s2tec + 4s3techn + 3s2teg + 3s2teh + 3s2tein 3s2teig 3s2teif + 3s2tell 3s2telz + a4s3tel + 3s2temm + 3s2temp + 3s2tep + s3s2ter t3s2tern + 3s2teue + 6s4teuro + + bs2ti + te2s3ti + ve2sti + 3s2tic + + 3s2tieb + 3s2tieg + + 3s2tif + 3s2til + 3s2tim + 3s2tink + 3s2titu + + a2s1to + gu2s1to + ku2s1to + i2s1tol i2s1tor + ve2s1to + + 2s1tung + 2s7tus + o2s1tul + + + + aus3s4 + ens3s4 + gs3s4 + .mis2s1 + s2s1b8 + + s2s3chen + s2s3d + s2s5ec + + + 2s2s1ei + s2s3f + s2s1g + s2s3h + s2s3k + s2s3l + s2s3m + + s2s3n + s2s3p8 + s2s5q + s2s3r + s2s3s2 + sss2t8 + + + as2s3te + is2s3te + us2s3te + �s2s3te + s2st3hei + s2s3ti + s2s1to + s2s1tr + + 6ss5um + s2s3v + s2s3w + s2s3z + + + + 1cker. + 1ckert + 1ckad + 1cke. + 1ckel + 1cken + 4ck1ent + 1ckere + 1ckern + 1ckeru + 1ckie + 1ckig + 1ckun + + + diff --git a/modules/analysis-common/src/yamlRestTest/java/org/elasticsearch/analysis/common/CommonAnalysisClientYamlTestSuiteIT.java b/modules/analysis-common/src/yamlRestTest/java/org/elasticsearch/analysis/common/CommonAnalysisClientYamlTestSuiteIT.java index 9ed086f0a00b9..082bdcdba4256 100644 --- a/modules/analysis-common/src/yamlRestTest/java/org/elasticsearch/analysis/common/CommonAnalysisClientYamlTestSuiteIT.java +++ b/modules/analysis-common/src/yamlRestTest/java/org/elasticsearch/analysis/common/CommonAnalysisClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.analysis.common; diff --git a/modules/apm/build.gradle b/modules/apm/build.gradle index 4c822e44da6f6..07e6c7a042135 100644 --- a/modules/apm/build.gradle +++ b/modules/apm/build.gradle @@ -7,9 +7,9 @@ apply plugin: 'elasticsearch.internal-es-plugin' esplugin { - name 'apm' - description 'Provides APM integration for Elasticsearch' - classname 'org.elasticsearch.telemetry.apm.APM' + name = 'apm' + description = 'Provides APM integration for Elasticsearch' + classname ='org.elasticsearch.telemetry.apm.APM' } def otelVersion = '1.31.0' @@ -19,7 +19,7 @@ dependencies { implementation "io.opentelemetry:opentelemetry-api:${otelVersion}" implementation "io.opentelemetry:opentelemetry-context:${otelVersion}" implementation "io.opentelemetry:opentelemetry-semconv:${otelSemconvVersion}" - runtimeOnly "co.elastic.apm:elastic-apm-agent:1.44.0" + runtimeOnly "co.elastic.apm:elastic-apm-agent:1.52.0" } tasks.named("dependencyLicenses").configure { diff --git a/modules/apm/src/main/java/module-info.java b/modules/apm/src/main/java/module-info.java index 0bea3078f5f00..4f0a303e85f79 100644 --- a/modules/apm/src/main/java/module-info.java +++ b/modules/apm/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.telemetry.apm { diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APM.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APM.java index bf3f01bd2052f..339a4ec24ca13 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APM.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APM.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.telemetry.apm; diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APMMeterRegistry.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APMMeterRegistry.java index 831e2f19e0126..52c3a1507da87 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APMMeterRegistry.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APMMeterRegistry.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.telemetry.apm; diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/AbstractInstrument.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/AbstractInstrument.java index 9329556ff0f3f..e0c77f77c7443 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/AbstractInstrument.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/AbstractInstrument.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.telemetry.apm; diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java index a6d0b049807b5..f66683a787bc0 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.telemetry.apm.internal; @@ -26,7 +27,7 @@ import java.util.Set; import java.util.function.Function; -import static org.elasticsearch.common.settings.Setting.Property.DeprecatedWarning; +import static org.elasticsearch.common.settings.Setting.Property.Deprecated; import static org.elasticsearch.common.settings.Setting.Property.NodeScope; import static org.elasticsearch.common.settings.Setting.Property.OperatorDynamic; @@ -249,7 +250,7 @@ private static Setting concreteAgentSetting(String namespace, String qua TELEMETRY_SETTING_PREFIX + "agent.", LEGACY_TRACING_APM_SETTING_PREFIX + "agent.", (namespace, qualifiedKey) -> qualifiedKey.startsWith(LEGACY_TRACING_APM_SETTING_PREFIX) - ? concreteAgentSetting(namespace, qualifiedKey, NodeScope, OperatorDynamic, DeprecatedWarning) + ? concreteAgentSetting(namespace, qualifiedKey, NodeScope, OperatorDynamic, Deprecated) : concreteAgentSetting(namespace, qualifiedKey, NodeScope, OperatorDynamic) ); @@ -261,7 +262,7 @@ private static Setting concreteAgentSetting(String namespace, String qua LEGACY_TRACING_APM_SETTING_PREFIX + "names.include", OperatorDynamic, NodeScope, - DeprecatedWarning + Deprecated ); public static final Setting> TELEMETRY_TRACING_NAMES_INCLUDE_SETTING = Setting.listSetting( @@ -280,7 +281,7 @@ private static Setting concreteAgentSetting(String namespace, String qua LEGACY_TRACING_APM_SETTING_PREFIX + "names.exclude", OperatorDynamic, NodeScope, - DeprecatedWarning + Deprecated ); public static final Setting> TELEMETRY_TRACING_NAMES_EXCLUDE_SETTING = Setting.listSetting( @@ -313,7 +314,7 @@ private static Setting concreteAgentSetting(String namespace, String qua ), OperatorDynamic, NodeScope, - DeprecatedWarning + Deprecated ); public static final Setting> TELEMETRY_TRACING_SANITIZE_FIELD_NAMES = Setting.listSetting( @@ -333,7 +334,7 @@ private static Setting concreteAgentSetting(String namespace, String qua false, OperatorDynamic, NodeScope, - DeprecatedWarning + Deprecated ); public static final Setting TELEMETRY_TRACING_ENABLED_SETTING = Setting.boolSetting( @@ -357,7 +358,7 @@ private static Setting concreteAgentSetting(String namespace, String qua public static final Setting TRACING_APM_SECRET_TOKEN_SETTING = SecureSetting.secureString( LEGACY_TRACING_APM_SETTING_PREFIX + "secret_token", null, - DeprecatedWarning + Deprecated ); public static final Setting TELEMETRY_SECRET_TOKEN_SETTING = SecureSetting.secureString( @@ -372,7 +373,7 @@ private static Setting concreteAgentSetting(String namespace, String qua public static final Setting TRACING_APM_API_KEY_SETTING = SecureSetting.secureString( LEGACY_TRACING_APM_SETTING_PREFIX + "api_key", null, - DeprecatedWarning + Deprecated ); public static final Setting TELEMETRY_API_KEY_SETTING = SecureSetting.secureString( diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMMeterService.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMMeterService.java index ae1204e75af1a..066b6178481ff 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMMeterService.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMMeterService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.telemetry.apm.internal; diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMTelemetryProvider.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMTelemetryProvider.java index d7b061b4b0d19..d0957aaed6744 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMTelemetryProvider.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMTelemetryProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.telemetry.apm.internal; diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/MetricNameValidator.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/MetricNameValidator.java index 9ab7412426db8..bc5665bedcf12 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/MetricNameValidator.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/MetricNameValidator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.telemetry.apm.internal; diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleAsyncCounterAdapter.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleAsyncCounterAdapter.java index ab735c41ca890..03f92ddb6edd5 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleAsyncCounterAdapter.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleAsyncCounterAdapter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.telemetry.apm.internal.metrics; diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleCounterAdapter.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleCounterAdapter.java index 398419f6c3f69..b6237693a262c 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleCounterAdapter.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleCounterAdapter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.telemetry.apm.internal.metrics; diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleGaugeAdapter.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleGaugeAdapter.java index 2a9c2d45981ed..185a6cef94fd6 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleGaugeAdapter.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleGaugeAdapter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.telemetry.apm.internal.metrics; diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleHistogramAdapter.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleHistogramAdapter.java index 47b54a84d0aa9..54facc3185f28 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleHistogramAdapter.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleHistogramAdapter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.telemetry.apm.internal.metrics; diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleUpDownCounterAdapter.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleUpDownCounterAdapter.java index 7cb7100ba2631..00c4e8e9b905e 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleUpDownCounterAdapter.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleUpDownCounterAdapter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.telemetry.apm.internal.metrics; diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongAsyncCounterAdapter.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongAsyncCounterAdapter.java index 1bc21ef2c831c..c1d61001f48ba 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongAsyncCounterAdapter.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongAsyncCounterAdapter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.telemetry.apm.internal.metrics; diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongCounterAdapter.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongCounterAdapter.java index 54a7c3e3cab29..bde8a3354f26f 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongCounterAdapter.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongCounterAdapter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.telemetry.apm.internal.metrics; diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongGaugeAdapter.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongGaugeAdapter.java index eab9ed2eb5278..f8b1bf46a6eef 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongGaugeAdapter.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongGaugeAdapter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.telemetry.apm.internal.metrics; diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongHistogramAdapter.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongHistogramAdapter.java index 286922027ce2a..7b6fe849ae7d6 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongHistogramAdapter.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongHistogramAdapter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.telemetry.apm.internal.metrics; diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongUpDownCounterAdapter.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongUpDownCounterAdapter.java index fa09740085aa3..210e829e7c462 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongUpDownCounterAdapter.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongUpDownCounterAdapter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.telemetry.apm.internal.metrics; diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/OtelHelper.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/OtelHelper.java index 1d760c8c12791..ea2a921c06fb6 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/OtelHelper.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/OtelHelper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.telemetry.apm.internal.metrics; diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracer.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracer.java index 0b020a24eeffb..ed2ce47d11dc3 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracer.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.telemetry.apm.internal.tracing; diff --git a/modules/apm/src/main/plugin-metadata/entitlement-policy.yaml b/modules/apm/src/main/plugin-metadata/entitlement-policy.yaml new file mode 100644 index 0000000000000..9c10bafca42f9 --- /dev/null +++ b/modules/apm/src/main/plugin-metadata/entitlement-policy.yaml @@ -0,0 +1,4 @@ +org.elasticsearch.telemetry.apm: + - create_class_loader +elastic.apm.agent: + - set_https_connection_properties diff --git a/modules/apm/src/main/plugin-metadata/plugin-security.policy b/modules/apm/src/main/plugin-metadata/plugin-security.policy index 57da3a2efd301..763ae7f582d38 100644 --- a/modules/apm/src/main/plugin-metadata/plugin-security.policy +++ b/modules/apm/src/main/plugin-metadata/plugin-security.policy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ grant { diff --git a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/APMMeterRegistryTests.java b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/APMMeterRegistryTests.java index 8144b8f9a33b4..52b33aa4955dd 100644 --- a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/APMMeterRegistryTests.java +++ b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/APMMeterRegistryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.telemetry.apm; diff --git a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/MeterRegistryConcurrencyTests.java b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/MeterRegistryConcurrencyTests.java index 11951a9bf1072..2951555596888 100644 --- a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/MeterRegistryConcurrencyTests.java +++ b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/MeterRegistryConcurrencyTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.telemetry.apm; diff --git a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/RecordingOtelMeter.java b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/RecordingOtelMeter.java index 793803fb8c277..a1c667a000db7 100644 --- a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/RecordingOtelMeter.java +++ b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/RecordingOtelMeter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.telemetry.apm; diff --git a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettingsTests.java b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettingsTests.java index 0c55471bd7a3e..a60048c82a3c9 100644 --- a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettingsTests.java +++ b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettingsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.telemetry.apm.internal; diff --git a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/MetricNameValidatorTests.java b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/MetricNameValidatorTests.java index 9a5479cc65a93..0198b1ffa56c4 100644 --- a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/MetricNameValidatorTests.java +++ b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/MetricNameValidatorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.telemetry.apm.internal; diff --git a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/TestAPMMeterService.java b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/TestAPMMeterService.java index bed611802d1e8..0ef49cbb748ab 100644 --- a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/TestAPMMeterService.java +++ b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/TestAPMMeterService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.telemetry.apm.internal; diff --git a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/metrics/AsyncCountersAdapterTests.java b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/metrics/AsyncCountersAdapterTests.java index 24b40063cd636..74b361c8e7a9e 100644 --- a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/metrics/AsyncCountersAdapterTests.java +++ b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/metrics/AsyncCountersAdapterTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.telemetry.apm.internal.metrics; diff --git a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/metrics/GaugeAdapterTests.java b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/metrics/GaugeAdapterTests.java index d5e605df1d096..49ba9de1aaed3 100644 --- a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/metrics/GaugeAdapterTests.java +++ b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/metrics/GaugeAdapterTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.telemetry.apm.internal.metrics; diff --git a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracerTests.java b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracerTests.java index 789b9294e2a6e..3a82aa129ed87 100644 --- a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracerTests.java +++ b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracerTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.telemetry.apm.internal.tracing; diff --git a/modules/build.gradle b/modules/build.gradle index 7707b60b38b25..be84095340422 100644 --- a/modules/build.gradle +++ b/modules/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ configure(subprojects.findAll { it.parent.path == project.path }) { diff --git a/modules/data-streams/build.gradle b/modules/data-streams/build.gradle index daf0c188cc83e..de9544a4e797d 100644 --- a/modules/data-streams/build.gradle +++ b/modules/data-streams/build.gradle @@ -8,8 +8,8 @@ apply plugin: 'elasticsearch.internal-java-rest-test' apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { - description 'Elasticsearch Expanded Pack Plugin - Data Streams' - classname 'org.elasticsearch.datastreams.DataStreamsPlugin' + description = 'Elasticsearch Expanded Pack Plugin - Data Streams' + classname ='org.elasticsearch.datastreams.DataStreamsPlugin' } restResources { @@ -28,15 +28,47 @@ tasks.withType(StandaloneRestIntegTestTask).configureEach { usesDefaultDistribution() } -if (BuildParams.inFipsJvm){ +if (buildParams.inFipsJvm){ // These fail in CI but only when run as part of checkPart2 and not individually. // Tracked in : tasks.named("javaRestTest").configure{enabled = false } tasks.named("yamlRestTest").configure{enabled = false } } -if (BuildParams.isSnapshotBuild() == false) { +if (buildParams.isSnapshotBuild() == false) { tasks.withType(Test).configureEach { systemProperty 'es.failure_store_feature_flag_enabled', 'true' } } + +tasks.named("yamlRestTestV7CompatTransform").configure({ task -> + task.skipTest("data_stream/10_basic/Create hidden data stream", "warning does not exist for compatibility") + + // Failure store configuration changed on 8.18 (earlier versions behind feature flag) + task.skipTest("data_stream/10_basic/Create data stream with failure store", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/10_basic/Delete data stream with failure store", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/10_basic/Delete data stream with failure store uninitialized", "Configuring the failure store via data stream templates is not supported anymore.") + + task.skipTest("data_stream/30_auto_create_data_stream/Don't initialize failure store during data stream auto-creation on successful index", "Configuring the failure store via data stream templates is not supported anymore.") + + task.skipTest("data_stream/150_tsdb/TSDB failures go to failure store", "Configuring the failure store via data stream templates is not supported anymore.") + + task.skipTest("data_stream/170_modify_data_stream/Modify a data stream's failure store", "Configuring the failure store via data stream templates is not supported anymore.") + + task.skipTest("data_stream/190_failure_store_redirection/Failure redirects to original failure store during index change if final pipeline changes target", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/190_failure_store_redirection/Ensure failure is redirected to correct failure store after a reroute processor", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/190_failure_store_redirection/Test failure store status with bulk request", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/190_failure_store_redirection/Redirect ingest failure in data stream to failure store", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/190_failure_store_redirection/Failure redirects to correct failure store when pipeline loop is detected", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/190_failure_store_redirection/Failure redirects to correct failure store when index loop is detected", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/190_failure_store_redirection/Failure redirects to original failure store during index change if self referenced", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/190_failure_store_redirection/Redirect shard failure in data stream to failure store", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/190_failure_store_redirection/Version conflicts are not redirected to failure store", "Configuring the failure store via data stream templates is not supported anymore.") + + task.skipTest("data_stream/200_rollover_failure_store/Lazily roll over a data stream's failure store after an ingest failure", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/200_rollover_failure_store/A failure store marked for lazy rollover should only be rolled over when there is a failure", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/200_rollover_failure_store/Roll over a data stream's failure store without conditions", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/200_rollover_failure_store/Lazily roll over a data stream's failure store after a shard failure", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/200_rollover_failure_store/Don't roll over a data stream's failure store when conditions aren't met", "Configuring the failure store via data stream templates is not supported anymore.") + task.skipTest("data_stream/200_rollover_failure_store/Roll over a data stream's failure store with conditions", "Configuring the failure store via data stream templates is not supported anymore.") +}) diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java index ed09bd2462bb3..ac73385a97d70 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; @@ -501,7 +502,7 @@ static void putComposableIndexTemplate(String id, List patterns, @Nullab request.indexTemplate( ComposableIndexTemplate.builder() .indexPatterns(patterns) - .template(new Template(settings, null, null, null)) + .template(Template.builder().settings(settings)) .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) .build() ); diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java index ebe5546c0907f..70218f17c67fe 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; @@ -66,6 +67,7 @@ import org.elasticsearch.cluster.metadata.DataStreamAction; import org.elasticsearch.cluster.metadata.DataStreamAlias; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexMetadataStats; import org.elasticsearch.cluster.metadata.IndexWriteLoad; @@ -1414,7 +1416,7 @@ public void testNoTimestampInDocument() throws Exception { IndexRequest indexRequest = new IndexRequest(dataStreamName).opType("create").source("{}", XContentType.JSON); Exception e = expectThrows(Exception.class, client().index(indexRequest)); - assertThat(e.getCause().getMessage(), equalTo("data stream timestamp field [@timestamp] is missing")); + assertThat(e.getCause().getCause().getMessage(), equalTo("data stream timestamp field [@timestamp] is missing")); } public void testMultipleTimestampValuesInDocument() throws Exception { @@ -1430,7 +1432,7 @@ public void testMultipleTimestampValuesInDocument() throws Exception { IndexRequest indexRequest = new IndexRequest(dataStreamName).opType("create") .source("{\"@timestamp\": [\"2020-12-12\",\"2022-12-12\"]}", XContentType.JSON); Exception e = expectThrows(Exception.class, client().index(indexRequest)); - assertThat(e.getCause().getMessage(), equalTo("data stream timestamp field [@timestamp] encountered multiple values")); + assertThat(e.getCause().getCause().getMessage(), equalTo("data stream timestamp field [@timestamp] encountered multiple values")); } public void testMixedAutoCreate() throws Exception { @@ -2440,9 +2442,16 @@ static void putComposableIndexTemplate( request.indexTemplate( ComposableIndexTemplate.builder() .indexPatterns(patterns) - .template(new Template(settings, mappings == null ? null : CompressedXContent.fromJSON(mappings), aliases, lifecycle)) + .template( + Template.builder() + .settings(settings) + .mappings(mappings == null ? null : CompressedXContent.fromJSON(mappings)) + .aliases(aliases) + .lifecycle(lifecycle) + .dataStreamOptions(DataStreamTestHelper.createDataStreamOptionsTemplate(withFailureStore)) + ) .metadata(metadata) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false, withFailureStore)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) .build() ); client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java index 5b5f3920f019b..373cf032cac3b 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java index d0dd2888f535f..8f45d054f01a6 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; @@ -57,7 +58,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -72,6 +72,7 @@ import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase { @@ -79,6 +80,8 @@ public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase { private static final Map DOCUMENT_SOURCE = Collections.singletonMap("@timestamp", 123); public static final String REPO = "repo"; public static final String SNAPSHOT = "snap"; + public static final String TEMPLATE_1_ID = "t1"; + public static final String TEMPLATE_2_ID = "t2"; private Client client; private String dsBackingIndexName; @@ -102,8 +105,8 @@ public void setup() throws Exception { Path location = randomRepoPath(); createRepository(REPO, "fs", location); - DataStreamIT.putComposableIndexTemplate("t1", List.of("ds", "other-ds")); - DataStreamIT.putComposableIndexTemplate("t2", """ + DataStreamIT.putComposableIndexTemplate(TEMPLATE_1_ID, List.of("ds", "other-ds")); + DataStreamIT.putComposableIndexTemplate(TEMPLATE_2_ID, """ { "properties": { "@timestamp": { @@ -131,27 +134,18 @@ public void setup() throws Exception { // Initialize the failure store. RolloverRequest rolloverRequest = new RolloverRequest("with-fs", null); rolloverRequest.setIndicesOptions( - IndicesOptions.builder(rolloverRequest.indicesOptions()) - .failureStoreOptions(b -> b.includeRegularIndices(false).includeFailureIndices(true)) - .build() + IndicesOptions.builder(rolloverRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.FAILURES).build() ); response = client.execute(RolloverAction.INSTANCE, rolloverRequest).get(); assertTrue(response.isAcknowledged()); // Resolve backing index names after data streams have been created: // (these names have a date component, and running around midnight could lead to test failures otherwise) - GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" }); - GetDataStreamAction.Response getDataStreamResponse = client.execute(GetDataStreamAction.INSTANCE, getDataStreamRequest).actionGet(); - dsBackingIndexName = getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(); - otherDsBackingIndexName = getDataStreamResponse.getDataStreams().get(1).getDataStream().getIndices().get(0).getName(); - fsBackingIndexName = getDataStreamResponse.getDataStreams().get(2).getDataStream().getIndices().get(0).getName(); - fsFailureIndexName = getDataStreamResponse.getDataStreams() - .get(2) - .getDataStream() - .getFailureIndices() - .getIndices() - .get(0) - .getName(); + List dataStreamInfos = getDataStreamInfo("*"); + dsBackingIndexName = dataStreamInfos.get(0).getDataStream().getIndices().get(0).getName(); + otherDsBackingIndexName = dataStreamInfos.get(1).getDataStream().getIndices().get(0).getName(); + fsBackingIndexName = dataStreamInfos.get(2).getDataStream().getIndices().get(0).getName(); + fsFailureIndexName = dataStreamInfos.get(2).getDataStream().getFailureIndices().getIndices().get(0).getName(); // Will be used in some tests, to test renaming while restoring a snapshot: ds2BackingIndexName = dsBackingIndexName.replace("-ds-", "-ds2-"); @@ -193,9 +187,7 @@ public void testSnapshotAndRestore() throws Exception { assertEquals(Collections.singletonList(dsBackingIndexName), getSnapshot(REPO, SNAPSHOT).indices()); - assertAcked( - client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "ds" })) - ); + assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, "ds"))); RestoreSnapshotResponse restoreSnapshotResponse = client.admin() .cluster() @@ -213,13 +205,10 @@ public void testSnapshotAndRestore() throws Exception { assertEquals(DOCUMENT_SOURCE, hits[0].getSourceAsMap()); }); - GetDataStreamAction.Response ds = client.execute( - GetDataStreamAction.INSTANCE, - new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "ds" }) - ).get(); - assertEquals(1, ds.getDataStreams().size()); - assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size()); - assertEquals(dsBackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName()); + List ds = getDataStreamInfo("ds"); + assertEquals(1, ds.size()); + assertEquals(1, ds.get(0).getDataStream().getIndices().size()); + assertEquals(dsBackingIndexName, ds.get(0).getDataStream().getIndices().get(0).getName()); GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(new GetAliasesRequest("my-alias")).actionGet(); assertThat(getAliasesResponse.getDataStreamAliases().keySet(), containsInAnyOrder("ds", "other-ds")); @@ -273,19 +262,18 @@ public void testSnapshotAndRestoreAllDataStreamsInPlace() throws Exception { assertEquals(DOCUMENT_SOURCE, hits[0].getSourceAsMap()); }); - GetDataStreamAction.Request getDataSteamRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" }); - GetDataStreamAction.Response ds = client.execute(GetDataStreamAction.INSTANCE, getDataSteamRequest).get(); + List dataStreamInfos = getDataStreamInfo("*"); assertThat( - ds.getDataStreams().stream().map(e -> e.getDataStream().getName()).collect(Collectors.toList()), + dataStreamInfos.stream().map(e -> e.getDataStream().getName()).collect(Collectors.toList()), contains(equalTo("ds"), equalTo("other-ds"), equalTo("with-fs")) ); - List backingIndices = ds.getDataStreams().get(0).getDataStream().getIndices(); + List backingIndices = dataStreamInfos.get(0).getDataStream().getIndices(); assertThat(backingIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(dsBackingIndexName)); - backingIndices = ds.getDataStreams().get(1).getDataStream().getIndices(); + backingIndices = dataStreamInfos.get(1).getDataStream().getIndices(); assertThat(backingIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(otherDsBackingIndexName)); - backingIndices = ds.getDataStreams().get(2).getDataStream().getIndices(); + backingIndices = dataStreamInfos.get(2).getDataStream().getIndices(); assertThat(backingIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(fsBackingIndexName)); - List failureIndices = ds.getDataStreams().get(2).getDataStream().getFailureIndices().getIndices(); + List failureIndices = dataStreamInfos.get(2).getDataStream().getFailureIndices().getIndices(); assertThat(failureIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(fsFailureIndexName)); } @@ -332,14 +320,10 @@ public void testSnapshotAndRestoreInPlace() { assertEquals(DOCUMENT_SOURCE, hits[0].getSourceAsMap()); }); - GetDataStreamAction.Request getDataSteamRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "ds" }); - GetDataStreamAction.Response ds = client.execute(GetDataStreamAction.INSTANCE, getDataSteamRequest).actionGet(); - assertThat( - ds.getDataStreams().stream().map(e -> e.getDataStream().getName()).collect(Collectors.toList()), - contains(equalTo("ds")) - ); - List backingIndices = ds.getDataStreams().get(0).getDataStream().getIndices(); - assertThat(ds.getDataStreams().get(0).getDataStream().getIndices(), hasSize(1)); + List dsInfo = getDataStreamInfo("ds"); + assertThat(dsInfo.stream().map(e -> e.getDataStream().getName()).collect(Collectors.toList()), contains(equalTo("ds"))); + List backingIndices = dsInfo.get(0).getDataStream().getIndices(); + assertThat(dsInfo.get(0).getDataStream().getIndices(), hasSize(1)); assertThat(backingIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(equalTo(dsBackingIndexName))); // The backing index created as part of rollover should still exist (but just not part of the data stream) @@ -352,39 +336,40 @@ public void testSnapshotAndRestoreInPlace() { } public void testFailureStoreSnapshotAndRestore() throws Exception { + String dataStreamName = "with-fs"; CreateSnapshotResponse createSnapshotResponse = client.admin() .cluster() .prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT) .setWaitForCompletion(true) - .setIndices("with-fs") + .setIndices(dataStreamName) .setIncludeGlobalState(false) .get(); RestStatus status = createSnapshotResponse.getSnapshotInfo().status(); assertEquals(RestStatus.OK, status); + assertThat(getSnapshot(REPO, SNAPSHOT).dataStreams(), containsInAnyOrder(dataStreamName)); assertThat(getSnapshot(REPO, SNAPSHOT).indices(), containsInAnyOrder(fsBackingIndexName, fsFailureIndexName)); - assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, "with-fs"))); + assertAcked( + client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, dataStreamName)) + ); { RestoreSnapshotResponse restoreSnapshotResponse = client.admin() .cluster() .prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT) .setWaitForCompletion(true) - .setIndices("with-fs") + .setIndices(dataStreamName) .get(); assertEquals(2, restoreSnapshotResponse.getRestoreInfo().successfulShards()); - GetDataStreamAction.Response ds = client.execute( - GetDataStreamAction.INSTANCE, - new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "with-fs" }) - ).get(); - assertEquals(1, ds.getDataStreams().size()); - assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size()); - assertEquals(fsBackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName()); - assertEquals(fsFailureIndexName, ds.getDataStreams().get(0).getDataStream().getFailureIndices().getIndices().get(0).getName()); + List dataStreamInfos = getDataStreamInfo(dataStreamName); + assertEquals(1, dataStreamInfos.size()); + assertEquals(1, dataStreamInfos.get(0).getDataStream().getIndices().size()); + assertEquals(fsBackingIndexName, dataStreamInfos.get(0).getDataStream().getIndices().get(0).getName()); + assertEquals(fsFailureIndexName, dataStreamInfos.get(0).getDataStream().getFailureIndices().getIndices().get(0).getName()); } { // With rename pattern @@ -392,21 +377,18 @@ public void testFailureStoreSnapshotAndRestore() throws Exception { .cluster() .prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT) .setWaitForCompletion(true) - .setIndices("with-fs") + .setIndices(dataStreamName) .setRenamePattern("-fs") .setRenameReplacement("-fs2") .get(); assertEquals(2, restoreSnapshotResponse.getRestoreInfo().successfulShards()); - GetDataStreamAction.Response ds = client.execute( - GetDataStreamAction.INSTANCE, - new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "with-fs2" }) - ).get(); - assertEquals(1, ds.getDataStreams().size()); - assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size()); - assertEquals(fs2BackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName()); - assertEquals(fs2FailureIndexName, ds.getDataStreams().get(0).getDataStream().getFailureIndices().getIndices().get(0).getName()); + List dataStreamInfos = getDataStreamInfo("with-fs2"); + assertEquals(1, dataStreamInfos.size()); + assertEquals(1, dataStreamInfos.get(0).getDataStream().getIndices().size()); + assertEquals(fs2BackingIndexName, dataStreamInfos.get(0).getDataStream().getIndices().get(0).getName()); + assertEquals(fs2FailureIndexName, dataStreamInfos.get(0).getDataStream().getFailureIndices().getIndices().get(0).getName()); } } @@ -472,13 +454,10 @@ public void testSnapshotAndRestoreAllIncludeSpecificDataStream() throws Exceptio assertEquals(DOCUMENT_SOURCE, hits[0].getSourceAsMap()); }); - GetDataStreamAction.Response ds = client.execute( - GetDataStreamAction.INSTANCE, - new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamToSnapshot }) - ).get(); - assertEquals(1, ds.getDataStreams().size()); - assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size()); - assertEquals(backingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName()); + List dataStreamInfos = getDataStreamInfo(dataStreamToSnapshot); + assertEquals(1, dataStreamInfos.size()); + assertEquals(1, dataStreamInfos.get(0).getDataStream().getIndices().size()); + assertEquals(backingIndexName, dataStreamInfos.get(0).getDataStream().getIndices().get(0).getName()); GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(new GetAliasesRequest("my-alias")).actionGet(); assertThat(getAliasesResponse.getDataStreamAliases().keySet(), contains(dataStreamToSnapshot)); @@ -531,13 +510,10 @@ public void testSnapshotAndRestoreReplaceAll() throws Exception { assertEquals(DOCUMENT_SOURCE, hits[0].getSourceAsMap()); }); - GetDataStreamAction.Response ds = client.execute( - GetDataStreamAction.INSTANCE, - new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" }) - ).get(); - assertEquals(3, ds.getDataStreams().size()); + List dataStreamInfos = getDataStreamInfo("*"); + assertEquals(3, dataStreamInfos.size()); assertThat( - ds.getDataStreams().stream().map(i -> i.getDataStream().getName()).collect(Collectors.toList()), + dataStreamInfos.stream().map(i -> i.getDataStream().getName()).collect(Collectors.toList()), containsInAnyOrder("ds", "other-ds", "with-fs") ); @@ -591,19 +567,16 @@ public void testSnapshotAndRestoreAll() throws Exception { assertEquals(DOCUMENT_SOURCE, hits[0].getSourceAsMap()); }); - GetDataStreamAction.Response ds = client.execute( - GetDataStreamAction.INSTANCE, - new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" }) - ).get(); - assertEquals(3, ds.getDataStreams().size()); - assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size()); - assertEquals(dsBackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName()); - assertEquals(1, ds.getDataStreams().get(1).getDataStream().getIndices().size()); - assertEquals(otherDsBackingIndexName, ds.getDataStreams().get(1).getDataStream().getIndices().get(0).getName()); - assertEquals(1, ds.getDataStreams().get(2).getDataStream().getIndices().size()); - assertEquals(fsBackingIndexName, ds.getDataStreams().get(2).getDataStream().getIndices().get(0).getName()); - assertEquals(1, ds.getDataStreams().get(2).getDataStream().getFailureIndices().getIndices().size()); - assertEquals(fsFailureIndexName, ds.getDataStreams().get(2).getDataStream().getFailureIndices().getIndices().get(0).getName()); + List dataStreamInfos = getDataStreamInfo("*"); + assertEquals(3, dataStreamInfos.size()); + assertEquals(1, dataStreamInfos.get(0).getDataStream().getIndices().size()); + assertEquals(dsBackingIndexName, dataStreamInfos.get(0).getDataStream().getIndices().get(0).getName()); + assertEquals(1, dataStreamInfos.get(1).getDataStream().getIndices().size()); + assertEquals(otherDsBackingIndexName, dataStreamInfos.get(1).getDataStream().getIndices().get(0).getName()); + assertEquals(1, dataStreamInfos.get(2).getDataStream().getIndices().size()); + assertEquals(fsBackingIndexName, dataStreamInfos.get(2).getDataStream().getIndices().get(0).getName()); + assertEquals(1, dataStreamInfos.get(2).getDataStream().getFailureIndices().getIndices().size()); + assertEquals(fsFailureIndexName, dataStreamInfos.get(2).getDataStream().getFailureIndices().getIndices().get(0).getName()); GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(new GetAliasesRequest("my-alias")).actionGet(); assertThat(getAliasesResponse.getDataStreamAliases().keySet(), containsInAnyOrder("ds", "other-ds")); @@ -662,19 +635,16 @@ public void testSnapshotAndRestoreIncludeAliasesFalse() throws Exception { assertEquals(DOCUMENT_SOURCE, hits[0].getSourceAsMap()); }); - GetDataStreamAction.Response ds = client.execute( - GetDataStreamAction.INSTANCE, - new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" }) - ).get(); - assertEquals(3, ds.getDataStreams().size()); - assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size()); - assertEquals(dsBackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName()); - assertEquals(1, ds.getDataStreams().get(1).getDataStream().getIndices().size()); - assertEquals(otherDsBackingIndexName, ds.getDataStreams().get(1).getDataStream().getIndices().get(0).getName()); - assertEquals(1, ds.getDataStreams().get(2).getDataStream().getIndices().size()); - assertEquals(fsBackingIndexName, ds.getDataStreams().get(2).getDataStream().getIndices().get(0).getName()); - assertEquals(1, ds.getDataStreams().get(2).getDataStream().getIndices().size()); - assertEquals(fsFailureIndexName, ds.getDataStreams().get(2).getDataStream().getFailureIndices().getIndices().get(0).getName()); + List dataStreamInfos = getDataStreamInfo("*"); + assertEquals(3, dataStreamInfos.size()); + assertEquals(1, dataStreamInfos.get(0).getDataStream().getIndices().size()); + assertEquals(dsBackingIndexName, dataStreamInfos.get(0).getDataStream().getIndices().get(0).getName()); + assertEquals(1, dataStreamInfos.get(1).getDataStream().getIndices().size()); + assertEquals(otherDsBackingIndexName, dataStreamInfos.get(1).getDataStream().getIndices().get(0).getName()); + assertEquals(1, dataStreamInfos.get(2).getDataStream().getIndices().size()); + assertEquals(fsBackingIndexName, dataStreamInfos.get(2).getDataStream().getIndices().get(0).getName()); + assertEquals(1, dataStreamInfos.get(2).getDataStream().getIndices().size()); + assertEquals(fsFailureIndexName, dataStreamInfos.get(2).getDataStream().getFailureIndices().getIndices().get(0).getName()); GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(new GetAliasesRequest("*")).actionGet(); assertThat(getAliasesResponse.getDataStreamAliases(), anEmptyMap()); @@ -716,13 +686,10 @@ public void testRename() throws Exception { .setRenameReplacement("ds2") .get(); - GetDataStreamAction.Response ds = client.execute( - GetDataStreamAction.INSTANCE, - new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "ds2" }) - ).get(); - assertEquals(1, ds.getDataStreams().size()); - assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size()); - assertEquals(ds2BackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName()); + List dataStreamInfos = getDataStreamInfo("ds2"); + assertEquals(1, dataStreamInfos.size()); + assertEquals(1, dataStreamInfos.get(0).getDataStream().getIndices().size()); + assertEquals(ds2BackingIndexName, dataStreamInfos.get(0).getDataStream().getIndices().get(0).getName()); assertResponse( client.prepareSearch("ds2"), response -> assertEquals(DOCUMENT_SOURCE, response.getHits().getHits()[0].getSourceAsMap()) @@ -774,13 +741,10 @@ public void testRenameWriteDataStream() throws Exception { .setRenameReplacement("other-ds2") .get(); - GetDataStreamAction.Response ds = client.execute( - GetDataStreamAction.INSTANCE, - new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "other-ds2" }) - ).get(); - assertEquals(1, ds.getDataStreams().size()); - assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size()); - assertEquals(otherDs2BackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName()); + List dataStreamInfos = getDataStreamInfo("other-ds2"); + assertEquals(1, dataStreamInfos.size()); + assertEquals(1, dataStreamInfos.get(0).getDataStream().getIndices().size()); + assertEquals(otherDs2BackingIndexName, dataStreamInfos.get(0).getDataStream().getIndices().get(0).getName()); GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(new GetAliasesRequest("my-alias")).actionGet(); assertThat(getAliasesResponse.getDataStreamAliases().keySet(), containsInAnyOrder("ds", "other-ds", "other-ds2")); @@ -844,9 +808,8 @@ public void testBackingIndexIsNotRenamedWhenRestoringDataStream() { assertThat(restoreSnapshotResponse.status(), is(RestStatus.OK)); - GetDataStreamAction.Request getDSRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "ds" }); - GetDataStreamAction.Response response = client.execute(GetDataStreamAction.INSTANCE, getDSRequest).actionGet(); - assertThat(response.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(), is(dsBackingIndexName)); + List dataStreamInfos = getDataStreamInfo("ds"); + assertThat(dataStreamInfos.get(0).getDataStream().getIndices().get(0).getName(), is(dsBackingIndexName)); } public void testDataStreamAndBackingIndicesAreRenamedUsingRegex() { @@ -883,17 +846,15 @@ public void testDataStreamAndBackingIndicesAreRenamedUsingRegex() { assertThat(restoreSnapshotResponse.status(), is(RestStatus.OK)); // assert "ds" was restored as "test-ds" and the backing index has a valid name - GetDataStreamAction.Request getRenamedDS = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "test-ds" }); - GetDataStreamAction.Response response = client.execute(GetDataStreamAction.INSTANCE, getRenamedDS).actionGet(); + List dataStreamInfos = getDataStreamInfo("test-ds"); assertThat( - response.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(), + dataStreamInfos.get(0).getDataStream().getIndices().get(0).getName(), is(DataStream.getDefaultBackingIndexName("test-ds", 1L)) ); // data stream "ds" should still exist in the system - GetDataStreamAction.Request getDSRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "ds" }); - response = client.execute(GetDataStreamAction.INSTANCE, getDSRequest).actionGet(); - assertThat(response.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(), is(dsBackingIndexName)); + dataStreamInfos = getDataStreamInfo("ds"); + assertThat(dataStreamInfos.get(0).getDataStream().getIndices().get(0).getName(), is(dsBackingIndexName)); } public void testWildcards() throws Exception { @@ -919,16 +880,13 @@ public void testWildcards() throws Exception { assertEquals(RestStatus.OK, restoreSnapshotResponse.status()); - GetDataStreamAction.Response ds = client.execute( - GetDataStreamAction.INSTANCE, - new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "ds2" }) - ).get(); - assertEquals(1, ds.getDataStreams().size()); - assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size()); - assertEquals(ds2BackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName()); + List dataStreamInfos = getDataStreamInfo("ds2"); + assertEquals(1, dataStreamInfos.size()); + assertEquals(1, dataStreamInfos.get(0).getDataStream().getIndices().size()); + assertEquals(ds2BackingIndexName, dataStreamInfos.get(0).getDataStream().getIndices().get(0).getName()); assertThat( "we renamed the restored data stream to one that doesn't match any existing composable template", - ds.getDataStreams().get(0).getIndexTemplate(), + dataStreamInfos.get(0).getIndexTemplate(), is(nullValue()) ); } @@ -950,7 +908,7 @@ public void testDataStreamNotStoredWhenIndexRequested() { ); } - public void testDataStreamNotRestoredWhenIndexRequested() throws Exception { + public void testDataStreamNotRestoredWhenIndexRequested() { CreateSnapshotResponse createSnapshotResponse = client.admin() .cluster() .prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO, "snap2") @@ -979,7 +937,7 @@ public void testDataStreamNotRestoredWhenIndexRequested() throws Exception { expectThrows(ResourceNotFoundException.class, client.execute(GetDataStreamAction.INSTANCE, getRequest)); } - public void testDataStreamNotIncludedInLimitedSnapshot() throws ExecutionException, InterruptedException { + public void testDataStreamNotIncludedInLimitedSnapshot() { final String snapshotName = "test-snap"; CreateSnapshotResponse createSnapshotResponse = client.admin() .cluster() @@ -1037,12 +995,7 @@ public void testDeleteDataStreamDuringSnapshot() throws Exception { assertDocCount(dataStream, 100L); // Resolve backing index name after the data stream has been created because it has a date component, // and running around midnight could lead to test failures otherwise - GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request( - TEST_REQUEST_TIMEOUT, - new String[] { dataStream } - ); - GetDataStreamAction.Response getDataStreamResponse = client.execute(GetDataStreamAction.INSTANCE, getDataStreamRequest).actionGet(); - String backingIndexName = getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(); + String backingIndexName = getDataStreamInfo(dataStream).get(0).getDataStream().getIndices().get(0).getName(); logger.info("--> snapshot"); ActionFuture future = client1.admin() @@ -1230,7 +1183,7 @@ public void testSnapshotDSDuringRolloverAndDeleteOldIndex() throws Exception { assertEquals(restoreSnapshotResponse.failedShards(), 0); } - public void testExcludeDSFromSnapshotWhenExcludingItsIndices() { + public void testExcludeDSFromSnapshotWhenExcludingAnyOfItsIndices() { final String snapshot = "test-snapshot"; final String indexWithoutDataStream = "test-idx-no-ds"; createIndexWithContent(indexWithoutDataStream); @@ -1246,10 +1199,47 @@ public void testExcludeDSFromSnapshotWhenExcludingItsIndices() { .getRestoreInfo(); assertThat(restoreInfo.failedShards(), is(0)); assertThat(restoreInfo.successfulShards(), is(1)); + + // Exclude only failure store indices + { + String dataStreamName = "with-fs"; + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT) + .setWaitForCompletion(true) + .setIndices(dataStreamName + "*", "-.fs*") + .setIncludeGlobalState(false) + .get(); + + RestStatus status = createSnapshotResponse.getSnapshotInfo().status(); + assertEquals(RestStatus.OK, status); + + SnapshotInfo retrievedSnapshot = getSnapshot(REPO, SNAPSHOT); + assertThat(retrievedSnapshot.dataStreams(), contains(dataStreamName)); + assertThat(retrievedSnapshot.indices(), containsInAnyOrder(fsBackingIndexName)); + + assertAcked( + safeGet(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, "*"))) + ); + + RestoreInfo restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT) + .setWaitForCompletion(true) + .setIndices(dataStreamName) + .get() + .getRestoreInfo(); + + assertThat(restoreSnapshotResponse, notNullValue()); + assertThat(restoreSnapshotResponse.successfulShards(), equalTo(restoreSnapshotResponse.totalShards())); + assertThat(restoreSnapshotResponse.failedShards(), is(0)); + + GetDataStreamAction.Response.DataStreamInfo dataStream = getDataStreamInfo(dataStreamName).get(0); + assertThat(dataStream.getDataStream().getBackingIndices().getIndices(), not(empty())); + assertThat(dataStream.getDataStream().getFailureIndices().getIndices(), empty()); + } } /** - * This test is a copy of the {@link #testExcludeDSFromSnapshotWhenExcludingItsIndices()} the only difference + * This test is a copy of the {@link #testExcludeDSFromSnapshotWhenExcludingAnyOfItsIndices()} ()} the only difference * is that one include the global state and one doesn't. In general this shouldn't matter that's why it used to be * a random parameter of the test, but because of #107515 it fails when we include the global state. Keep them * separate until this is fixed. @@ -1279,10 +1269,7 @@ public void testRestoreSnapshotFully() throws Exception { createIndexWithContent(indexName); createFullSnapshot(REPO, snapshotName); - assertAcked( - client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" })) - .get() - ); + assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, "*")).get()); assertAcked(client.admin().indices().prepareDelete("*").setIndicesOptions(IndicesOptions.lenientExpandOpenHidden()).get()); RestoreSnapshotResponse restoreSnapshotResponse = client.admin() @@ -1292,8 +1279,7 @@ public void testRestoreSnapshotFully() throws Exception { .get(); assertEquals(RestStatus.OK, restoreSnapshotResponse.status()); - GetDataStreamAction.Request getRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" }); - assertThat(client.execute(GetDataStreamAction.INSTANCE, getRequest).get().getDataStreams(), hasSize(3)); + assertThat(getDataStreamInfo("*"), hasSize(3)); assertNotNull(client.admin().indices().prepareGetIndex().setIndices(indexName).get()); } @@ -1321,7 +1307,7 @@ public void testRestoreDataStreamAliasWithConflictingDataStream() throws Excepti } } - public void testRestoreDataStreamAliasWithConflictingIndicesAlias() throws Exception { + public void testRestoreDataStreamAliasWithConflictingIndicesAlias() { var snapshotName = "test-snapshot"; createFullSnapshot(REPO, snapshotName); client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, "*")).actionGet(); @@ -1334,4 +1320,9 @@ public void testRestoreDataStreamAliasWithConflictingIndicesAlias() throws Excep ); assertThat(e.getMessage(), containsString("data stream alias and indices alias have the same name (my-alias)")); } + + protected List getDataStreamInfo(String... dataStreamNames) { + GetDataStreamAction.Request getRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, dataStreamNames); + return safeGet(client.execute(GetDataStreamAction.INSTANCE, getRequest)).getDataStreams(); + } } diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataTierDataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataTierDataStreamIT.java index 13b186d9da509..c08a3548127ec 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataTierDataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataTierDataStreamIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java index 18ba5f4bc1213..5e72de1a9d29a 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; @@ -23,6 +24,7 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.bytes.BytesArray; @@ -197,9 +199,7 @@ public void testRejectionFromFailureStore() throws IOException { // Initialize failure store. var rolloverRequest = new RolloverRequest(dataStream, null); rolloverRequest.setIndicesOptions( - IndicesOptions.builder(rolloverRequest.indicesOptions()) - .failureStoreOptions(opts -> opts.includeFailureIndices(true).includeRegularIndices(false)) - .build() + IndicesOptions.builder(rolloverRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.FAILURES).build() ); var rolloverResponse = client().execute(RolloverAction.INSTANCE, rolloverRequest).actionGet(); var failureStoreIndex = rolloverResponse.getNewIndex(); @@ -287,8 +287,8 @@ private void putComposableIndexTemplate(boolean failureStore) throws IOException request.indexTemplate( ComposableIndexTemplate.builder() .indexPatterns(List.of(dataStream + "*")) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false, failureStore)) - .template(new Template(null, new CompressedXContent(""" + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(Template.builder().mappings(new CompressedXContent(""" { "dynamic": false, "properties": { @@ -299,7 +299,7 @@ private void putComposableIndexTemplate(boolean failureStore) throws IOException "type": "long" } } - }"""), null)) + }""")).dataStreamOptions(DataStreamTestHelper.createDataStreamOptionsTemplate(failureStore))) .build() ); client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LazyRolloverDuringDisruptionIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LazyRolloverDuringDisruptionIT.java index 726f2c21cec7f..00dfd5c65b126 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LazyRolloverDuringDisruptionIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LazyRolloverDuringDisruptionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; @@ -17,17 +18,19 @@ import org.elasticsearch.action.datastreams.GetDataStreamAction; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.disruption.IntermittentLongGCDisruption; -import org.elasticsearch.test.disruption.SingleNodeDisruption; import org.elasticsearch.xcontent.XContentType; import java.util.Collection; import java.util.List; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutionException; import static org.hamcrest.Matchers.equalTo; @@ -42,7 +45,7 @@ protected Collection> nodePlugins() { } public void testRolloverIsExecutedOnce() throws ExecutionException, InterruptedException { - String masterNode = internalCluster().startMasterOnlyNode(); + internalCluster().startMasterOnlyNode(); internalCluster().startDataOnlyNodes(3); ensureStableCluster(4); @@ -50,7 +53,7 @@ public void testRolloverIsExecutedOnce() throws ExecutionException, InterruptedE createDataStream(dataStreamName); // Mark it to lazy rollover - new RolloverRequestBuilder(client()).setRolloverTarget(dataStreamName).lazy(true).execute().get(); + safeGet(new RolloverRequestBuilder(client()).setRolloverTarget(dataStreamName).lazy(true).execute()); // Verify that the data stream is marked for rollover and that it has currently one index DataStream dataStream = getDataStream(dataStreamName); @@ -58,9 +61,22 @@ public void testRolloverIsExecutedOnce() throws ExecutionException, InterruptedE assertThat(dataStream.getBackingIndices().getIndices().size(), equalTo(1)); // Introduce a disruption to the master node that should delay the rollover execution - SingleNodeDisruption masterNodeDisruption = new IntermittentLongGCDisruption(random(), masterNode, 100, 200, 30000, 60000); - internalCluster().setDisruptionScheme(masterNodeDisruption); - masterNodeDisruption.startDisrupting(); + final var barrier = new CyclicBarrier(2); + internalCluster().getCurrentMasterNodeInstance(ClusterService.class) + .submitUnbatchedStateUpdateTask("block", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) { + safeAwait(barrier); + safeAwait(barrier); + return currentState; + } + + @Override + public void onFailure(Exception e) { + fail(e); + } + }); + safeAwait(barrier); // Start indexing operations int docs = randomIntBetween(5, 10); @@ -83,10 +99,10 @@ public void onFailure(Exception e) { } // End the disruption so that all pending tasks will complete - masterNodeDisruption.stopDisrupting(); + safeAwait(barrier); // Wait for all the indexing requests to be processed successfully - countDownLatch.await(); + safeAwait(countDownLatch); // Verify that the rollover has happened once dataStream = getDataStream(dataStreamName); @@ -95,10 +111,12 @@ public void onFailure(Exception e) { } private DataStream getDataStream(String dataStreamName) { - return client().execute( - GetDataStreamAction.INSTANCE, - new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamName }) - ).actionGet().getDataStreams().get(0).getDataStream(); + return safeGet( + client().execute( + GetDataStreamAction.INSTANCE, + new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamName }) + ) + ).getDataStreams().get(0).getDataStream(); } private void createDataStream(String dataStreamName) throws InterruptedException, ExecutionException { @@ -110,10 +128,9 @@ private void createDataStream(String dataStreamName) throws InterruptedException .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) .build() ); - final AcknowledgedResponse putComposableTemplateResponse = client().execute( - TransportPutComposableIndexTemplateAction.TYPE, - putComposableTemplateRequest - ).actionGet(); + final AcknowledgedResponse putComposableTemplateResponse = safeGet( + client().execute(TransportPutComposableIndexTemplateAction.TYPE, putComposableTemplateRequest) + ); assertThat(putComposableTemplateResponse.isAcknowledged(), is(true)); final CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request( @@ -121,8 +138,9 @@ private void createDataStream(String dataStreamName) throws InterruptedException TEST_REQUEST_TIMEOUT, dataStreamName ); - final AcknowledgedResponse createDataStreamResponse = client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest) - .get(); + final AcknowledgedResponse createDataStreamResponse = safeGet( + client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest) + ); assertThat(createDataStreamResponse.isAcknowledged(), is(true)); } } diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java index 8db3ed6ddc9c9..cf8380a5c859c 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/ResolveClusterDataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/ResolveClusterDataStreamIT.java index 7fdc3b660433d..4c85958498da0 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/ResolveClusterDataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/ResolveClusterDataStreamIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; @@ -452,7 +453,7 @@ void putComposableIndexTemplate(Client client, String id, List patterns, request.indexTemplate( ComposableIndexTemplate.builder() .indexPatterns(patterns) - .template(new Template(null, null, aliases, null)) + .template(Template.builder().aliases(aliases)) .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) .build() ); diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamIT.java index a07081e8f0dd9..8f410a10376fa 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamSnapshotIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamSnapshotIT.java index ba9657f71b23e..2083807b1227f 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamSnapshotIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamSnapshotIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java index a0a0681dbd245..68049a750c3b9 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; @@ -19,6 +20,7 @@ import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.IndexDocFailureStoreStatus; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; @@ -169,7 +171,7 @@ public void testTimeRanges() throws Exception { var indexRequest = new IndexRequest("k8s").opType(DocWriteRequest.OpType.CREATE); time = randomBoolean() ? endTime : endTime.plusSeconds(randomIntBetween(1, 99)); indexRequest.source(DOC.replace("$time", formatInstant(time)), XContentType.JSON); - expectThrows(IllegalArgumentException.class, () -> client().index(indexRequest).actionGet()); + expectThrows(IndexDocFailureStoreStatus.ExceptionWithFailureStoreStatus.class, () -> client().index(indexRequest).actionGet()); } // Fetch UpdateTimeSeriesRangeService and increment time range of latest backing index: @@ -410,7 +412,7 @@ public void testSkippingShards() throws Exception { assertResponse(client().search(searchRequest), searchResponse -> { ElasticsearchAssertions.assertNoSearchHits(searchResponse); assertThat(searchResponse.getTotalShards(), equalTo(2)); - assertThat(searchResponse.getSkippedShards(), equalTo(1)); + assertThat(searchResponse.getSkippedShards(), equalTo(2)); assertThat(searchResponse.getSuccessfulShards(), equalTo(2)); }); } diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java index 64e4cc2ba9577..17e9cca07a054 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java index 27d8bdd2ac27c..8bb8405412295 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle; diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java index 9fafc23a388f4..dd3f1e74d4f4e 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle; @@ -200,12 +201,10 @@ public Collection getSystemDataStreamDescriptors() { ComposableIndexTemplate.builder() .indexPatterns(List.of(".test-data-stream")) .template( - new Template( - Settings.EMPTY, - mappings, - null, - DataStreamLifecycle.newBuilder().dataRetention(randomMillisUpToYear9999()).build() - ) + Template.builder() + .settings(Settings.EMPTY) + .mappings(mappings) + .lifecycle(DataStreamLifecycle.newBuilder().dataRetention(randomMillisUpToYear9999()).build()) ) .dataStreamTemplate(new DataStreamTemplate()) .build(), diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java index 65f911d27bf65..19067d85a6805 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle; @@ -39,6 +40,7 @@ import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamAction; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -344,7 +346,7 @@ public void testOriginationDate() throws Exception { request.indexTemplate( ComposableIndexTemplate.builder() .indexPatterns(List.of("index_*")) - .template(new Template(null, CompressedXContent.fromJSON(mapping), null, null)) + .template(Template.builder().mappings(CompressedXContent.fromJSON(mapping))) .build() ); client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); @@ -1220,9 +1222,15 @@ static void putComposableIndexTemplate( request.indexTemplate( ComposableIndexTemplate.builder() .indexPatterns(patterns) - .template(new Template(settings, mappings == null ? null : CompressedXContent.fromJSON(mappings), null, lifecycle)) + .template( + Template.builder() + .settings(settings) + .mappings(mappings == null ? null : CompressedXContent.fromJSON(mappings)) + .lifecycle(lifecycle) + .dataStreamOptions(DataStreamTestHelper.createDataStreamOptionsTemplate(withFailureStore)) + ) .metadata(metadata) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false, withFailureStore)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) .build() ); client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); @@ -1267,14 +1275,12 @@ public Collection getSystemDataStreamDescriptors() { .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) .indexPatterns(List.of(DataStream.BACKING_INDEX_PREFIX + SYSTEM_DATA_STREAM_NAME + "*")) .template( - new Template( - Settings.EMPTY, - null, - null, - DataStreamLifecycle.newBuilder() - .dataRetention(TimeValue.timeValueDays(SYSTEM_DATA_STREAM_RETENTION_DAYS)) - .build() - ) + Template.builder() + .settings(Settings.EMPTY) + .lifecycle( + DataStreamLifecycle.newBuilder() + .dataRetention(TimeValue.timeValueDays(SYSTEM_DATA_STREAM_RETENTION_DAYS)) + ) ) .build(), Map.of(), diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java index 5a5b451a62c3a..3c100d9cfe615 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle; @@ -444,7 +445,12 @@ static void putComposableIndexTemplate( request.indexTemplate( ComposableIndexTemplate.builder() .indexPatterns(patterns) - .template(new Template(settings, mappings == null ? null : CompressedXContent.fromJSON(mappings), null, lifecycle)) + .template( + Template.builder() + .settings(settings) + .mappings(mappings == null ? null : CompressedXContent.fromJSON(mappings)) + .lifecycle(lifecycle) + ) .metadata(metadata) .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) .build() diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/AbstractDataStreamIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/AbstractDataStreamIT.java index 027ac7c736c8a..01c63be448e62 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/AbstractDataStreamIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/AbstractDataStreamIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; @@ -43,6 +44,7 @@ public abstract class AbstractDataStreamIT extends ESRestTestCase { // Disable apm-data so the index templates it installs do not impact // tests such as testIgnoreDynamicBeyondLimit. .setting("xpack.apm_data.enabled", "false") + .setting("xpack.otel_data.registry.enabled", "false") .build(); protected RestClient client; diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/AutoCreateDataStreamIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/AutoCreateDataStreamIT.java index 4ffdc128f8ec0..77a277df484e5 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/AutoCreateDataStreamIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/AutoCreateDataStreamIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamOptionsIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamOptionsIT.java new file mode 100644 index 0000000000000..482867d072fc2 --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamOptionsIT.java @@ -0,0 +1,264 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.datastreams; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.RestStatus; +import org.junit.Before; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; + +/** + * This should be a yaml test, but in order to write one we would need to expose the new APIs in the rest-api-spec. + * We do not want to do that until the feature flag is removed. For this reason, we temporarily, test the new APIs here. + * Please convert this to a yaml test when the feature flag is removed. + */ +public class DataStreamOptionsIT extends DisabledSecurityDataStreamTestCase { + + private static final String DATA_STREAM_NAME = "failure-data-stream"; + + @SuppressWarnings("unchecked") + @Before + public void setup() throws IOException { + Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/ds-template"); + putComposableIndexTemplateRequest.setJsonEntity(""" + { + "index_patterns": ["failure-data-stream"], + "template": { + "settings": { + "number_of_replicas": 0 + }, + "data_stream_options": { + "failure_store": { + "enabled": true + } + } + }, + "data_stream": { + } + } + """); + assertOK(client().performRequest(putComposableIndexTemplateRequest)); + + assertOK(client().performRequest(new Request("PUT", "/_data_stream/" + DATA_STREAM_NAME))); + // Initialize the failure store. + assertOK(client().performRequest(new Request("POST", DATA_STREAM_NAME + "/_rollover?target_failure_store"))); + ensureGreen(DATA_STREAM_NAME); + + final Response dataStreamResponse = client().performRequest(new Request("GET", "/_data_stream/" + DATA_STREAM_NAME)); + List dataStreams = (List) entityAsMap(dataStreamResponse).get("data_streams"); + assertThat(dataStreams.size(), is(1)); + Map dataStream = (Map) dataStreams.get(0); + assertThat(dataStream.get("name"), equalTo(DATA_STREAM_NAME)); + assertThat(((Map) dataStream.get("failure_store")).get("enabled"), is(true)); + List backingIndices = getIndices(dataStream); + assertThat(backingIndices.size(), is(1)); + List failureStore = getFailureStore(dataStream); + assertThat(failureStore.size(), is(1)); + } + + public void testExplicitlyResetDataStreamOptions() throws IOException { + Request putComponentTemplateRequest = new Request("POST", "/_component_template/with-options"); + putComponentTemplateRequest.setJsonEntity(""" + { + "template": { + "data_stream_options": { + "failure_store": { + "enabled": true + } + } + } + } + """); + assertOK(client().performRequest(putComponentTemplateRequest)); + + Request invalidRequest = new Request("POST", "/_index_template/other-template"); + invalidRequest.setJsonEntity(""" + { + "index_patterns": ["something-else"], + "composed_of" : ["with-options"], + "template": { + "settings": { + "number_of_replicas": 0 + } + } + } + """); + Exception error = expectThrows(ResponseException.class, () -> client().performRequest(invalidRequest)); + assertThat( + error.getMessage(), + containsString("specifies data stream options that can only be used in combination with a data stream") + ); + + // Check that when we nullify the data stream options we can create use any component template in a non data stream template + Request otherRequest = new Request("POST", "/_index_template/other-template"); + otherRequest.setJsonEntity(""" + { + "index_patterns": ["something-else"], + "composed_of" : ["with-options"], + "template": { + "settings": { + "number_of_replicas": 0 + }, + "data_stream_options": null + } + } + """); + assertOK(client().performRequest(otherRequest)); + } + + public void testBehaviorWithEachFailureStoreOptionAndClusterSetting() throws IOException { + { + // Default data stream options + assertAcknowledged(client().performRequest(new Request("DELETE", "/_data_stream/" + DATA_STREAM_NAME + "/_options"))); + setDataStreamFailureStoreClusterSetting(DATA_STREAM_NAME); + assertDataStreamOptions(null); + assertFailureStoreValuesInGetDataStreamResponse(true, 1); + assertRedirectsDocWithBadMappingToFailureStore(); + setDataStreamFailureStoreClusterSetting("does-not-match-failure-data-stream"); + assertDataStreamOptions(null); + assertFailureStoreValuesInGetDataStreamResponse(false, 1); + assertFailsDocWithBadMapping(); + setDataStreamFailureStoreClusterSetting(null); // should get same behaviour as when we set it to something non-matching + assertDataStreamOptions(null); + assertFailureStoreValuesInGetDataStreamResponse(false, 1); + assertFailsDocWithBadMapping(); + } + { + // Data stream options with failure store enabled + Request enableRequest = new Request("PUT", "/_data_stream/" + DATA_STREAM_NAME + "/_options"); + enableRequest.setJsonEntity(""" + { + "failure_store": { + "enabled": true + } + }"""); + assertAcknowledged(client().performRequest(enableRequest)); + setDataStreamFailureStoreClusterSetting(DATA_STREAM_NAME); + assertDataStreamOptions(true); + assertFailureStoreValuesInGetDataStreamResponse(true, 1); + assertRedirectsDocWithBadMappingToFailureStore(); + setDataStreamFailureStoreClusterSetting("does-not-match-failure-data-stream"); // should have no effect as enabled in options + assertDataStreamOptions(true); + assertFailureStoreValuesInGetDataStreamResponse(true, 1); + assertRedirectsDocWithBadMappingToFailureStore(); + setDataStreamFailureStoreClusterSetting(null); // same as previous + assertDataStreamOptions(true); + assertFailureStoreValuesInGetDataStreamResponse(true, 1); + assertRedirectsDocWithBadMappingToFailureStore(); + } + { + // Data stream options with failure store disabled + Request disableRequest = new Request("PUT", "/_data_stream/" + DATA_STREAM_NAME + "/_options"); + disableRequest.setJsonEntity(""" + { + "failure_store": { + "enabled": false + } + }"""); + assertAcknowledged(client().performRequest(disableRequest)); + setDataStreamFailureStoreClusterSetting(DATA_STREAM_NAME); // should have no effect as disabled in options + assertDataStreamOptions(false); + assertFailureStoreValuesInGetDataStreamResponse(false, 1); + assertFailsDocWithBadMapping(); + setDataStreamFailureStoreClusterSetting("does-not-match-failure-data-stream"); + assertDataStreamOptions(false); + assertFailureStoreValuesInGetDataStreamResponse(false, 1); + assertFailsDocWithBadMapping(); + setDataStreamFailureStoreClusterSetting(null); + assertDataStreamOptions(false); + assertFailureStoreValuesInGetDataStreamResponse(false, 1); + assertFailsDocWithBadMapping(); + } + } + + @SuppressWarnings("unchecked") + private void assertFailureStoreValuesInGetDataStreamResponse(boolean failureStoreEnabled, int failureStoreSize) throws IOException { + final Response dataStreamResponse = client().performRequest(new Request("GET", "/_data_stream/" + DATA_STREAM_NAME)); + List dataStreams = (List) entityAsMap(dataStreamResponse).get("data_streams"); + assertThat(dataStreams.size(), is(1)); + Map dataStream = (Map) dataStreams.get(0); + assertThat(dataStream.get("name"), equalTo(DATA_STREAM_NAME)); + assertThat(dataStream.containsKey("failure_store"), is(true)); + // Ensure the failure store is set to the provided value + assertThat(((Map) dataStream.get("failure_store")).get("enabled"), equalTo(failureStoreEnabled)); + // And the failure indices preserved + List failureStore = getFailureStore(dataStream); + assertThat(failureStore.size(), is(failureStoreSize)); + } + + @SuppressWarnings("unchecked") + private void assertDataStreamOptions(Boolean failureStoreEnabled) throws IOException { + final Response dataStreamResponse = client().performRequest(new Request("GET", "/_data_stream/" + DATA_STREAM_NAME + "/_options")); + List dataStreams = (List) entityAsMap(dataStreamResponse).get("data_streams"); + assertThat(dataStreams.size(), is(1)); + Map dataStream = (Map) dataStreams.get(0); + assertThat(dataStream.get("name"), equalTo(DATA_STREAM_NAME)); + Map> options = (Map>) dataStream.get("options"); + if (failureStoreEnabled == null) { + assertThat(options, nullValue()); + } else { + assertThat(options.containsKey("failure_store"), is(true)); + assertThat(options.get("failure_store").get("enabled"), equalTo(failureStoreEnabled)); + } + } + + @SuppressWarnings("unchecked") + private List getFailureStore(Map response) { + var failureStore = (Map) response.get("failure_store"); + return getIndices(failureStore); + + } + + @SuppressWarnings("unchecked") + private List getIndices(Map response) { + List> indices = (List>) response.get("indices"); + return indices.stream().map(index -> index.get("index_name")).toList(); + } + + private static void setDataStreamFailureStoreClusterSetting(String value) throws IOException { + updateClusterSettings( + Settings.builder().put(DataStreamFailureStoreSettings.DATA_STREAM_FAILURE_STORED_ENABLED_SETTING.getKey(), value).build() + ); + } + + private Response putDocumentWithBadMapping() throws IOException { + Request request = new Request("POST", DATA_STREAM_NAME + "/_doc"); + request.setJsonEntity(""" + { + "@timestamp": "not a timestamp", + "foo": "bar" + } + """); + return client().performRequest(request); + } + + private void assertRedirectsDocWithBadMappingToFailureStore() throws IOException { + Response response = putDocumentWithBadMapping(); + String failureStoreResponse = (String) entityAsMap(response).get("failure_store"); + assertThat(failureStoreResponse, is("used")); + } + + private void assertFailsDocWithBadMapping() { + ResponseException e = assertThrows(ResponseException.class, this::putDocumentWithBadMapping); + assertThat(e.getResponse().getStatusLine().getStatusCode(), is(RestStatus.BAD_REQUEST.getStatus())); + } +} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamUpgradeRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamUpgradeRestIT.java index 39cdf77d04810..4ccffea00317c 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamUpgradeRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamUpgradeRestIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamWithSecurityIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamWithSecurityIT.java new file mode 100644 index 0000000000000..2ba373945ad50 --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamWithSecurityIT.java @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.datastreams; + +import org.apache.http.HttpHost; +import org.elasticsearch.client.Request; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.FeatureFlag; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.cluster.util.resource.Resource; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.ClassRule; + +public class DataStreamWithSecurityIT extends ESRestTestCase { + + private static final String PASSWORD = "secret-test-password"; + private static final String DATA_STREAM_NAME = "my-ds"; + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .feature(FeatureFlag.FAILURE_STORE_ENABLED) + .setting("xpack.watcher.enabled", "false") + .setting("xpack.ml.enabled", "false") + .setting("xpack.security.enabled", "true") + .setting("xpack.security.transport.ssl.enabled", "false") + .setting("xpack.security.http.ssl.enabled", "false") + .user("test_admin", PASSWORD, "superuser", false) + .user("limited_user", PASSWORD, "only_get", false) + .rolesFile(Resource.fromClasspath("roles.yml")) + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @Override + protected Settings restClientSettings() { + // If this test is running in a test framework that handles its own authorization, we don't want to overwrite it. + if (super.restClientSettings().keySet().contains(ThreadContext.PREFIX + ".Authorization")) { + return super.restClientSettings(); + } else { + // Note: We use the admin user because the other one is too unprivileged, so it breaks the initialization of the test + String token = basicAuthHeaderValue("test_admin", new SecureString(PASSWORD.toCharArray())); + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); + } + } + + private Settings simpleUserRestClientSettings() { + // Note: This user is assigned the role "only_get". That role is defined in roles.yml. + String token = basicAuthHeaderValue("limited_user", new SecureString(PASSWORD.toCharArray())); + return Settings.builder().put(super.restClientSettings()).put(ThreadContext.PREFIX + ".Authorization", token).build(); + } + + public void testGetDataStreamWithoutPermission() throws Exception { + Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/my-ds-template"); + putComposableIndexTemplateRequest.setJsonEntity(""" + { + "index_patterns": ["my-ds*"], + "data_stream": {} + } + """); + assertOK(adminClient().performRequest(putComposableIndexTemplateRequest)); + assertOK(adminClient().performRequest(new Request("PUT", "/_data_stream/" + DATA_STREAM_NAME))); + Request createDocRequest = new Request("POST", "/" + DATA_STREAM_NAME + "/_doc"); + createDocRequest.setJsonEntity("{ \"@timestamp\": \"2022-01-01\", \"message\": \"foo\" }"); + assertOK(adminClient().performRequest(createDocRequest)); + + // Both the verbose and non-verbose versions should work with the "simple" user + try (var simpleUserClient = buildClient(simpleUserRestClientSettings(), getClusterHosts().toArray(new HttpHost[0]))) { + Request getDs = new Request("GET", "/_data_stream"); + assertOK(simpleUserClient.performRequest(getDs)); + + Request getDsVerbose = new Request("GET", "/_data_stream?verbose=true"); + assertOK(simpleUserClient.performRequest(getDsVerbose)); + } + } + +} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamsRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamsRestIT.java index a60d36b0460a5..ff842d435463f 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamsRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamsRestIT.java @@ -1,13 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.xcontent.support.XContentMapValues; @@ -70,10 +72,12 @@ public void testHiddenDataStreamImplicitHiddenSearch() throws IOException { // Create a template Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/hidden"); putComposableIndexTemplateRequest.setJsonEntity("{\"index_patterns\": [\".hidden\"], \"data_stream\": {\"hidden\": true}}"); + putComposableIndexTemplateRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic")); assertOK(client().performRequest(putComposableIndexTemplateRequest)); Request createDocRequest = new Request("POST", "/.hidden/_doc?refresh=true"); createDocRequest.setJsonEntity("{ \"@timestamp\": \"2020-10-22\", \"a\": 1 }"); + createDocRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic")); assertOK(client().performRequest(createDocRequest)); diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DisabledSecurityDataStreamTestCase.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DisabledSecurityDataStreamTestCase.java index ae33c06b497db..9839f9abb080e 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DisabledSecurityDataStreamTestCase.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DisabledSecurityDataStreamTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/EcsLogsDataStreamIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/EcsLogsDataStreamIT.java index e43b1e451c312..f0fc3337d92d0 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/EcsLogsDataStreamIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/EcsLogsDataStreamIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/FailureStoreQueryParamIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/FailureStoreQueryParamIT.java index 4af3a3844e453..85b914be30b2c 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/FailureStoreQueryParamIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/FailureStoreQueryParamIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; @@ -41,10 +42,14 @@ public void setup() throws IOException { "template": { "settings": { "number_of_replicas": 0 + }, + "data_stream_options": { + "failure_store": { + "enabled": true + } } }, "data_stream": { - "failure_store": true } } """); diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LazyRolloverDataStreamIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LazyRolloverDataStreamIT.java index 1f2a5b2508606..7230331e21b14 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LazyRolloverDataStreamIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LazyRolloverDataStreamIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java index 9ab32f29f4a79..9a3d3bd2fc6e4 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamRestIT.java index 1500674e3aee9..4b9b29d7187e1 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamRestIT.java @@ -1,23 +1,31 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.FormatNames; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.repositories.fs.FsRepository; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.rest.ESRestTestCase; +import org.hamcrest.Matchers; import org.junit.Before; import org.junit.ClassRule; @@ -40,6 +48,7 @@ public class LogsDataStreamRestIT extends ESRestTestCase { public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .distribution(DistributionType.DEFAULT) .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial") .build(); @Override @@ -64,7 +73,7 @@ private static void waitForLogs(RestClient client) throws Exception { }); } - private static final String LOGS_TEMPLATE = """ + static final String LOGS_TEMPLATE = """ { "index_patterns": [ "logs-*-*" ], "data_stream": {}, @@ -101,7 +110,7 @@ private static void waitForLogs(RestClient client) throws Exception { } }"""; - private static final String STANDARD_TEMPLATE = """ + static final String LOGS_STANDARD_INDEX_MODE = """ { "index_patterns": [ "logs-*-*" ], "data_stream": {}, @@ -134,6 +143,39 @@ private static void waitForLogs(RestClient client) throws Exception { } }"""; + static final String STANDARD_TEMPLATE = """ + { + "index_patterns": [ "standard-*-*" ], + "data_stream": {}, + "priority": 201, + "template": { + "settings": { + "index": { + "mode": "standard" + } + }, + "mappings": { + "properties": { + "@timestamp" : { + "type": "date" + }, + "host.name": { + "type": "keyword" + }, + "pid": { + "type": "long" + }, + "method": { + "type": "keyword" + }, + "ip_address": { + "type": "ip" + } + } + } + } + }"""; + private static final String TIME_SERIES_TEMPLATE = """ { "index_patterns": [ "logs-*-*" ], @@ -174,7 +216,7 @@ private static void waitForLogs(RestClient client) throws Exception { } }"""; - private static final String DOC_TEMPLATE = """ + static final String DOC_TEMPLATE = """ { "@timestamp": "%s", "host.name": "%s", @@ -202,7 +244,7 @@ public void testLogsIndexing() throws IOException { randomLongBetween(1_000_000L, 2_000_000L) ) ); - assertDataStreamBackingIndexMode("logsdb", 0); + assertDataStreamBackingIndexMode("logsdb", 0, DATA_STREAM_NAME); rolloverDataStream(client, DATA_STREAM_NAME); indexDocument( client, @@ -217,7 +259,7 @@ public void testLogsIndexing() throws IOException { randomLongBetween(1_000_000L, 2_000_000L) ) ); - assertDataStreamBackingIndexMode("logsdb", 1); + assertDataStreamBackingIndexMode("logsdb", 1, DATA_STREAM_NAME); } public void testLogsStandardIndexModeSwitch() throws IOException { @@ -236,9 +278,9 @@ public void testLogsStandardIndexModeSwitch() throws IOException { randomLongBetween(1_000_000L, 2_000_000L) ) ); - assertDataStreamBackingIndexMode("logsdb", 0); + assertDataStreamBackingIndexMode("logsdb", 0, DATA_STREAM_NAME); - putTemplate(client, "custom-template", STANDARD_TEMPLATE); + putTemplate(client, "custom-template", LOGS_STANDARD_INDEX_MODE); rolloverDataStream(client, DATA_STREAM_NAME); indexDocument( client, @@ -253,7 +295,7 @@ public void testLogsStandardIndexModeSwitch() throws IOException { randomLongBetween(1_000_000L, 2_000_000L) ) ); - assertDataStreamBackingIndexMode("standard", 1); + assertDataStreamBackingIndexMode("standard", 1, DATA_STREAM_NAME); putTemplate(client, "custom-template", LOGS_TEMPLATE); rolloverDataStream(client, DATA_STREAM_NAME); @@ -270,7 +312,7 @@ public void testLogsStandardIndexModeSwitch() throws IOException { randomLongBetween(1_000_000L, 2_000_000L) ) ); - assertDataStreamBackingIndexMode("logsdb", 2); + assertDataStreamBackingIndexMode("logsdb", 2, DATA_STREAM_NAME); } public void testLogsTimeSeriesIndexModeSwitch() throws IOException { @@ -289,7 +331,24 @@ public void testLogsTimeSeriesIndexModeSwitch() throws IOException { randomLongBetween(1_000_000L, 2_000_000L) ) ); - assertDataStreamBackingIndexMode("logsdb", 0); + assertDataStreamBackingIndexMode("logsdb", 0, DATA_STREAM_NAME); + + putTemplate(client, "custom-template", LOGS_STANDARD_INDEX_MODE); + rolloverDataStream(client, DATA_STREAM_NAME); + indexDocument( + client, + DATA_STREAM_NAME, + document( + Instant.now(), + randomAlphaOfLength(10), + randomNonNegativeLong(), + randomFrom("PUT", "POST", "GET"), + randomAlphaOfLength(64), + randomIp(randomBoolean()), + randomLongBetween(1_000_000L, 2_000_000L) + ) + ); + assertDataStreamBackingIndexMode("standard", 1, DATA_STREAM_NAME); putTemplate(client, "custom-template", TIME_SERIES_TEMPLATE); rolloverDataStream(client, DATA_STREAM_NAME); @@ -306,7 +365,24 @@ public void testLogsTimeSeriesIndexModeSwitch() throws IOException { randomLongBetween(1_000_000L, 2_000_000L) ) ); - assertDataStreamBackingIndexMode("time_series", 1); + assertDataStreamBackingIndexMode("time_series", 2, DATA_STREAM_NAME); + + putTemplate(client, "custom-template", LOGS_STANDARD_INDEX_MODE); + rolloverDataStream(client, DATA_STREAM_NAME); + indexDocument( + client, + DATA_STREAM_NAME, + document( + Instant.now(), + randomAlphaOfLength(10), + randomNonNegativeLong(), + randomFrom("PUT", "POST", "GET"), + randomAlphaOfLength(64), + randomIp(randomBoolean()), + randomLongBetween(1_000_000L, 2_000_000L) + ) + ); + assertDataStreamBackingIndexMode("standard", 3, DATA_STREAM_NAME); putTemplate(client, "custom-template", LOGS_TEMPLATE); rolloverDataStream(client, DATA_STREAM_NAME); @@ -323,14 +399,196 @@ public void testLogsTimeSeriesIndexModeSwitch() throws IOException { randomLongBetween(1_000_000L, 2_000_000L) ) ); - assertDataStreamBackingIndexMode("logsdb", 2); + assertDataStreamBackingIndexMode("logsdb", 4, DATA_STREAM_NAME); } - private void assertDataStreamBackingIndexMode(final String indexMode, int backingIndex) throws IOException { - assertThat(getSettings(client, getWriteBackingIndex(client, DATA_STREAM_NAME, backingIndex)).get("index.mode"), is(indexMode)); + public void testLogsDBToStandardReindex() throws IOException { + // LogsDB data stream + putTemplate(client, "logs-template", LOGS_TEMPLATE); + createDataStream(client, "logs-apache-kafka"); + + // Standard data stream + putTemplate(client, "standard-template", STANDARD_TEMPLATE); + createDataStream(client, "standard-apache-kafka"); + + // Index some documents in the LogsDB index + for (int i = 0; i < 10; i++) { + indexDocument( + client, + "logs-apache-kafka", + document( + Instant.now().plusSeconds(10), + randomAlphaOfLength(10), + randomNonNegativeLong(), + randomFrom("PUT", "POST", "GET"), + randomAlphaOfLength(64), + randomIp(randomBoolean()), + randomLongBetween(1_000_000L, 2_000_000L) + ) + ); + } + assertDataStreamBackingIndexMode("logsdb", 0, "logs-apache-kafka"); + assertDocCount(client, "logs-apache-kafka", 10); + + // Reindex a LogsDB data stream into a standard data stream + final Request reindexRequest = new Request("POST", "/_reindex?refresh=true"); + reindexRequest.setJsonEntity(""" + { + "source": { + "index": "logs-apache-kafka" + }, + "dest": { + "index": "standard-apache-kafka", + "op_type": "create" + } + } + """); + assertOK(client.performRequest(reindexRequest)); + assertDataStreamBackingIndexMode("standard", 0, "standard-apache-kafka"); + assertDocCount(client, "standard-apache-kafka", 10); } - private String document( + public void testStandardToLogsDBReindex() throws IOException { + // LogsDB data stream + putTemplate(client, "logs-template", LOGS_TEMPLATE); + createDataStream(client, "logs-apache-kafka"); + + // Standard data stream + putTemplate(client, "standard-template", STANDARD_TEMPLATE); + createDataStream(client, "standard-apache-kafka"); + + // Index some documents in a standard index + for (int i = 0; i < 10; i++) { + indexDocument( + client, + "standard-apache-kafka", + document( + Instant.now().plusSeconds(10), + randomAlphaOfLength(10), + randomNonNegativeLong(), + randomFrom("PUT", "POST", "GET"), + randomAlphaOfLength(64), + randomIp(randomBoolean()), + randomLongBetween(1_000_000L, 2_000_000L) + ) + ); + } + assertDataStreamBackingIndexMode("standard", 0, "standard-apache-kafka"); + assertDocCount(client, "standard-apache-kafka", 10); + + // Reindex a standard data stream into a LogsDB data stream + final Request reindexRequest = new Request("POST", "/_reindex?refresh=true"); + reindexRequest.setJsonEntity(""" + { + "source": { + "index": "standard-apache-kafka" + }, + "dest": { + "index": "logs-apache-kafka", + "op_type": "create" + } + } + """); + assertOK(client.performRequest(reindexRequest)); + assertDataStreamBackingIndexMode("logsdb", 0, "logs-apache-kafka"); + assertDocCount(client, "logs-apache-kafka", 10); + } + + public void testLogsDBSnapshotCreateRestoreMount() throws IOException { + final String repository = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + registerRepository(repository, FsRepository.TYPE, Settings.builder().put("location", randomAlphaOfLength(6))); + + final String index = randomAlphaOfLength(12).toLowerCase(Locale.ROOT); + createIndex(client, index, Settings.builder().put("index.mode", IndexMode.LOGSDB.getName()).build()); + + for (int i = 0; i < 10; i++) { + indexDocument( + client, + index, + document( + Instant.now().plusSeconds(10), + randomAlphaOfLength(10), + randomNonNegativeLong(), + randomFrom("PUT", "POST", "GET"), + randomAlphaOfLength(64), + randomIp(randomBoolean()), + randomLongBetween(1_000_000L, 2_000_000L) + ) + ); + } + + final String snapshot = randomAlphaOfLength(8).toLowerCase(Locale.ROOT); + deleteSnapshot(repository, snapshot, true); + createSnapshot(client, repository, snapshot, true, index); + wipeDataStreams(); + wipeAllIndices(); + restoreSnapshot(client, repository, snapshot, true, index); + + final String restoreIndex = randomAlphaOfLength(7).toLowerCase(Locale.ROOT); + final Request mountRequest = new Request("POST", "/_snapshot/" + repository + '/' + snapshot + "/_mount"); + mountRequest.addParameter("wait_for_completion", "true"); + mountRequest.setJsonEntity("{\"index\": \"" + index + "\",\"renamed_index\": \"" + restoreIndex + "\"}"); + + assertOK(client.performRequest(mountRequest)); + assertDocCount(client, restoreIndex, 10); + assertThat(getSettings(client, restoreIndex).get("index.mode"), Matchers.equalTo(IndexMode.LOGSDB.getName())); + } + + // NOTE: this test will fail on snapshot creation after fixing + // https://github.com/elastic/elasticsearch/issues/112735 + public void testLogsDBSourceOnlySnapshotCreation() throws IOException { + final String repository = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + registerRepository(repository, FsRepository.TYPE, Settings.builder().put("location", randomAlphaOfLength(6))); + // A source-only repository delegates storage to another repository + final String sourceOnlyRepository = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + registerRepository( + sourceOnlyRepository, + "source", + Settings.builder().put("delegate_type", FsRepository.TYPE).put("location", repository) + ); + + final String index = randomAlphaOfLength(12).toLowerCase(Locale.ROOT); + createIndex(client, index, Settings.builder().put("index.mode", IndexMode.LOGSDB.getName()).build()); + + for (int i = 0; i < 10; i++) { + indexDocument( + client, + index, + document( + Instant.now().plusSeconds(10), + randomAlphaOfLength(10), + randomNonNegativeLong(), + randomFrom("PUT", "POST", "GET"), + randomAlphaOfLength(64), + randomIp(randomBoolean()), + randomLongBetween(1_000_000L, 2_000_000L) + ) + ); + } + + final String snapshot = randomAlphaOfLength(8).toLowerCase(Locale.ROOT); + deleteSnapshot(sourceOnlyRepository, snapshot, true); + createSnapshot(client, sourceOnlyRepository, snapshot, true, index); + wipeDataStreams(); + wipeAllIndices(); + // Can't snapshot _source only on an index that has incomplete source ie. has _source disabled or filters the source + final ResponseException responseException = expectThrows( + ResponseException.class, + () -> restoreSnapshot(client, sourceOnlyRepository, snapshot, true, index) + ); + assertThat(responseException.getMessage(), Matchers.containsString("wasn't fully snapshotted")); + } + + private static void registerRepository(final String repository, final String type, final Settings.Builder settings) throws IOException { + registerRepository(repository, type, false, settings.build()); + } + + private void assertDataStreamBackingIndexMode(final String indexMode, int backingIndex, final String dataStreamName) + throws IOException { + assertThat(getSettings(client, getWriteBackingIndex(client, dataStreamName, backingIndex)).get("index.mode"), is(indexMode)); + } + + static String document( final Instant timestamp, final String hostname, long pid, @@ -357,14 +615,14 @@ private static void createDataStream(final RestClient client, final String dataS assertOK(client.performRequest(request)); } - private static void putTemplate(final RestClient client, final String templateName, final String mappings) throws IOException { + static void putTemplate(final RestClient client, final String templateName, final String mappings) throws IOException { final Request request = new Request("PUT", "/_index_template/" + templateName); request.setJsonEntity(mappings); assertOK(client.performRequest(request)); } - private static void indexDocument(final RestClient client, String dataStreamName, String doc) throws IOException { - final Request request = new Request("POST", "/" + dataStreamName + "/_doc?refresh=true"); + static void indexDocument(final RestClient client, String indexOrtDataStream, String doc) throws IOException { + final Request request = new Request("POST", "/" + indexOrtDataStream + "/_doc?refresh=true"); request.setJsonEntity(doc); final Response response = client.performRequest(request); assertOK(response); @@ -392,4 +650,46 @@ private static Map getSettings(final RestClient client, final St final Request request = new Request("GET", "/" + indexName + "/_settings?flat_settings"); return ((Map>) entityAsMap(client.performRequest(request)).get(indexName)).get("settings"); } + + private static void createSnapshot( + RestClient restClient, + String repository, + String snapshot, + boolean waitForCompletion, + final String... indices + ) throws IOException { + final Request request = new Request(HttpPut.METHOD_NAME, "_snapshot/" + repository + '/' + snapshot); + request.addParameter("wait_for_completion", Boolean.toString(waitForCompletion)); + request.setJsonEntity(""" + "indices": $indices + """.replace("$indices", String.join(", ", indices))); + + final Response response = restClient.performRequest(request); + assertThat( + "Failed to create snapshot [" + snapshot + "] in repository [" + repository + "]: " + response, + response.getStatusLine().getStatusCode(), + equalTo(RestStatus.OK.getStatus()) + ); + } + + private static void restoreSnapshot( + final RestClient client, + final String repository, + String snapshot, + boolean waitForCompletion, + final String... indices + ) throws IOException { + final Request request = new Request(HttpPost.METHOD_NAME, "_snapshot/" + repository + '/' + snapshot + "/_restore"); + request.addParameter("wait_for_completion", Boolean.toString(waitForCompletion)); + request.setJsonEntity(""" + "indices": $indices + """.replace("$indices", String.join(", ", indices))); + + final Response response = client.performRequest(request); + assertThat( + "Failed to restore snapshot [" + snapshot + "] from repository [" + repository + "]: " + response, + response.getStatusLine().getStatusCode(), + equalTo(RestStatus.OK.getStatus()) + ); + } } diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/MetricsDataStreamIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/MetricsDataStreamIT.java index 6cc300378a312..848e53503493c 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/MetricsDataStreamIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/MetricsDataStreamIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/MultiClustersIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/MultiClustersIT.java new file mode 100644 index 0000000000000..83df25f3f9ba9 --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/MultiClustersIT.java @@ -0,0 +1,363 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.datastreams; + +import org.apache.http.HttpHost; +import org.apache.lucene.tests.util.English; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.Randomness; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.function.Predicate; +import java.util.stream.Stream; + +import static org.elasticsearch.datastreams.AbstractDataStreamIT.createDataStream; +import static org.elasticsearch.datastreams.LogsDataStreamRestIT.LOGS_TEMPLATE; +import static org.elasticsearch.datastreams.LogsDataStreamRestIT.STANDARD_TEMPLATE; +import static org.elasticsearch.datastreams.LogsDataStreamRestIT.indexDocument; +import static org.elasticsearch.datastreams.LogsDataStreamRestIT.putTemplate; +import static org.elasticsearch.index.mapper.DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; + +public class MultiClustersIT extends ESRestTestCase { + static List localLogsDocs = null; + static List remoteLogsDocs = null; + static List localStandardDocs = null; + static List remoteStandardDocs = null; + + public static ElasticsearchCluster remoteCluster = ElasticsearchCluster.local() + .name("remote_cluster") + .distribution(DistributionType.DEFAULT) + .module("data-streams") + .module("x-pack-stack") + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial") + .setting("cluster.logsdb.enabled", "true") + .build(); + + public static ElasticsearchCluster localCluster = ElasticsearchCluster.local() + .name("local_cluster") + .distribution(DistributionType.DEFAULT) + .module("data-streams") + .module("x-pack-stack") + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial") + .setting("cluster.logsdb.enabled", "true") + .setting("node.roles", "[data,ingest,master,remote_cluster_client]") + .setting("cluster.remote.remote_cluster.seeds", () -> "\"" + remoteCluster.getTransportEndpoint(0) + "\"") + .setting("cluster.remote.connections_per_cluster", "1") + .setting("cluster.remote.remote_cluster.skip_unavailable", "false") + .build(); + + @ClassRule + public static TestRule clusterRule = RuleChain.outerRule(remoteCluster).around(localCluster); + + private RestClient localClusterClient() throws IOException { + var clusterHosts = parseClusterHosts(localCluster.getHttpAddresses()); + return buildClient(restClientSettings(), clusterHosts.toArray(new HttpHost[0])); + } + + private RestClient remoteClusterClient() throws IOException { + var clusterHosts = parseClusterHosts(remoteCluster.getHttpAddresses()); + return buildClient(restClientSettings(), clusterHosts.toArray(new HttpHost[0])); + } + + private record Document(long timestamp, String cluster, String hostName, long pid, String method, long messageId, String message) { + + @SuppressWarnings("unchecked") + static Document fromHit(Map hit) { + long timestamp = DEFAULT_DATE_TIME_FORMATTER.parseMillis(hit.get("@timestamp").toString()); + String cluster = (String) hit.get("cluster"); + String hostName = (String) hit.get("host.name"); + if (hostName == null) { + Map host = (Map) hit.get("host"); + hostName = (String) host.get("name"); + } + long pid = ((Number) hit.get("pid")).longValue(); + String method = (String) hit.get("method"); + long messageId = ((Number) hit.get("message_id")).longValue(); + String message = (String) hit.get("message"); + return new Document(timestamp, cluster, hostName, pid, method, messageId, message); + } + + String toJson() throws IOException { + XContentBuilder builder = JsonXContent.contentBuilder() + .startObject() + .field("@timestamp", timestamp) + .field("cluster", cluster) + .field("host.name", hostName) + .field("pid", pid) + .field("method", method) + .field("message_id", messageId) + .field("message", message) + .endObject(); + return Strings.toString(builder); + } + } + + static String randomHostName() { + return randomFrom("qa-", "staging-", "prod-") + between(1, 3); + } + + static List indexDocuments(RestClient client, String cluster, String index, int startMessageId) throws IOException { + int numDocs = between(0, 100); + List docs = new ArrayList<>(numDocs); + long timestamp = DEFAULT_DATE_TIME_FORMATTER.parseMillis("2024-09-15T00:00:00Z"); + for (int i = 0; i < numDocs; i++) { + timestamp += between(0, 5) * 1000L; + long pid = randomLongBetween(1, 10); + String method = randomFrom("GET", "PUT", "POST", "DELETE"); + String message = English.intToEnglish(between(1, 1000000)); + docs.add(new Document(timestamp, cluster, randomHostName(), pid, method, startMessageId + i, message)); + } + Randomness.shuffle(docs); + for (Document doc : docs) { + indexDocument(client, index, doc.toJson()); + if (rarely()) { + refresh(client, index); + } + } + refresh(client, index); + return docs; + } + + @Before + public void setUpIndices() throws Exception { + if (localLogsDocs != null) { + return; + } + try (RestClient client = localClusterClient()) { + putTemplate(client, "logs-template", LOGS_TEMPLATE); + putTemplate(client, "standard-template", STANDARD_TEMPLATE); + + createDataStream(client, "logs-apache-kafka"); + localLogsDocs = indexDocuments(client, "local", "logs-apache-kafka", 0); + assertDocCount(client, "logs-apache-kafka", localLogsDocs.size()); + + createDataStream(client, "standard-apache-kafka"); + localStandardDocs = indexDocuments(client, "local", "standard-apache-kafka", 1000); + assertDocCount(client, "standard-apache-kafka", localStandardDocs.size()); + } + try (RestClient client = remoteClusterClient()) { + putTemplate(client, "logs-template", LOGS_TEMPLATE); + putTemplate(client, "standard-template", STANDARD_TEMPLATE); + + createDataStream(client, "logs-apache-kafka"); + remoteLogsDocs = indexDocuments(client, "remote", "logs-apache-kafka", 2000); + assertDocCount(client, "logs-apache-kafka", remoteLogsDocs.size()); + + createDataStream(client, "standard-apache-kafka"); + remoteStandardDocs = indexDocuments(client, "remote", "standard-apache-kafka", 3000); + assertDocCount(client, "standard-apache-kafka", remoteStandardDocs.size()); + } + } + + public void testSource() throws IOException { + XContentBuilder searchSource = JsonXContent.contentBuilder().startObject().field("_source", true).field("size", 500); + final boolean sorted = randomBoolean(); + if (sorted) { + searchSource.startArray("sort"); + searchSource.value("message_id"); + searchSource.endArray(); + } + final Predicate filterHost; + if (randomBoolean()) { + String host = randomHostName(); + filterHost = s -> s.equals(host); + searchSource.startObject("query"); + searchSource.startObject("term"); + searchSource.startObject("host.name"); + searchSource.field("value", host); + searchSource.endObject(); + searchSource.endObject(); + searchSource.endObject(); + } else { + filterHost = s -> true; + } + searchSource.endObject(); + // remote only + { + var request = new Request("POST", "/*:l*,*:s*/_search"); + request.setJsonEntity(Strings.toString(searchSource)); + if (randomBoolean()) { + request.addParameter("ccs_minimize_roundtrips", Boolean.toString(randomBoolean())); + } + Response resp = client().performRequest(request); + assertOK(resp); + Stream hits = extractHits(resp).stream().map(Document::fromHit); + if (sorted == false) { + hits = hits.sorted(Comparator.comparingLong(Document::messageId)); + } + var expectedHits = Stream.of(remoteLogsDocs, remoteStandardDocs) + .flatMap(Collection::stream) + .filter(d -> filterHost.test(d.hostName)) + .sorted(Comparator.comparingLong(Document::messageId)) + .toList(); + assertThat(hits.toList(), equalTo(expectedHits)); + } + // both clusters + { + var request = new Request("POST", "/*,*:*/_search"); + request.setJsonEntity(Strings.toString(searchSource)); + if (randomBoolean()) { + request.addParameter("ccs_minimize_roundtrips", Boolean.toString(randomBoolean())); + } + Response resp = client().performRequest(request); + assertOK(resp); + Stream hits = extractHits(resp).stream().map(Document::fromHit); + if (sorted == false) { + hits = hits.sorted(Comparator.comparingLong(Document::messageId)); + } + var expectedHits = Stream.of(localLogsDocs, localStandardDocs, remoteLogsDocs, remoteStandardDocs) + .flatMap(Collection::stream) + .filter(d -> filterHost.test(d.hostName)) + .sorted(Comparator.comparingLong(Document::messageId)) + .toList(); + assertThat(hits.toList(), equalTo(expectedHits)); + } + + } + + public void testFields() throws IOException { + XContentBuilder searchSource = JsonXContent.contentBuilder() + .startObject() + .array("fields", "message_id", "host.name") + .field("size", 500); + final boolean sorted = randomBoolean(); + if (sorted) { + searchSource.startArray("sort"); + searchSource.value("message_id"); + searchSource.endArray(); + } + final Predicate filterHost; + if (randomBoolean()) { + String host = randomHostName(); + filterHost = s -> s.equals(host); + searchSource.startObject("query"); + searchSource.startObject("term"); + searchSource.startObject("host.name"); + searchSource.field("value", host); + searchSource.endObject(); + searchSource.endObject(); + searchSource.endObject(); + } else { + filterHost = s -> true; + } + searchSource.endObject(); + record Fields(long messageId, String hostName) { + @SuppressWarnings("unchecked") + static Fields fromResponse(Map hit) { + List hostName = (List) hit.get("host.name"); + assertThat(hostName, hasSize(1)); + List messageId = (List) hit.get("message_id"); + assertThat(messageId, hasSize(1)); + return new Fields(messageId.get(0).longValue(), hostName.get(0)); + } + } + // remote only + { + var request = new Request("POST", "/*:l*,*:s*/_search"); + request.setJsonEntity(Strings.toString(searchSource)); + if (randomBoolean()) { + request.addParameter("ccs_minimize_roundtrips", Boolean.toString(randomBoolean())); + } + Response resp = client().performRequest(request); + assertOK(resp); + Stream hits = extractFields(resp).stream().map(Fields::fromResponse); + if (sorted == false) { + hits = hits.sorted(Comparator.comparingLong(Fields::messageId)); + } + var expectedHits = Stream.of(remoteLogsDocs, remoteStandardDocs) + .flatMap(Collection::stream) + .filter(d -> filterHost.test(d.hostName)) + .map(d -> new Fields(d.messageId, d.hostName)) + .sorted(Comparator.comparingLong(Fields::messageId)) + .toList(); + assertThat(hits.toList(), equalTo(expectedHits)); + } + // both clusters + { + var request = new Request("POST", "/*,*:*/_search"); + request.setJsonEntity(Strings.toString(searchSource)); + if (randomBoolean()) { + request.addParameter("ccs_minimize_roundtrips", Boolean.toString(randomBoolean())); + } + Response resp = client().performRequest(request); + assertOK(resp); + Stream hits = extractFields(resp).stream().map(Fields::fromResponse); + if (sorted == false) { + hits = hits.sorted(Comparator.comparingLong(Fields::messageId)); + } + var expectedHits = Stream.of(localLogsDocs, localStandardDocs, remoteLogsDocs, remoteStandardDocs) + .flatMap(Collection::stream) + .filter(d -> filterHost.test(d.hostName)) + .map(d -> new Fields(d.messageId, d.hostName)) + .sorted(Comparator.comparingLong(Fields::messageId)) + .toList(); + assertThat(hits.toList(), equalTo(expectedHits)); + } + } + + @SuppressWarnings("unchecked") + private static List> extractHits(final Response response) throws IOException { + final Map map = XContentHelper.convertToMap(XContentType.JSON.xContent(), response.getEntity().getContent(), true); + final Map hitsMap = (Map) map.get("hits"); + final List> hitsList = (List>) hitsMap.get("hits"); + return hitsList.stream().map(hit -> (Map) hit.get("_source")).toList(); + } + + @SuppressWarnings("unchecked") + private static List> extractFields(final Response response) throws IOException { + final Map map = XContentHelper.convertToMap(XContentType.JSON.xContent(), response.getEntity().getContent(), true); + final Map hitsMap = (Map) map.get("hits"); + final List> hitsList = (List>) hitsMap.get("hits"); + return hitsList.stream().map(hit -> (Map) hit.get("fields")).toList(); + } + + @Override + protected String getTestRestCluster() { + return localCluster.getHttpAddresses(); + } + + @Override + protected boolean preserveIndicesUponCompletion() { + return true; + } + + @Override + protected boolean preserveClusterUponCompletion() { + return true; + } + + @Override + protected boolean preserveDataStreamsUponCompletion() { + return true; + } +} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java index e8fb6c8b064c4..9be0c18d18213 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; @@ -24,7 +25,6 @@ import java.util.Set; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.backingIndexEqualTo; -import static org.hamcrest.Matchers.aMapWithSize; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; @@ -53,6 +53,7 @@ public class TsdbDataStreamRestIT extends DisabledSecurityDataStreamTestCase { "number_of_replicas": 1, "number_of_shards": 2, "mode": "time_series" + SOURCEMODE } }, "mappings":{ @@ -200,15 +201,35 @@ public class TsdbDataStreamRestIT extends DisabledSecurityDataStreamTestCase { {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "elephant", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876eb4", "ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}} """; + private static String getTemplate() { + return TEMPLATE.replace("SOURCEMODE", randomFrom("", """ + , "mapping": { "source": { "mode": "stored" } }""", """ + , "mapping": { "source": { "mode": "synthetic" } }""")); + } + + private static boolean trialStarted = false; + @Before public void setup() throws IOException { + if (trialStarted == false) { + // Start trial to support synthetic source. + Request startTrial = new Request("POST", "/_license/start_trial"); + startTrial.addParameter("acknowledge", "true"); + try { + client().performRequest(startTrial); + } catch (Exception e) { + // Ignore failures, the API is not present in Serverless. + } + trialStarted = true; + } + // Add component template: var request = new Request("POST", "/_component_template/custom_template"); request.setJsonEntity(COMPONENT_TEMPLATE); assertOK(client().performRequest(request)); // Add composable index template request = new Request("POST", "/_index_template/1"); - request.setJsonEntity(TEMPLATE); + request.setJsonEntity(getTemplate()); assertOK(client().performRequest(request)); } @@ -219,7 +240,7 @@ public void testTsdbDataStreams() throws Exception { public void testTsdbDataStreamsNanos() throws Exception { // Overwrite template to use date_nanos field type: var putComposableIndexTemplateRequest = new Request("POST", "/_index_template/1"); - putComposableIndexTemplateRequest.setJsonEntity(TEMPLATE.replace("date", "date_nanos")); + putComposableIndexTemplateRequest.setJsonEntity(getTemplate().replace("date", "date_nanos")); assertOK(client().performRequest(putComposableIndexTemplateRequest)); assertTsdbDataStream(); @@ -406,7 +427,6 @@ public void testSimulateTsdbDataStreamTemplate() throws Exception { var response = client().performRequest(simulateIndexTemplateRequest); assertOK(response); var responseBody = entityAsMap(response); - assertThat(ObjectPath.evaluate(responseBody, "template.settings.index"), aMapWithSize(6)); assertThat(ObjectPath.evaluate(responseBody, "template.settings.index.number_of_shards"), equalTo("2")); assertThat(ObjectPath.evaluate(responseBody, "template.settings.index.number_of_replicas"), equalTo("1")); assertThat(ObjectPath.evaluate(responseBody, "template.settings.index.mode"), equalTo("time_series")); @@ -492,7 +512,7 @@ public void testMigrateRegularDataStreamToTsdbDataStream() throws Exception { // Update template putComposableIndexTemplateRequest = new Request("POST", "/_index_template/1"); - putComposableIndexTemplateRequest.setJsonEntity(TEMPLATE); + putComposableIndexTemplateRequest.setJsonEntity(getTemplate()); assertOK(client().performRequest(putComposableIndexTemplateRequest)); var rolloverRequest = new Request("POST", "/k8s/_rollover"); diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamDeleteLifecycleWithPermissionsRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamDeleteLifecycleWithPermissionsRestIT.java index d074027b0e474..68196b0011c1b 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamDeleteLifecycleWithPermissionsRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamDeleteLifecycleWithPermissionsRestIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle; diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamGlobalRetentionIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamGlobalRetentionIT.java index 514eb6d8742ea..c8cb14c1ef481 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamGlobalRetentionIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamGlobalRetentionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle; diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecyclePermissionsTestCase.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecyclePermissionsTestCase.java index cd21a31d343c7..ae854c9d1a915 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecyclePermissionsTestCase.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecyclePermissionsTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle; diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleStatsIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleStatsIT.java index cce9132d99d19..c82b08b87300b 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleStatsIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleStatsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle; diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamUpdateLifecycleWithPermissionsRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamUpdateLifecycleWithPermissionsRestIT.java index 98c90bb992e21..a279ce90c1ced 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamUpdateLifecycleWithPermissionsRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamUpdateLifecycleWithPermissionsRestIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle; diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java deleted file mode 100644 index f0279702812c4..0000000000000 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java +++ /dev/null @@ -1,299 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.datastreams.logsdb; - -import org.elasticsearch.client.Request; -import org.elasticsearch.client.ResponseException; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.test.cluster.ElasticsearchCluster; -import org.elasticsearch.test.cluster.local.distribution.DistributionType; -import org.junit.Before; -import org.junit.ClassRule; - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.function.Function; - -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; - -@SuppressWarnings("unchecked") -public class LogsIndexModeCustomSettingsIT extends LogsIndexModeRestTestIT { - @ClassRule() - public static ElasticsearchCluster cluster = ElasticsearchCluster.local() - .distribution(DistributionType.DEFAULT) - .module("constant-keyword") - .module("data-streams") - .module("mapper-extras") - .module("x-pack-aggregate-metric") - .module("x-pack-stack") - .setting("xpack.security.enabled", "false") - .setting("xpack.license.self_generated.type", "trial") - .setting("cluster.logsdb.enabled", "true") - .build(); - - @Override - protected String getTestRestCluster() { - return cluster.getHttpAddresses(); - } - - @Before - public void setup() throws Exception { - client = client(); - waitForLogs(client); - } - - private RestClient client; - - public void testOverrideIndexSorting() throws IOException { - var indexSortOverrideTemplate = """ - { - "template": { - "settings": { - "index": { - "sort.field": ["cluster", "custom.timestamp"], - "sort.order":["desc", "asc"] - } - }, - "mappings": { - "properties": { - "cluster": { - "type": "keyword" - }, - "custom": { - "properties": { - "timestamp": { - "type": "date" - } - } - } - } - } - } - }"""; - - assertOK(putComponentTemplate(client, "logs@custom", indexSortOverrideTemplate)); - assertOK(createDataStream(client, "logs-custom-dev")); - - var indexSortField = (List) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.sort.field"); - assertThat(indexSortField, equalTo(List.of("cluster", "custom.timestamp"))); - - var indexSortOrder = (List) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.sort.order"); - assertThat(indexSortOrder, equalTo(List.of("desc", "asc"))); - - // @timestamp is a default mapping and should still be present - var mapping = getMapping(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0)); - String type = (String) subObject("properties").andThen(subObject("@timestamp")).apply(mapping).get("type"); - assertThat(type, equalTo("date")); - } - - public void testConfigureStoredSource() throws IOException { - var storedSourceMapping = """ - { - "template": { - "mappings": { - "_source": { - "mode": "stored" - } - } - } - }"""; - - Exception e = assertThrows(ResponseException.class, () -> putComponentTemplate(client, "logs@custom", storedSourceMapping)); - assertThat( - e.getMessage(), - containsString("updating component template [logs@custom] results in invalid composable template [logs]") - ); - assertThat(e.getMessage(), containsString("Indices with with index mode [logsdb] only support synthetic source")); - - assertOK(createDataStream(client, "logs-custom-dev")); - - var mapping = getMapping(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0)); - String sourceMode = (String) subObject("_source").apply(mapping).get("mode"); - assertThat(sourceMode, equalTo("synthetic")); - } - - public void testOverrideIndexCodec() throws IOException { - var indexCodecOverrideTemplate = """ - { - "template": { - "settings": { - "index": { - "codec": "default" - } - } - } - }"""; - - assertOK(putComponentTemplate(client, "logs@custom", indexCodecOverrideTemplate)); - assertOK(createDataStream(client, "logs-custom-dev")); - - var indexCodec = (String) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.codec"); - assertThat(indexCodec, equalTo("default")); - } - - public void testOverrideTimestampField() throws IOException { - var timestampMappingOverrideTemplate = """ - { - "template": { - "mappings": { - "properties": { - "@timestamp": { - "type": "date_nanos" - } - } - } - } - }"""; - - assertOK(putComponentTemplate(client, "logs@custom", timestampMappingOverrideTemplate)); - assertOK(createDataStream(client, "logs-custom-dev")); - - var mapping = getMapping(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0)); - String type = (String) subObject("properties").andThen(subObject("@timestamp")).apply(mapping).get("type"); - assertThat(type, equalTo("date_nanos")); - } - - public void testOverrideHostNameField() throws IOException { - var timestampMappingOverrideTemplate = """ - { - "template": { - "mappings": { - "properties": { - "host": { - "properties": { - "name": { - "type": "keyword", - "index": false, - "ignore_above": 10 - } - } - } - } - } - } - }"""; - - assertOK(putComponentTemplate(client, "logs@custom", timestampMappingOverrideTemplate)); - assertOK(createDataStream(client, "logs-custom-dev")); - - var mapping = getMapping(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0)); - var hostNameFieldParameters = subObject("properties").andThen(subObject("host")) - .andThen(subObject("properties")) - .andThen(subObject("name")) - .apply(mapping); - assertThat((String) hostNameFieldParameters.get("type"), equalTo("keyword")); - assertThat((boolean) hostNameFieldParameters.get("index"), equalTo(false)); - assertThat((int) hostNameFieldParameters.get("ignore_above"), equalTo(10)); - } - - public void testOverrideIndexSortingWithCustomTimestampField() throws IOException { - var timestampMappingAndIndexSortOverrideTemplate = """ - { - "template": { - "settings": { - "index": { - "sort.field": ["@timestamp", "cluster"], - "sort.order":["asc", "asc"] - } - }, - "mappings": { - "properties": { - "@timestamp": { - "type": "date_nanos" - }, - "cluster": { - "type": "keyword" - } - } - } - } - }"""; - - assertOK(putComponentTemplate(client, "logs@custom", timestampMappingAndIndexSortOverrideTemplate)); - assertOK(createDataStream(client, "logs-custom-dev")); - - var mapping = getMapping(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0)); - String type = (String) subObject("properties").andThen(subObject("@timestamp")).apply(mapping).get("type"); - assertThat(type, equalTo("date_nanos")); - - var indexSortField = (List) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.sort.field"); - assertThat(indexSortField, equalTo(List.of("@timestamp", "cluster"))); - - var indexSortOrder = (List) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.sort.order"); - assertThat(indexSortOrder, equalTo(List.of("asc", "asc"))); - } - - public void testOverrideIgnoreMalformed() throws IOException { - var ignoreMalformedOverrideTemplate = """ - { - "template": { - "settings": { - "index": { - "mapping": { - "ignore_malformed": false - } - } - } - } - }"""; - - assertOK(putComponentTemplate(client, "logs@custom", ignoreMalformedOverrideTemplate)); - assertOK(createDataStream(client, "logs-custom-dev")); - - var ignoreMalformedIndexSetting = (String) getSetting( - client, - getDataStreamBackingIndex(client, "logs-custom-dev", 0), - "index.mapping.ignore_malformed" - ); - assertThat(ignoreMalformedIndexSetting, equalTo("false")); - } - - public void testOverrideIgnoreDynamicBeyondLimit() throws IOException { - var ignoreMalformedOverrideTemplate = """ - { - "template": { - "settings": { - "index": { - "mapping": { - "total_fields": { - "ignore_dynamic_beyond_limit": false - } - } - } - } - } - }"""; - - assertOK(putComponentTemplate(client, "logs@custom", ignoreMalformedOverrideTemplate)); - assertOK(createDataStream(client, "logs-custom-dev")); - - var ignoreDynamicBeyondLimitIndexSetting = (String) getSetting( - client, - getDataStreamBackingIndex(client, "logs-custom-dev", 0), - "index.mapping.total_fields.ignore_dynamic_beyond_limit" - ); - assertThat(ignoreDynamicBeyondLimitIndexSetting, equalTo("false")); - } - - private static Map getMapping(final RestClient client, final String indexName) throws IOException { - final Request request = new Request("GET", "/" + indexName + "/_mapping"); - - Map mappings = ((Map>) entityAsMap(client.performRequest(request)).get(indexName)).get( - "mappings" - ); - - return mappings; - } - - private Function> subObject(String key) { - return (mapAsObject) -> (Map) ((Map) mapAsObject).get(key); - } -} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java deleted file mode 100644 index fada21224e3b2..0000000000000 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.datastreams.logsdb; - -import org.elasticsearch.client.RestClient; -import org.elasticsearch.index.IndexMode; -import org.elasticsearch.test.cluster.ElasticsearchCluster; -import org.elasticsearch.test.cluster.local.distribution.DistributionType; -import org.hamcrest.Matchers; -import org.junit.Before; -import org.junit.ClassRule; - -import java.io.IOException; - -import static org.hamcrest.Matchers.equalTo; - -public class LogsIndexModeDisabledRestTestIT extends LogsIndexModeRestTestIT { - - @ClassRule() - public static ElasticsearchCluster cluster = ElasticsearchCluster.local() - .distribution(DistributionType.DEFAULT) - .module("constant-keyword") - .module("data-streams") - .module("mapper-extras") - .module("x-pack-aggregate-metric") - .module("x-pack-stack") - .setting("xpack.security.enabled", "false") - .setting("xpack.license.self_generated.type", "trial") - .build(); - - @Override - protected String getTestRestCluster() { - return cluster.getHttpAddresses(); - } - - @Before - public void setup() throws Exception { - client = client(); - waitForLogs(client); - } - - private RestClient client; - - public void testLogsSettingsIndexModeDisabled() throws IOException { - assertOK(createDataStream(client, "logs-custom-dev")); - final String indexMode = (String) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.mode"); - assertThat(indexMode, Matchers.not(equalTo(IndexMode.LOGSDB.getName()))); - } - -} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java deleted file mode 100644 index a4277748ea9bd..0000000000000 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java +++ /dev/null @@ -1,226 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.datastreams.logsdb; - -import org.elasticsearch.client.Response; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.index.IndexMode; -import org.elasticsearch.test.cluster.ElasticsearchCluster; -import org.elasticsearch.test.cluster.local.distribution.DistributionType; -import org.hamcrest.Matchers; -import org.junit.Before; -import org.junit.ClassRule; - -import java.io.IOException; - -import static org.hamcrest.Matchers.equalTo; - -public class LogsIndexModeEnabledRestTestIT extends LogsIndexModeRestTestIT { - - @ClassRule() - public static ElasticsearchCluster cluster = ElasticsearchCluster.local() - .distribution(DistributionType.DEFAULT) - .module("constant-keyword") - .module("data-streams") - .module("mapper-extras") - .module("x-pack-aggregate-metric") - .module("x-pack-stack") - .setting("xpack.security.enabled", "false") - .setting("xpack.license.self_generated.type", "trial") - .setting("cluster.logsdb.enabled", "true") - .build(); - - @Override - protected String getTestRestCluster() { - return cluster.getHttpAddresses(); - } - - @Before - public void setup() throws Exception { - client = client(); - waitForLogs(client); - } - - private RestClient client; - - private static final String MAPPINGS = """ - { - "template": { - "mappings": { - "properties": { - "method": { - "type": "keyword" - }, - "message": { - "type": "text" - } - } - } - } - }"""; - - private static final String ALTERNATE_HOST_MAPPING = """ - { - "template": { - "mappings": { - "properties": { - "method": { - "type": "keyword" - }, - "message": { - "type": "text" - }, - "host.cloud_region": { - "type": "keyword" - }, - "host.availability_zone": { - "type": "keyword" - } - } - } - } - }"""; - - private static final String HOST_MAPPING_AS_OBJECT_DEFAULT_SUBOBJECTS = """ - { - "template": { - "mappings": { - "properties": { - "method": { - "type": "keyword" - }, - "message": { - "type": "text" - }, - "host": { - "type": "object", - "properties": { - "cloud_region": { - "type": "keyword" - }, - "availability_zone": { - "type": "keyword" - }, - "name": { - "type": "keyword" - } - } - } - } - } - } - }"""; - - private static final String HOST_MAPPING_AS_OBJECT_NON_DEFAULT_SUBOBJECTS = """ - { - "template": { - "mappings": { - "dynamic": "strict", - "properties": { - "method": { - "type": "keyword" - }, - "message": { - "type": "text" - }, - "host": { - "type": "object", - "subobjects": false, - "properties": { - "cloud_region": { - "type": "keyword" - }, - "availability_zone": { - "type": "keyword" - }, - "name": { - "type": "keyword" - } - } - } - } - } - } - }"""; - - private static String BULK_INDEX_REQUEST = """ - { "create": {}} - { "@timestamp": "2023-01-01T05:11:00Z", "host.name": "foo", "method" : "PUT", "message": "foo put message" } - { "create": {}} - { "@timestamp": "2023-01-01T05:12:00Z", "host.name": "bar", "method" : "POST", "message": "bar post message" } - { "create": {}} - { "@timestamp": "2023-01-01T05:12:00Z", "host.name": "baz", "method" : "PUT", "message": "baz put message" } - { "create": {}} - { "@timestamp": "2023-01-01T05:13:00Z", "host.name": "baz", "method" : "PUT", "message": "baz put message" } - """; - - private static String BULK_INDEX_REQUEST_WITH_HOST = """ - { "create": {}} - { "@timestamp": "2023-01-01T05:11:00Z", "method" : "PUT", "message": "foo put message", \ - "host": { "cloud_region" : "us-west", "availability_zone" : "us-west-4a", "name" : "ahdta-876584" } } - { "create": {}} - { "@timestamp": "2023-01-01T05:12:00Z", "method" : "POST", "message": "bar post message", \ - "host": { "cloud_region" : "us-west", "availability_zone" : "us-west-4b", "name" : "tyrou-447898" } } - { "create": {}} - { "@timestamp": "2023-01-01T05:12:00Z", "method" : "PUT", "message": "baz put message", \ - "host": { "cloud_region" : "us-west", "availability_zone" : "us-west-4a", "name" : "uuopl-162899" } } - { "create": {}} - { "@timestamp": "2023-01-01T05:13:00Z", "method" : "PUT", "message": "baz put message", \ - "host": { "cloud_region" : "us-west", "availability_zone" : "us-west-4b", "name" : "fdfgf-881197" } } - """; - - public void testCreateDataStream() throws IOException { - assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); - assertOK(createDataStream(client, "logs-custom-dev")); - final String indexMode = (String) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.mode"); - assertThat(indexMode, equalTo(IndexMode.LOGSDB.getName())); - } - - public void testBulkIndexing() throws IOException { - assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); - assertOK(createDataStream(client, "logs-custom-dev")); - final Response response = bulkIndex(client, "logs-custom-dev", () -> BULK_INDEX_REQUEST); - assertOK(response); - assertThat(entityAsMap(response).get("errors"), Matchers.equalTo(false)); - } - - public void testBulkIndexingWithFlatHostProperties() throws IOException { - assertOK(putComponentTemplate(client, "logs@custom", ALTERNATE_HOST_MAPPING)); - assertOK(createDataStream(client, "logs-custom-dev")); - final Response response = bulkIndex(client, "logs-custom-dev", () -> BULK_INDEX_REQUEST_WITH_HOST); - assertOK(response); - assertThat(entityAsMap(response).get("errors"), Matchers.equalTo(false)); - } - - public void testBulkIndexingWithObjectHostDefaultSubobjectsProperties() throws IOException { - assertOK(putComponentTemplate(client, "logs@custom", HOST_MAPPING_AS_OBJECT_DEFAULT_SUBOBJECTS)); - assertOK(createDataStream(client, "logs-custom-dev")); - final Response response = bulkIndex(client, "logs-custom-dev", () -> BULK_INDEX_REQUEST_WITH_HOST); - assertOK(response); - assertThat(entityAsMap(response).get("errors"), Matchers.equalTo(false)); - } - - public void testBulkIndexingWithObjectHostSubobjectsFalseProperties() throws IOException { - assertOK(putComponentTemplate(client, "logs@custom", HOST_MAPPING_AS_OBJECT_NON_DEFAULT_SUBOBJECTS)); - assertOK(createDataStream(client, "logs-custom-dev")); - final Response response = bulkIndex(client, "logs-custom-dev", () -> BULK_INDEX_REQUEST_WITH_HOST); - assertOK(response); - assertThat(entityAsMap(response).get("errors"), Matchers.equalTo(false)); - } - - public void testRolloverDataStream() throws IOException { - assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); - assertOK(createDataStream(client, "logs-custom-dev")); - final String firstBackingIndex = getDataStreamBackingIndex(client, "logs-custom-dev", 0); - assertOK(rolloverDataStream(client, "logs-custom-dev")); - final String secondBackingIndex = getDataStreamBackingIndex(client, "logs-custom-dev", 1); - assertThat(firstBackingIndex, Matchers.not(equalTo(secondBackingIndex))); - assertThat(getDataStreamBackingIndices(client, "logs-custom-dev").size(), equalTo(2)); - } -} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java deleted file mode 100644 index ff45096146280..0000000000000 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.datastreams.logsdb; - -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.ResponseException; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.test.rest.ESRestTestCase; - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -public abstract class LogsIndexModeRestTestIT extends ESRestTestCase { - protected static void waitForLogs(RestClient client) throws Exception { - assertBusy(() -> { - try { - final Request request = new Request("GET", "_index_template/logs"); - assertOK(client.performRequest(request)); - } catch (ResponseException e) { - fail(e.getMessage()); - } - }); - } - - protected static Response putComponentTemplate(final RestClient client, final String templateName, final String mappings) - throws IOException { - final Request request = new Request("PUT", "/_component_template/" + templateName); - request.setJsonEntity(mappings); - return client.performRequest(request); - } - - protected static Response createDataStream(final RestClient client, final String dataStreamName) throws IOException { - return client.performRequest(new Request("PUT", "_data_stream/" + dataStreamName)); - } - - protected static Response rolloverDataStream(final RestClient client, final String dataStreamName) throws IOException { - return client.performRequest(new Request("POST", "/" + dataStreamName + "/_rollover")); - } - - @SuppressWarnings("unchecked") - protected static String getDataStreamBackingIndex(final RestClient client, final String dataStreamName, int backingIndex) - throws IOException { - final Request request = new Request("GET", "_data_stream/" + dataStreamName); - final List dataStreams = (List) entityAsMap(client.performRequest(request)).get("data_streams"); - final Map dataStream = (Map) dataStreams.get(0); - final List> backingIndices = (List>) dataStream.get("indices"); - return backingIndices.get(backingIndex).get("index_name"); - } - - @SuppressWarnings("unchecked") - protected static List getDataStreamBackingIndices(final RestClient client, final String dataStreamName) throws IOException { - final Request request = new Request("GET", "_data_stream/" + dataStreamName); - final List dataStreams = (List) entityAsMap(client.performRequest(request)).get("data_streams"); - final Map dataStream = (Map) dataStreams.get(0); - final List> backingIndices = (List>) dataStream.get("indices"); - return backingIndices.stream().map(map -> map.get("indices")).collect(Collectors.toList()); - } - - @SuppressWarnings("unchecked") - protected static Object getSetting(final RestClient client, final String indexName, final String setting) throws IOException { - final Request request = new Request("GET", "/" + indexName + "/_settings?flat_settings=true&include_defaults=true"); - final Map settings = ((Map>) entityAsMap(client.performRequest(request)).get(indexName)) - .get("settings"); - - return settings.get(setting); - } - - protected static Response bulkIndex(final RestClient client, final String dataStreamName, final Supplier bulkSupplier) - throws IOException { - var bulkRequest = new Request("POST", "/" + dataStreamName + "/_bulk"); - bulkRequest.setJsonEntity(bulkSupplier.get()); - bulkRequest.addParameter("refresh", "true"); - return client.performRequest(bulkRequest); - } -} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java deleted file mode 100644 index 9bf1c394f9105..0000000000000 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java +++ /dev/null @@ -1,332 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.datastreams.logsdb.qa; - -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.ResponseException; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.time.DateFormatter; -import org.elasticsearch.common.time.FormatNames; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.core.Tuple; -import org.elasticsearch.datastreams.logsdb.qa.matchers.MatchResult; -import org.elasticsearch.datastreams.logsdb.qa.matchers.Matcher; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; -import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; -import org.elasticsearch.xcontent.XContentType; -import org.hamcrest.Matchers; - -import java.io.IOException; -import java.time.Instant; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Map; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; - -/** - * Basic challenge test - we index same documents into an index with standard index mode and an index with logsdb index mode. - * Then we verify that results of common operations are the same modulo knows differences like synthetic source modifications. - * This test uses simple mapping and document structure in order to allow easier debugging of the test itself. - */ -public class StandardVersusLogsIndexModeChallengeRestIT extends AbstractChallengeRestTest { - private final int numShards = randomBoolean() ? randomIntBetween(2, 5) : 0; - private final int numReplicas = randomBoolean() ? randomIntBetween(1, 3) : 0; - private final boolean fullyDynamicMapping = randomBoolean(); - - public StandardVersusLogsIndexModeChallengeRestIT() { - super("standard-apache-baseline", "logs-apache-contender", "baseline-template", "contender-template", 101, 101); - } - - @Override - public void baselineMappings(XContentBuilder builder) throws IOException { - if (fullyDynamicMapping == false) { - builder.startObject() - .startObject("properties") - - .startObject("@timestamp") - .field("type", "date") - .endObject() - - .startObject("host.name") - .field("type", "keyword") - .field("ignore_above", randomIntBetween(1000, 1200)) - .endObject() - - .startObject("message") - .field("type", "keyword") - .field("ignore_above", randomIntBetween(1000, 1200)) - .endObject() - - .startObject("method") - .field("type", "keyword") - .field("ignore_above", randomIntBetween(1000, 1200)) - .endObject() - - .startObject("memory_usage_bytes") - .field("type", "long") - .field("ignore_malformed", randomBoolean()) - .endObject() - - .endObject() - - .endObject(); - } else { - // We want dynamic mapping, but we need host.name to be a keyword instead of text to support aggregations. - builder.startObject() - .startObject("properties") - - .startObject("host.name") - .field("type", "keyword") - .field("ignore_above", randomIntBetween(1000, 1200)) - .endObject() - - .endObject() - .endObject(); - } - } - - @Override - public void contenderMappings(XContentBuilder builder) throws IOException { - builder.startObject(); - builder.field("subobjects", false); - - if (fullyDynamicMapping == false) { - builder.startObject("properties") - - .startObject("@timestamp") - .field("type", "date") - .endObject() - - .startObject("host.name") - .field("type", "keyword") - .field("ignore_above", randomIntBetween(1000, 1200)) - .endObject() - - .startObject("message") - .field("type", "keyword") - .field("ignore_above", randomIntBetween(1000, 1200)) - .endObject() - - .startObject("method") - .field("type", "keyword") - .field("ignore_above", randomIntBetween(1000, 1200)) - .endObject() - - .startObject("memory_usage_bytes") - .field("type", "long") - .field("ignore_malformed", randomBoolean()) - .endObject() - - .endObject(); - } - - builder.endObject(); - } - - @Override - public void commonSettings(Settings.Builder builder) { - if (numShards > 0) { - builder.put("index.number_of_shards", numShards); - } - if (numReplicas > 0) { - builder.put("index.number_of_replicas", numReplicas); - } - builder.put("index.mapping.total_fields.limit", 5000); - } - - @Override - public void contenderSettings(Settings.Builder builder) { - builder.put("index.mode", "logsdb"); - } - - @Override - public void baselineSettings(Settings.Builder builder) {} - - @Override - public void beforeStart() throws Exception { - waitForLogs(client()); - } - - protected static void waitForLogs(RestClient client) throws Exception { - assertBusy(() -> { - try { - final Request request = new Request("GET", "_index_template/logs"); - assertOK(client.performRequest(request)); - } catch (ResponseException e) { - fail(e.getMessage()); - } - }); - } - - public void testMatchAllQuery() throws IOException { - int numberOfDocuments = ESTestCase.randomIntBetween(100, 200); - final List documents = generateDocuments(numberOfDocuments); - - assertDocumentIndexing(documents); - - final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()) - .size(numberOfDocuments); - - final MatchResult matchResult = Matcher.matchSource() - .mappings(getContenderMappings(), getBaselineMappings()) - .settings(getContenderSettings(), getBaselineSettings()) - .expected(getQueryHits(queryBaseline(searchSourceBuilder))) - .ignoringSort(true) - .isEqualTo(getQueryHits(queryContender(searchSourceBuilder))); - assertTrue(matchResult.getMessage(), matchResult.isMatch()); - } - - public void testTermsQuery() throws IOException { - int numberOfDocuments = ESTestCase.randomIntBetween(100, 200); - final List documents = generateDocuments(numberOfDocuments); - - assertDocumentIndexing(documents); - - final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.termQuery("method", "put")) - .size(numberOfDocuments); - - final MatchResult matchResult = Matcher.matchSource() - .mappings(getContenderMappings(), getBaselineMappings()) - .settings(getContenderSettings(), getBaselineSettings()) - .expected(getQueryHits(queryBaseline(searchSourceBuilder))) - .ignoringSort(true) - .isEqualTo(getQueryHits(queryContender(searchSourceBuilder))); - assertTrue(matchResult.getMessage(), matchResult.isMatch()); - } - - public void testHistogramAggregation() throws IOException { - int numberOfDocuments = ESTestCase.randomIntBetween(100, 200); - final List documents = generateDocuments(numberOfDocuments); - - assertDocumentIndexing(documents); - - final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()) - .size(numberOfDocuments) - .aggregation(new HistogramAggregationBuilder("agg").field("memory_usage_bytes").interval(100.0D)); - - final MatchResult matchResult = Matcher.mappings(getContenderMappings(), getBaselineMappings()) - .settings(getContenderSettings(), getBaselineSettings()) - .expected(getAggregationBuckets(queryBaseline(searchSourceBuilder), "agg")) - .ignoringSort(true) - .isEqualTo(getAggregationBuckets(queryContender(searchSourceBuilder), "agg")); - assertTrue(matchResult.getMessage(), matchResult.isMatch()); - } - - public void testTermsAggregation() throws IOException { - int numberOfDocuments = ESTestCase.randomIntBetween(100, 200); - final List documents = generateDocuments(numberOfDocuments); - - assertDocumentIndexing(documents); - - final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()) - .size(0) - .aggregation(new TermsAggregationBuilder("agg").field("host.name").size(numberOfDocuments)); - - final MatchResult matchResult = Matcher.mappings(getContenderMappings(), getBaselineMappings()) - .settings(getContenderSettings(), getBaselineSettings()) - .expected(getAggregationBuckets(queryBaseline(searchSourceBuilder), "agg")) - .ignoringSort(true) - .isEqualTo(getAggregationBuckets(queryContender(searchSourceBuilder), "agg")); - assertTrue(matchResult.getMessage(), matchResult.isMatch()); - } - - public void testDateHistogramAggregation() throws IOException { - int numberOfDocuments = ESTestCase.randomIntBetween(100, 200); - final List documents = generateDocuments(numberOfDocuments); - - assertDocumentIndexing(documents); - - final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()) - .aggregation(AggregationBuilders.dateHistogram("agg").field("@timestamp").calendarInterval(DateHistogramInterval.SECOND)) - .size(0); - - final MatchResult matchResult = Matcher.mappings(getContenderMappings(), getBaselineMappings()) - .settings(getContenderSettings(), getBaselineSettings()) - .expected(getAggregationBuckets(queryBaseline(searchSourceBuilder), "agg")) - .ignoringSort(true) - .isEqualTo(getAggregationBuckets(queryContender(searchSourceBuilder), "agg")); - assertTrue(matchResult.getMessage(), matchResult.isMatch()); - } - - private List generateDocuments(int numberOfDocuments) throws IOException { - final List documents = new ArrayList<>(); - // This is static in order to be able to identify documents between test runs. - var startingPoint = ZonedDateTime.of(2024, 1, 1, 10, 0, 0, 0, ZoneId.of("UTC")).toInstant(); - for (int i = 0; i < numberOfDocuments; i++) { - documents.add(generateDocument(startingPoint.plus(i, ChronoUnit.SECONDS))); - } - - return documents; - } - - protected XContentBuilder generateDocument(final Instant timestamp) throws IOException { - return XContentFactory.jsonBuilder() - .startObject() - .field("@timestamp", DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(timestamp)) - .field("host.name", randomFrom("foo", "bar", "baz")) - .field("message", randomFrom("a message", "another message", "still another message", "one more message")) - .field("method", randomFrom("put", "post", "get")) - .field("memory_usage_bytes", randomLongBetween(1000, 2000)) - .endObject(); - } - - @SuppressWarnings("unchecked") - private static List> getQueryHits(final Response response) throws IOException { - final Map map = XContentHelper.convertToMap(XContentType.JSON.xContent(), response.getEntity().getContent(), true); - final Map hitsMap = (Map) map.get("hits"); - - final List> hitsList = (List>) hitsMap.get("hits"); - assertThat(hitsList.size(), greaterThan(0)); - - return hitsList.stream() - .sorted(Comparator.comparingInt((Map hit) -> Integer.parseInt((String) hit.get("_id")))) - .map(hit -> (Map) hit.get("_source")) - .toList(); - } - - @SuppressWarnings("unchecked") - private static List> getAggregationBuckets(final Response response, final String aggName) throws IOException { - final Map map = XContentHelper.convertToMap(XContentType.JSON.xContent(), response.getEntity().getContent(), true); - final Map aggs = (Map) map.get("aggregations"); - final Map agg = (Map) aggs.get(aggName); - - var buckets = (List>) agg.get("buckets"); - assertThat(buckets.size(), greaterThan(0)); - - return buckets; - } - - private void assertDocumentIndexing(List documents) throws IOException { - final Tuple tuple = indexDocuments(() -> documents, () -> documents); - - assertThat(tuple.v1().getStatusLine().getStatusCode(), Matchers.equalTo(RestStatus.OK.getStatus())); - var baselineResponseBody = entityAsMap(tuple.v1()); - assertThat("errors in baseline bulk response:\n " + baselineResponseBody, baselineResponseBody.get("errors"), equalTo(false)); - - assertThat(tuple.v2().getStatusLine().getStatusCode(), Matchers.equalTo(RestStatus.OK.getStatus())); - var contenderResponseBody = entityAsMap(tuple.v2()); - assertThat("errors in contender bulk response:\n " + contenderResponseBody, contenderResponseBody.get("errors"), equalTo(false)); - } -} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java deleted file mode 100644 index 404914cda6a7e..0000000000000 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.datastreams.logsdb.qa; - -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.time.DateFormatter; -import org.elasticsearch.common.time.FormatNames; -import org.elasticsearch.core.CheckedConsumer; -import org.elasticsearch.index.mapper.Mapper; -import org.elasticsearch.index.mapper.ObjectMapper; -import org.elasticsearch.logsdb.datageneration.DataGenerator; -import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification; -import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; -import org.elasticsearch.logsdb.datageneration.datasource.DataSourceHandler; -import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; -import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; -import org.elasticsearch.logsdb.datageneration.fields.PredefinedField; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; - -import java.io.IOException; -import java.time.Instant; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Challenge test (see {@link StandardVersusLogsIndexModeChallengeRestIT}) that uses randomly generated - * mapping and documents in order to cover more code paths and permutations. - */ -public class StandardVersusLogsIndexModeRandomDataChallengeRestIT extends StandardVersusLogsIndexModeChallengeRestIT { - private final ObjectMapper.Subobjects subobjects; - private final boolean keepArraySource; - - private final DataGenerator dataGenerator; - - public StandardVersusLogsIndexModeRandomDataChallengeRestIT() { - super(); - this.subobjects = randomFrom(ObjectMapper.Subobjects.values()); - this.keepArraySource = randomBoolean(); - - var specificationBuilder = DataGeneratorSpecification.builder().withFullyDynamicMapping(randomBoolean()); - if (subobjects != ObjectMapper.Subobjects.ENABLED) { - specificationBuilder = specificationBuilder.withNestedFieldsLimit(0); - } - this.dataGenerator = new DataGenerator(specificationBuilder.withDataSourceHandlers(List.of(new DataSourceHandler() { - @Override - public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequest.ObjectMappingParametersGenerator request) { - if (subobjects == ObjectMapper.Subobjects.ENABLED) { - // Use default behavior - return null; - } - - assert request.isNested() == false; - - // "enabled: false" is not compatible with subobjects: false - // "dynamic: false/strict/runtime" is not compatible with subobjects: false - return new DataSourceResponse.ObjectMappingParametersGenerator(() -> { - var parameters = new HashMap(); - parameters.put("subobjects", subobjects.toString()); - if (ESTestCase.randomBoolean()) { - parameters.put("dynamic", "true"); - } - if (ESTestCase.randomBoolean()) { - parameters.put("enabled", "true"); - } - return parameters; - }); - } - })) - .withPredefinedFields( - List.of( - // Customized because it always needs doc_values for aggregations. - new PredefinedField.WithGenerator("host.name", new FieldDataGenerator() { - @Override - public CheckedConsumer mappingWriter() { - return b -> b.startObject().field("type", "keyword").endObject(); - } - - @Override - public CheckedConsumer fieldValueGenerator() { - return b -> b.value(randomAlphaOfLength(5)); - } - }), - // Needed for terms query - new PredefinedField.WithGenerator("method", new FieldDataGenerator() { - @Override - public CheckedConsumer mappingWriter() { - return b -> b.startObject().field("type", "keyword").endObject(); - } - - @Override - public CheckedConsumer fieldValueGenerator() { - return b -> b.value(randomFrom("put", "post", "get")); - } - }), - - // Needed for histogram aggregation - new PredefinedField.WithGenerator("memory_usage_bytes", new FieldDataGenerator() { - @Override - public CheckedConsumer mappingWriter() { - return b -> b.startObject().field("type", "long").endObject(); - } - - @Override - public CheckedConsumer fieldValueGenerator() { - // We can generate this using standard long field but we would get "too many buckets" - return b -> b.value(randomLongBetween(1000, 2000)); - } - }) - ) - ) - .build()); - } - - @Override - public void baselineMappings(XContentBuilder builder) throws IOException { - dataGenerator.writeMapping(builder); - } - - @Override - public void contenderMappings(XContentBuilder builder) throws IOException { - if (subobjects != ObjectMapper.Subobjects.ENABLED) { - dataGenerator.writeMapping(builder, Map.of("subobjects", subobjects.toString())); - } else { - dataGenerator.writeMapping(builder); - } - } - - @Override - public void contenderSettings(Settings.Builder builder) { - if (keepArraySource) { - builder.put(Mapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING.getKey(), "arrays"); - } - } - - @Override - protected XContentBuilder generateDocument(final Instant timestamp) throws IOException { - var document = XContentFactory.jsonBuilder(); - dataGenerator.generateDocument(document, doc -> { - doc.field("@timestamp", DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(timestamp)); - }); - - return document; - } -} diff --git a/modules/data-streams/src/javaRestTest/resources/roles.yml b/modules/data-streams/src/javaRestTest/resources/roles.yml index 74c238fdae4f2..bc38f60849e4e 100644 --- a/modules/data-streams/src/javaRestTest/resources/roles.yml +++ b/modules/data-streams/src/javaRestTest/resources/roles.yml @@ -16,4 +16,10 @@ under_privilged: - read - write - view_index_metadata +only_get: + indices: + - names: [ 'my-ds*' ] + privileges: + - read + - view_index_metadata no_privilege: diff --git a/modules/data-streams/src/main/java/module-info.java b/modules/data-streams/src/main/java/module-info.java index f21edca356ad9..2d49029c1023c 100644 --- a/modules/data-streams/src/main/java/module-info.java +++ b/modules/data-streams/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.datastreams { @@ -16,6 +17,7 @@ exports org.elasticsearch.datastreams.action to org.elasticsearch.server; exports org.elasticsearch.datastreams.lifecycle.action to org.elasticsearch.server; exports org.elasticsearch.datastreams.lifecycle; + exports org.elasticsearch.datastreams.options.action to org.elasticsearch.server; provides org.elasticsearch.features.FeatureSpecification with org.elasticsearch.datastreams.DataStreamFeatures; } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java index 464a11ce8a062..f60a3e5c47a7f 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; @@ -25,6 +26,7 @@ public class DataStreamFeatures implements FeatureSpecification { public static final NodeFeature DATA_STREAM_LIFECYCLE = new NodeFeature("data_stream.lifecycle"); + public static final NodeFeature DATA_STREAM_FAILURE_STORE_TSDB_FIX = new NodeFeature("data_stream.failure_store.tsdb_fix"); @Override public Map getHistoricalFeatures() { @@ -40,4 +42,9 @@ public Set getFeatures() { DataStreamGlobalRetention.GLOBAL_RETENTION // Added in 8.14 ); } + + @Override + public Set getTestFeatures() { + return Set.of(DATA_STREAM_FAILURE_STORE_TSDB_FIX); + } } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java index 2a1f33831c8c2..d6a0fd86265e5 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; @@ -58,7 +59,7 @@ public class DataStreamIndexSettingsProvider implements IndexSettingProvider { public Settings getAdditionalIndexSettings( String indexName, @Nullable String dataStreamName, - boolean isTimeSeries, + @Nullable IndexMode templateIndexMode, Metadata metadata, Instant resolvedAt, Settings indexTemplateAndCreateRequestSettings, @@ -69,15 +70,16 @@ public Settings getAdditionalIndexSettings( // First backing index is created and then data stream is rolled over (in a single cluster state update). // So at this point we can't check index_mode==time_series, // so checking that index_mode==null|standard and templateIndexMode == TIME_SERIES + boolean isMigratingToTimeSeries = templateIndexMode == IndexMode.TIME_SERIES; boolean migrating = dataStream != null && (dataStream.getIndexMode() == null || dataStream.getIndexMode() == IndexMode.STANDARD) - && isTimeSeries; + && isMigratingToTimeSeries; IndexMode indexMode; if (migrating) { indexMode = IndexMode.TIME_SERIES; } else if (dataStream != null) { - indexMode = isTimeSeries ? dataStream.getIndexMode() : null; - } else if (isTimeSeries) { + indexMode = isMigratingToTimeSeries ? dataStream.getIndexMode() : null; + } else if (isMigratingToTimeSeries) { indexMode = IndexMode.TIME_SERIES; } else { indexMode = null; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java index 3f3a70a94de2a..cb7445705537a 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; @@ -22,6 +23,7 @@ import org.elasticsearch.action.datastreams.lifecycle.GetDataStreamLifecycleAction; import org.elasticsearch.action.datastreams.lifecycle.PutDataStreamLifecycleAction; import org.elasticsearch.client.internal.OriginSettingClient; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -55,6 +57,15 @@ import org.elasticsearch.datastreams.lifecycle.rest.RestExplainDataStreamLifecycleAction; import org.elasticsearch.datastreams.lifecycle.rest.RestGetDataStreamLifecycleAction; import org.elasticsearch.datastreams.lifecycle.rest.RestPutDataStreamLifecycleAction; +import org.elasticsearch.datastreams.options.action.DeleteDataStreamOptionsAction; +import org.elasticsearch.datastreams.options.action.GetDataStreamOptionsAction; +import org.elasticsearch.datastreams.options.action.PutDataStreamOptionsAction; +import org.elasticsearch.datastreams.options.action.TransportDeleteDataStreamOptionsAction; +import org.elasticsearch.datastreams.options.action.TransportGetDataStreamOptionsAction; +import org.elasticsearch.datastreams.options.action.TransportPutDataStreamOptionsAction; +import org.elasticsearch.datastreams.options.rest.RestDeleteDataStreamOptionsAction; +import org.elasticsearch.datastreams.options.rest.RestGetDataStreamOptionsAction; +import org.elasticsearch.datastreams.options.rest.RestPutDataStreamOptionsAction; import org.elasticsearch.datastreams.rest.RestCreateDataStreamAction; import org.elasticsearch.datastreams.rest.RestDataStreamsStatsAction; import org.elasticsearch.datastreams.rest.RestDeleteDataStreamAction; @@ -228,6 +239,11 @@ public Collection createComponents(PluginServices services) { actions.add(new ActionHandler<>(DeleteDataStreamLifecycleAction.INSTANCE, TransportDeleteDataStreamLifecycleAction.class)); actions.add(new ActionHandler<>(ExplainDataStreamLifecycleAction.INSTANCE, TransportExplainDataStreamLifecycleAction.class)); actions.add(new ActionHandler<>(GetDataStreamLifecycleStatsAction.INSTANCE, TransportGetDataStreamLifecycleStatsAction.class)); + if (DataStream.isFailureStoreFeatureFlagEnabled()) { + actions.add(new ActionHandler<>(GetDataStreamOptionsAction.INSTANCE, TransportGetDataStreamOptionsAction.class)); + actions.add(new ActionHandler<>(PutDataStreamOptionsAction.INSTANCE, TransportPutDataStreamOptionsAction.class)); + actions.add(new ActionHandler<>(DeleteDataStreamOptionsAction.INSTANCE, TransportDeleteDataStreamOptionsAction.class)); + } return actions; } @@ -260,6 +276,11 @@ public List getRestHandlers( handlers.add(new RestDeleteDataStreamLifecycleAction()); handlers.add(new RestExplainDataStreamLifecycleAction()); handlers.add(new RestDataStreamLifecycleStatsAction()); + if (DataStream.isFailureStoreFeatureFlagEnabled()) { + handlers.add(new RestGetDataStreamOptionsAction()); + handlers.add(new RestPutDataStreamOptionsAction()); + handlers.add(new RestDeleteDataStreamOptionsAction()); + } return handlers; } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeService.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeService.java index 3bbc37cd87ad5..44a64254f8da9 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeService.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/CreateDataStreamTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/CreateDataStreamTransportAction.java index 77e9d20e34c0e..560e4972b3841 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/CreateDataStreamTransportAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/CreateDataStreamTransportAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.action; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java index d85522d7099b6..1d3b1b676282a 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.action; @@ -30,6 +31,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.engine.ReadOnlyEngine; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.store.StoreStats; import org.elasticsearch.indices.IndicesService; @@ -129,7 +131,7 @@ protected void shardOperation( DataStream dataStream = indexAbstraction.getParentDataStream(); assert dataStream != null; long maxTimestamp = 0L; - try (Engine.Searcher searcher = indexShard.acquireSearcher("data_stream_stats")) { + try (Engine.Searcher searcher = indexShard.acquireSearcher(ReadOnlyEngine.FIELD_RANGE_SEARCH_SOURCE)) { IndexReader indexReader = searcher.getIndexReader(); byte[] maxPackedValue = PointValues.getMaxPackedValue(indexReader, DataStream.TIMESTAMP_FIELD_NAME); if (maxPackedValue != null) { diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java index 8f50f07c358fc..c770b84ee98a3 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.action; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/MigrateToDataStreamTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/MigrateToDataStreamTransportAction.java index cdb342a3ddf3b..7333cc039bef5 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/MigrateToDataStreamTransportAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/MigrateToDataStreamTransportAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.action; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/ModifyDataStreamsTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/ModifyDataStreamsTransportAction.java index 777d239e0ace5..dd232eefc5ac1 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/ModifyDataStreamsTransportAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/ModifyDataStreamsTransportAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.action; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/PromoteDataStreamTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/PromoteDataStreamTransportAction.java index c901b7a707b6c..b9f5bdea9c90e 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/PromoteDataStreamTransportAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/PromoteDataStreamTransportAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.action; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java index 647001d2c9ef5..2d310fef0be7e 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.action; @@ -18,11 +19,13 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.health.ClusterStateHealth; import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings; import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -62,6 +65,7 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadAction private final SystemIndices systemIndices; private final ClusterSettings clusterSettings; private final DataStreamGlobalRetentionSettings globalRetentionSettings; + private final DataStreamFailureStoreSettings dataStreamFailureStoreSettings; private final Client client; @Inject @@ -73,6 +77,7 @@ public TransportGetDataStreamsAction( IndexNameExpressionResolver indexNameExpressionResolver, SystemIndices systemIndices, DataStreamGlobalRetentionSettings globalRetentionSettings, + DataStreamFailureStoreSettings dataStreamFailureStoreSettings, Client client ) { super( @@ -89,7 +94,8 @@ public TransportGetDataStreamsAction( this.systemIndices = systemIndices; this.globalRetentionSettings = globalRetentionSettings; clusterSettings = clusterService.getClusterSettings(); - this.client = client; + this.dataStreamFailureStoreSettings = dataStreamFailureStoreSettings; + this.client = new OriginSettingClient(client, "stack"); } @Override @@ -120,6 +126,7 @@ public void onResponse(DataStreamsStatsAction.Response response) { systemIndices, clusterSettings, globalRetentionSettings, + dataStreamFailureStoreSettings, maxTimestamps ) ); @@ -132,7 +139,16 @@ public void onFailure(Exception e) { }); } else { listener.onResponse( - innerOperation(state, request, indexNameExpressionResolver, systemIndices, clusterSettings, globalRetentionSettings, null) + innerOperation( + state, + request, + indexNameExpressionResolver, + systemIndices, + clusterSettings, + globalRetentionSettings, + dataStreamFailureStoreSettings, + null + ) ); } } @@ -144,11 +160,16 @@ static GetDataStreamAction.Response innerOperation( SystemIndices systemIndices, ClusterSettings clusterSettings, DataStreamGlobalRetentionSettings globalRetentionSettings, + DataStreamFailureStoreSettings dataStreamFailureStoreSettings, @Nullable Map maxTimestamps ) { List dataStreams = getDataStreams(state, indexNameExpressionResolver, request); List dataStreamInfos = new ArrayList<>(dataStreams.size()); for (DataStream dataStream : dataStreams) { + // For this action, we are returning whether the failure store is effectively enabled, either in metadata or by cluster setting. + // Users can use the get data stream options API to find out whether it is explicitly enabled in metadata. + boolean failureStoreEffectivelyEnabled = DataStream.isFailureStoreFeatureFlagEnabled() + && dataStream.isFailureStoreEffectivelyEnabled(dataStreamFailureStoreSettings); final String indexTemplate; boolean indexTemplatePreferIlmValue = true; String ilmPolicyName = null; @@ -252,6 +273,7 @@ public int compareTo(IndexInfo o) { dataStreamInfos.add( new GetDataStreamAction.Response.DataStreamInfo( dataStream, + failureStoreEffectivelyEnabled, streamHealth.getStatus(), indexTemplate, ilmPolicyName, diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleErrorStore.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleErrorStore.java index a1f0c639f51f1..64afe1840d9af 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleErrorStore.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleErrorStore.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java index d8f8ae9d080a7..7d2828e30d5ab 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle; @@ -945,7 +946,7 @@ private Set maybeExecuteForceMerge(ClusterState state, List indice UpdateSettingsRequest updateMergePolicySettingsRequest = new UpdateSettingsRequest(); updateMergePolicySettingsRequest.indicesOptions( IndicesOptions.builder(updateMergePolicySettingsRequest.indicesOptions()) - .failureStoreOptions(new IndicesOptions.FailureStoreOptions(true, true)) + .selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE) .build() ); updateMergePolicySettingsRequest.indices(indexName); @@ -1407,9 +1408,7 @@ static RolloverRequest getDefaultRolloverRequest( RolloverRequest rolloverRequest = new RolloverRequest(dataStream, null).masterNodeTimeout(TimeValue.MAX_VALUE); if (rolloverFailureStore) { rolloverRequest.setIndicesOptions( - IndicesOptions.builder(rolloverRequest.indicesOptions()) - .failureStoreOptions(opts -> opts.includeFailureIndices(true).includeRegularIndices(false)) - .build() + IndicesOptions.builder(rolloverRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.FAILURES).build() ); } rolloverRequest.setConditions(rolloverConfiguration.resolveRolloverConditions(dataRetention)); diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamLifecycleAction.java index 70f822ddee72a..1595348649528 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamLifecycleAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle.action; @@ -33,7 +34,26 @@ private DeleteDataStreamLifecycleAction() {/* no instances */} public static final class Request extends AcknowledgedRequest implements IndicesRequest.Replaceable { private String[] names; - private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true, false); + private IndicesOptions indicesOptions = IndicesOptions.builder() + .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) + .wildcardOptions( + IndicesOptions.WildcardOptions.builder() + .matchOpen(true) + .matchClosed(true) + .includeHidden(false) + .resolveAliases(false) + .allowEmptyExpressions(true) + .build() + ) + .gatekeeperOptions( + IndicesOptions.GatekeeperOptions.builder() + .allowAliasToMultipleIndices(false) + .allowClosedIndices(true) + .ignoreThrottled(false) + .allowFailureIndices(false) + .build() + ) + .build(); public Request(StreamInput in) throws IOException { super(in); diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleStatsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleStatsAction.java index 71f07c8cac668..e7b4b6a45b2ef 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleStatsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleStatsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle.action; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportDeleteDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportDeleteDataStreamLifecycleAction.java index 9dd5794d980f6..ace74c58d73b2 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportDeleteDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportDeleteDataStreamLifecycleAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle.action; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java index 0ffb5809c2f0f..7c3c80ff70de8 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle.action; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java index 452295aab0ce9..1a2907240b3b0 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle.action; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsAction.java index 3a5ef0e86e889..6bbf2a6e01d0c 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle.action; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportPutDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportPutDataStreamLifecycleAction.java index ed9f263bdf03a..44baa6bffedfe 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportPutDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportPutDataStreamLifecycleAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle.action; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/downsampling/DeleteSourceAndAddDownsampleIndexExecutor.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/downsampling/DeleteSourceAndAddDownsampleIndexExecutor.java index bf31146d711b2..3db1fa36d47aa 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/downsampling/DeleteSourceAndAddDownsampleIndexExecutor.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/downsampling/DeleteSourceAndAddDownsampleIndexExecutor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle.downsampling; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/downsampling/DeleteSourceAndAddDownsampleToDS.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/downsampling/DeleteSourceAndAddDownsampleToDS.java index 3e49499740349..fff530a9a6978 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/downsampling/DeleteSourceAndAddDownsampleToDS.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/downsampling/DeleteSourceAndAddDownsampleToDS.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle.downsampling; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthIndicatorService.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthIndicatorService.java index 90154c1190421..ef886302368d0 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthIndicatorService.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthIndicatorService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle.health; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthInfoPublisher.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthInfoPublisher.java index 12abbe125cffb..552bf75adae23 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthInfoPublisher.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthInfoPublisher.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle.health; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDataStreamLifecycleStatsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDataStreamLifecycleStatsAction.java index a3959ae818218..72ed1b5e724c6 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDataStreamLifecycleStatsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDataStreamLifecycleStatsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle.rest; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDeleteDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDeleteDataStreamLifecycleAction.java index a8a64eaf5cfa3..8e7b324b500d8 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDeleteDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDeleteDataStreamLifecycleAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle.rest; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestExplainDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestExplainDataStreamLifecycleAction.java index 82350130e57af..bae7dffa059e6 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestExplainDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestExplainDataStreamLifecycleAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle.rest; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestGetDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestGetDataStreamLifecycleAction.java index ee325ed9655be..4a1050d9723aa 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestGetDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestGetDataStreamLifecycleAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle.rest; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestPutDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestPutDataStreamLifecycleAction.java index 4b285eaf59b97..2d975845fcd27 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestPutDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestPutDataStreamLifecycleAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle.rest; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/DeleteDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/DeleteDataStreamOptionsAction.java new file mode 100644 index 0000000000000..98a29dd636ddf --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/DeleteDataStreamOptionsAction.java @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.datastreams.options.action; + +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; + +/** + * Removes the data stream options configuration from the requested data streams. + */ +public class DeleteDataStreamOptionsAction { + + public static final ActionType INSTANCE = new ActionType<>("indices:admin/data_stream/options/delete"); + + private DeleteDataStreamOptionsAction() {/* no instances */} + + public static final class Request extends AcknowledgedRequest implements IndicesRequest.Replaceable { + + private String[] names; + private IndicesOptions indicesOptions = IndicesOptions.builder() + .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) + .wildcardOptions( + IndicesOptions.WildcardOptions.builder().matchOpen(true).matchClosed(true).allowEmptyExpressions(true).resolveAliases(false) + ) + .gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowAliasToMultipleIndices(false).allowClosedIndices(true)) + .build(); + + public Request(StreamInput in) throws IOException { + super(in); + this.names = in.readOptionalStringArray(); + this.indicesOptions = IndicesOptions.readIndicesOptions(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalStringArray(names); + indicesOptions.writeIndicesOptions(out); + } + + public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String[] names) { + super(masterNodeTimeout, ackTimeout); + this.names = names; + } + + public String[] getNames() { + return names; + } + + @Override + public String[] indices() { + return names; + } + + @Override + public IndicesOptions indicesOptions() { + return indicesOptions; + } + + public Request indicesOptions(IndicesOptions indicesOptions) { + this.indicesOptions = indicesOptions; + return this; + } + + @Override + public boolean includeDataStreams() { + return true; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Arrays.equals(names, request.names) && Objects.equals(indicesOptions, request.indicesOptions); + } + + @Override + public int hashCode() { + int result = Objects.hash(indicesOptions); + result = 31 * result + Arrays.hashCode(names); + return result; + } + + @Override + public IndicesRequest indices(String... indices) { + this.names = indices; + return this; + } + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/GetDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/GetDataStreamOptionsAction.java new file mode 100644 index 0000000000000..c1354da1129ca --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/GetDataStreamOptionsAction.java @@ -0,0 +1,223 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.datastreams.options.action; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.cluster.metadata.DataStreamOptions; +import org.elasticsearch.common.collect.Iterators; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ChunkedToXContentObject; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; + +/** + * This action retrieves the data stream options from every data stream. Currently, data stream options only support + * failure store. + */ +public class GetDataStreamOptionsAction { + + public static final ActionType INSTANCE = new ActionType<>("indices:admin/data_stream/options/get"); + + private GetDataStreamOptionsAction() {/* no instances */} + + public static class Request extends MasterNodeReadRequest implements IndicesRequest.Replaceable { + + private String[] names; + private IndicesOptions indicesOptions = IndicesOptions.builder() + .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) + .wildcardOptions( + IndicesOptions.WildcardOptions.builder().matchOpen(true).matchClosed(true).allowEmptyExpressions(true).resolveAliases(false) + ) + .gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowAliasToMultipleIndices(false).allowClosedIndices(true)) + .build(); + private boolean includeDefaults = false; + + public Request(TimeValue masterNodeTimeout, String[] names) { + super(masterNodeTimeout); + this.names = names; + } + + public Request(TimeValue masterNodeTimeout, String[] names, boolean includeDefaults) { + super(masterNodeTimeout); + this.names = names; + this.includeDefaults = includeDefaults; + } + + public String[] getNames() { + return names; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + public Request(StreamInput in) throws IOException { + super(in); + this.names = in.readOptionalStringArray(); + this.indicesOptions = IndicesOptions.readIndicesOptions(in); + this.includeDefaults = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalStringArray(names); + indicesOptions.writeIndicesOptions(out); + out.writeBoolean(includeDefaults); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Arrays.equals(names, request.names) + && indicesOptions.equals(request.indicesOptions) + && includeDefaults == request.includeDefaults; + } + + @Override + public int hashCode() { + int result = Objects.hash(indicesOptions, includeDefaults); + result = 31 * result + Arrays.hashCode(names); + return result; + } + + @Override + public String[] indices() { + return names; + } + + @Override + public IndicesOptions indicesOptions() { + return indicesOptions; + } + + public boolean includeDefaults() { + return includeDefaults; + } + + public Request indicesOptions(IndicesOptions indicesOptions) { + this.indicesOptions = indicesOptions; + return this; + } + + @Override + public boolean includeDataStreams() { + return true; + } + + @Override + public IndicesRequest indices(String... indices) { + this.names = indices; + return this; + } + + public Request includeDefaults(boolean includeDefaults) { + this.includeDefaults = includeDefaults; + return this; + } + } + + public static class Response extends ActionResponse implements ChunkedToXContentObject { + public static final ParseField DATA_STREAMS_FIELD = new ParseField("data_streams"); + + public record DataStreamEntry(String dataStreamName, DataStreamOptions dataStreamOptions) implements Writeable, ToXContentObject { + + public static final ParseField NAME_FIELD = new ParseField("name"); + public static final ParseField OPTIONS_FIELD = new ParseField("options"); + + DataStreamEntry(StreamInput in) throws IOException { + this(in.readString(), in.readOptionalWriteable(DataStreamOptions::read)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(dataStreamName); + out.writeOptionalWriteable(dataStreamOptions); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(NAME_FIELD.getPreferredName(), dataStreamName); + if (dataStreamOptions != null && dataStreamOptions.isEmpty() == false) { + builder.field(OPTIONS_FIELD.getPreferredName(), dataStreamOptions); + } + builder.endObject(); + return builder; + } + } + + private final List dataStreams; + + public Response(List dataStreams) { + this.dataStreams = dataStreams; + } + + public Response(StreamInput in) throws IOException { + this(in.readCollectionAsList(DataStreamEntry::new)); + } + + public List getDataStreams() { + return dataStreams; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeCollection(dataStreams); + } + + @Override + public Iterator toXContentChunked(ToXContent.Params outerParams) { + return Iterators.concat(Iterators.single((builder, params) -> { + builder.startObject(); + builder.startArray(DATA_STREAMS_FIELD.getPreferredName()); + return builder; + }), + Iterators.map(dataStreams.iterator(), entry -> (builder, params) -> entry.toXContent(builder, outerParams)), + Iterators.single((builder, params) -> { + builder.endArray(); + builder.endObject(); + return builder; + }) + ); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Response response = (Response) o; + return dataStreams.equals(response.dataStreams); + } + + @Override + public int hashCode() { + return Objects.hash(dataStreams); + } + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/PutDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/PutDataStreamOptionsAction.java new file mode 100644 index 0000000000000..d055a6972312a --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/PutDataStreamOptionsAction.java @@ -0,0 +1,165 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.datastreams.options.action; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.cluster.metadata.DataStreamFailureStore; +import org.elasticsearch.cluster.metadata.DataStreamOptions; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +/** + * Sets the data stream options that was provided in the request to the requested data streams. + */ +public class PutDataStreamOptionsAction { + + public static final ActionType INSTANCE = new ActionType<>("indices:admin/data_stream/options/put"); + + private PutDataStreamOptionsAction() {/* no instances */} + + public static final class Request extends AcknowledgedRequest implements IndicesRequest.Replaceable { + + public interface Factory { + Request create(@Nullable DataStreamFailureStore dataStreamFailureStore); + } + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "put_data_stream_options_request", + false, + (args, factory) -> factory.create((DataStreamFailureStore) args[0]) + ); + + static { + PARSER.declareObjectOrNull( + ConstructingObjectParser.optionalConstructorArg(), + (p, c) -> DataStreamFailureStore.PARSER.parse(p, null), + null, + new ParseField("failure_store") + ); + } + + public static Request parseRequest(XContentParser parser, Factory factory) { + return PARSER.apply(parser, factory); + } + + private String[] names; + private IndicesOptions indicesOptions = IndicesOptions.builder() + .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) + .wildcardOptions( + IndicesOptions.WildcardOptions.builder().matchOpen(true).matchClosed(true).allowEmptyExpressions(true).resolveAliases(false) + ) + .gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowAliasToMultipleIndices(false).allowClosedIndices(true)) + .build(); + private final DataStreamOptions options; + + public Request(StreamInput in) throws IOException { + super(in); + this.names = in.readStringArray(); + this.indicesOptions = IndicesOptions.readIndicesOptions(in); + options = DataStreamOptions.read(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeStringArray(names); + indicesOptions.writeIndicesOptions(out); + out.writeWriteable(options); + } + + public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String[] names, DataStreamOptions options) { + super(masterNodeTimeout, ackTimeout); + this.names = names; + this.options = options; + } + + public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String[] names, @Nullable DataStreamFailureStore failureStore) { + super(masterNodeTimeout, ackTimeout); + this.names = names; + this.options = new DataStreamOptions(failureStore); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (options.failureStore() == null) { + validationException = addValidationError("At least one option needs to be provided", validationException); + } + return validationException; + } + + public String[] getNames() { + return names; + } + + public DataStreamOptions getOptions() { + return options; + } + + @Override + public String[] indices() { + return names; + } + + @Override + public IndicesOptions indicesOptions() { + return indicesOptions; + } + + public Request indicesOptions(IndicesOptions indicesOptions) { + this.indicesOptions = indicesOptions; + return this; + } + + @Override + public boolean includeDataStreams() { + return true; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Arrays.equals(names, request.names) + && Objects.equals(indicesOptions, request.indicesOptions) + && options.equals(request.options); + } + + @Override + public int hashCode() { + int result = Objects.hash(indicesOptions, options); + result = 31 * result + Arrays.hashCode(names); + return result; + } + + @Override + public IndicesRequest indices(String... names) { + this.names = names; + return this; + } + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportDeleteDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportDeleteDataStreamOptionsAction.java new file mode 100644 index 0000000000000..ead23ed78222b --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportDeleteDataStreamOptionsAction.java @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.datastreams.options.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.datastreams.DataStreamsActionUtil; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetadataDataStreamsService; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.indices.SystemIndices; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.List; + +/** + * Transport action that resolves the data stream names from the request and removes any configured data stream options from them. + */ +public class TransportDeleteDataStreamOptionsAction extends AcknowledgedTransportMasterNodeAction { + + private final MetadataDataStreamsService metadataDataStreamsService; + private final SystemIndices systemIndices; + + @Inject + public TransportDeleteDataStreamOptionsAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + MetadataDataStreamsService metadataDataStreamsService, + SystemIndices systemIndices + ) { + super( + DeleteDataStreamOptionsAction.INSTANCE.name(), + transportService, + clusterService, + threadPool, + actionFilters, + DeleteDataStreamOptionsAction.Request::new, + indexNameExpressionResolver, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.metadataDataStreamsService = metadataDataStreamsService; + this.systemIndices = systemIndices; + } + + @Override + protected void masterOperation( + Task task, + DeleteDataStreamOptionsAction.Request request, + ClusterState state, + ActionListener listener + ) { + List dataStreamNames = DataStreamsActionUtil.getDataStreamNames( + indexNameExpressionResolver, + state, + request.getNames(), + request.indicesOptions() + ); + for (String name : dataStreamNames) { + systemIndices.validateDataStreamAccess(name, threadPool.getThreadContext()); + } + metadataDataStreamsService.removeDataStreamOptions(dataStreamNames, request.ackTimeout(), request.masterNodeTimeout(), listener); + } + + @Override + protected ClusterBlockException checkBlock(DeleteDataStreamOptionsAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportGetDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportGetDataStreamOptionsAction.java new file mode 100644 index 0000000000000..b032b35c943c0 --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportGetDataStreamOptionsAction.java @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.datastreams.options.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.datastreams.DataStreamsActionUtil; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.indices.SystemIndices; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Collects the data streams from the cluster state and then returns for each data stream its name and its + * data stream options. Currently, data stream options include only the failure store configuration. + */ +public class TransportGetDataStreamOptionsAction extends TransportMasterNodeReadAction< + GetDataStreamOptionsAction.Request, + GetDataStreamOptionsAction.Response> { + + private final SystemIndices systemIndices; + + @Inject + public TransportGetDataStreamOptionsAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + SystemIndices systemIndices + ) { + super( + GetDataStreamOptionsAction.INSTANCE.name(), + transportService, + clusterService, + threadPool, + actionFilters, + GetDataStreamOptionsAction.Request::new, + indexNameExpressionResolver, + GetDataStreamOptionsAction.Response::new, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.systemIndices = systemIndices; + } + + @Override + protected void masterOperation( + Task task, + GetDataStreamOptionsAction.Request request, + ClusterState state, + ActionListener listener + ) { + List requestedDataStreams = DataStreamsActionUtil.getDataStreamNames( + indexNameExpressionResolver, + state, + request.getNames(), + request.indicesOptions() + ); + Map dataStreams = state.metadata().dataStreams(); + for (String name : requestedDataStreams) { + systemIndices.validateDataStreamAccess(name, threadPool.getThreadContext()); + } + listener.onResponse( + new GetDataStreamOptionsAction.Response( + requestedDataStreams.stream() + .map(dataStreams::get) + .filter(Objects::nonNull) + .map( + dataStream -> new GetDataStreamOptionsAction.Response.DataStreamEntry( + dataStream.getName(), + dataStream.getDataStreamOptions() + ) + ) + .sorted(Comparator.comparing(GetDataStreamOptionsAction.Response.DataStreamEntry::dataStreamName)) + .toList() + ) + ); + } + + @Override + protected ClusterBlockException checkBlock(GetDataStreamOptionsAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportPutDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportPutDataStreamOptionsAction.java new file mode 100644 index 0000000000000..b1386232c44f9 --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportPutDataStreamOptionsAction.java @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.datastreams.options.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.datastreams.DataStreamsActionUtil; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetadataDataStreamsService; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.indices.SystemIndices; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.List; + +/** + * Transport action that resolves the data stream names from the request and sets the data stream lifecycle provided in the request. + */ +public class TransportPutDataStreamOptionsAction extends AcknowledgedTransportMasterNodeAction { + + private final MetadataDataStreamsService metadataDataStreamsService; + private final SystemIndices systemIndices; + + @Inject + public TransportPutDataStreamOptionsAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + MetadataDataStreamsService metadataDataStreamsService, + SystemIndices systemIndices + ) { + super( + PutDataStreamOptionsAction.INSTANCE.name(), + transportService, + clusterService, + threadPool, + actionFilters, + PutDataStreamOptionsAction.Request::new, + indexNameExpressionResolver, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.metadataDataStreamsService = metadataDataStreamsService; + this.systemIndices = systemIndices; + } + + @Override + protected void masterOperation( + Task task, + PutDataStreamOptionsAction.Request request, + ClusterState state, + ActionListener listener + ) { + List dataStreamNames = DataStreamsActionUtil.getDataStreamNames( + indexNameExpressionResolver, + state, + request.getNames(), + request.indicesOptions() + ); + for (String name : dataStreamNames) { + systemIndices.validateDataStreamAccess(name, threadPool.getThreadContext()); + } + metadataDataStreamsService.setDataStreamOptions( + dataStreamNames, + request.getOptions(), + request.ackTimeout(), + request.masterNodeTimeout(), + listener + ); + } + + @Override + protected ClusterBlockException checkBlock(PutDataStreamOptionsAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestDeleteDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestDeleteDataStreamOptionsAction.java new file mode 100644 index 0000000000000..96460632ff443 --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestDeleteDataStreamOptionsAction.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.datastreams.options.rest; + +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.datastreams.options.action.DeleteDataStreamOptionsAction; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestToXContentListener; + +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.DELETE; +import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout; + +@ServerlessScope(Scope.INTERNAL) +public class RestDeleteDataStreamOptionsAction extends BaseRestHandler { + + @Override + public String getName() { + return "delete_data_stream_options_action"; + } + + @Override + public List routes() { + return List.of(new Route(DELETE, "/_data_stream/{name}/_options")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { + final var deleteDataOptionsRequest = new DeleteDataStreamOptionsAction.Request( + getMasterNodeTimeout(request), + request.paramAsTime("timeout", AcknowledgedRequest.DEFAULT_ACK_TIMEOUT), + Strings.splitStringByCommaToArray(request.param("name")) + ); + deleteDataOptionsRequest.indicesOptions(IndicesOptions.fromRequest(request, deleteDataOptionsRequest.indicesOptions())); + return channel -> client.execute( + DeleteDataStreamOptionsAction.INSTANCE, + deleteDataOptionsRequest, + new RestToXContentListener<>(channel) + ); + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestGetDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestGetDataStreamOptionsAction.java new file mode 100644 index 0000000000000..6d6530efce1b9 --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestGetDataStreamOptionsAction.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.datastreams.options.rest; + +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.datastreams.options.action.GetDataStreamOptionsAction; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestUtils; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestRefCountedChunkedToXContentListener; + +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.GET; + +@ServerlessScope(Scope.PUBLIC) +public class RestGetDataStreamOptionsAction extends BaseRestHandler { + + @Override + public String getName() { + return "get_data_stream_options_action"; + } + + @Override + public List routes() { + return List.of(new Route(GET, "/_data_stream/{name}/_options")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { + GetDataStreamOptionsAction.Request getDataStreamOptionsRequest = new GetDataStreamOptionsAction.Request( + RestUtils.getMasterNodeTimeout(request), + Strings.splitStringByCommaToArray(request.param("name")) + ); + getDataStreamOptionsRequest.includeDefaults(request.paramAsBoolean("include_defaults", false)); + getDataStreamOptionsRequest.indicesOptions(IndicesOptions.fromRequest(request, getDataStreamOptionsRequest.indicesOptions())); + return channel -> client.execute( + GetDataStreamOptionsAction.INSTANCE, + getDataStreamOptionsRequest, + new RestRefCountedChunkedToXContentListener<>(channel) + ); + } + + @Override + public boolean allowSystemIndexAccessByDefault() { + return true; + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestPutDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestPutDataStreamOptionsAction.java new file mode 100644 index 0000000000000..9191b96b6039e --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestPutDataStreamOptionsAction.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.datastreams.options.rest; + +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.datastreams.options.action.PutDataStreamOptionsAction; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.PUT; +import static org.elasticsearch.rest.RestUtils.getAckTimeout; +import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout; + +@ServerlessScope(Scope.PUBLIC) +public class RestPutDataStreamOptionsAction extends BaseRestHandler { + + @Override + public String getName() { + return "put_data_stream_options_action"; + } + + @Override + public List routes() { + return List.of(new Route(PUT, "/_data_stream/{name}/_options")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + try (XContentParser parser = request.contentParser()) { + PutDataStreamOptionsAction.Request putOptionsRequest = PutDataStreamOptionsAction.Request.parseRequest( + parser, + (failureStore) -> new PutDataStreamOptionsAction.Request( + getMasterNodeTimeout(request), + getAckTimeout(request), + Strings.splitStringByCommaToArray(request.param("name")), + failureStore + ) + ); + putOptionsRequest.indicesOptions(IndicesOptions.fromRequest(request, putOptionsRequest.indicesOptions())); + return channel -> client.execute(PutDataStreamOptionsAction.INSTANCE, putOptionsRequest, new RestToXContentListener<>(channel)); + } + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestCreateDataStreamAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestCreateDataStreamAction.java index a6dc9cfc9fb18..0b35a0f23d683 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestCreateDataStreamAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestCreateDataStreamAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.rest; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestDataStreamsStatsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestDataStreamsStatsAction.java index 6ddffe51ad8cd..ed63598421664 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestDataStreamsStatsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestDataStreamsStatsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.rest; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestDeleteDataStreamAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestDeleteDataStreamAction.java index 551d14ced5104..990e7ac753c85 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestDeleteDataStreamAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestDeleteDataStreamAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.rest; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java index 29cda588bc26b..b61e38297397d 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java @@ -1,15 +1,17 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.rest; import org.elasticsearch.action.datastreams.GetDataStreamAction; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.set.Sets; @@ -28,6 +30,23 @@ @ServerlessScope(Scope.PUBLIC) public class RestGetDataStreamsAction extends BaseRestHandler { + private static final Set SUPPORTED_QUERY_PARAMETERS = Set.copyOf( + Sets.union( + RestRequest.INTERNAL_MARKER_REQUEST_PARAMETERS, + Set.of( + "name", + "include_defaults", + "master_timeout", + IndicesOptions.WildcardOptions.EXPAND_WILDCARDS, + IndicesOptions.ConcreteTargetOptions.IGNORE_UNAVAILABLE, + IndicesOptions.WildcardOptions.ALLOW_NO_INDICES, + IndicesOptions.GatekeeperOptions.IGNORE_THROTTLED, + "verbose" + ), + DataStream.isFailureStoreFeatureFlagEnabled() ? Set.of(IndicesOptions.FAILURE_STORE_QUERY_PARAM) : Set.of() + ) + ); + @Override public String getName() { return "get_data_streams_action"; @@ -62,19 +81,6 @@ public Set supportedCapabilities() { @Override public Set supportedQueryParameters() { - return Sets.union( - RestRequest.INTERNAL_MARKER_REQUEST_PARAMETERS, - Set.of( - "name", - "include_defaults", - "timeout", - "master_timeout", - IndicesOptions.WildcardOptions.EXPAND_WILDCARDS, - IndicesOptions.ConcreteTargetOptions.IGNORE_UNAVAILABLE, - IndicesOptions.WildcardOptions.ALLOW_NO_INDICES, - IndicesOptions.GatekeeperOptions.IGNORE_THROTTLED, - "verbose" - ) - ); + return SUPPORTED_QUERY_PARAMETERS; } } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestMigrateToDataStreamAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestMigrateToDataStreamAction.java index 7c935bf825095..b48df7e48a882 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestMigrateToDataStreamAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestMigrateToDataStreamAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.rest; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestModifyDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestModifyDataStreamsAction.java index 0b33e06eb5940..c88d38223b902 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestModifyDataStreamsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestModifyDataStreamsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.rest; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestPromoteDataStreamAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestPromoteDataStreamAction.java index 50ee1e193d5c8..84f0741e59e89 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestPromoteDataStreamAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestPromoteDataStreamAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.rest; diff --git a/modules/data-streams/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification b/modules/data-streams/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification index 3f1579eac4f85..db6ba7efd0652 100644 --- a/modules/data-streams/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification +++ b/modules/data-streams/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # org.elasticsearch.datastreams.DataStreamFeatures diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java index 6a1f8031ce7c6..3d08be1f24a42 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java index cf6911850921b..015752724cb5d 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; @@ -77,7 +78,7 @@ public void testGetAdditionalIndexSettings() throws Exception { Settings result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - true, + IndexMode.TIME_SERIES, metadata, now, settings, @@ -122,7 +123,7 @@ public void testGetAdditionalIndexSettingsIndexRoutingPathAlreadyDefined() throw Settings result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - true, + IndexMode.TIME_SERIES, metadata, now, settings, @@ -192,7 +193,7 @@ public void testGetAdditionalIndexSettingsMappingsMerging() throws Exception { Settings result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - true, + IndexMode.TIME_SERIES, metadata, now, settings, @@ -217,7 +218,7 @@ public void testGetAdditionalIndexSettingsNoMappings() { Settings result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - true, + IndexMode.TIME_SERIES, metadata, now, settings, @@ -242,7 +243,7 @@ public void testGetAdditionalIndexSettingsLookAheadTime() throws Exception { Settings result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - true, + IndexMode.TIME_SERIES, metadata, now, settings, @@ -267,7 +268,7 @@ public void testGetAdditionalIndexSettingsLookBackTime() throws Exception { Settings result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - true, + IndexMode.TIME_SERIES, metadata, now, settings, @@ -298,7 +299,7 @@ public void testGetAdditionalIndexSettingsDataStreamAlreadyCreated() throws Exce var result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - true, + IndexMode.TIME_SERIES, metadata, now, settings, @@ -335,7 +336,7 @@ public void testGetAdditionalIndexSettingsDataStreamAlreadyCreatedTimeSettingsMi () -> provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - true, + IndexMode.TIME_SERIES, metadata, now, settings, @@ -361,7 +362,7 @@ public void testGetAdditionalIndexSettingsNonTsdbTemplate() { Settings result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - false, + null, metadata, Instant.ofEpochMilli(1L), settings, @@ -381,7 +382,7 @@ public void testGetAdditionalIndexSettingsMigrateToTsdb() { Settings result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 2), dataStreamName, - true, + IndexMode.TIME_SERIES, metadata, now, settings, @@ -414,7 +415,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromTsdb() { Settings result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 2), dataStreamName, - false, + null, metadata, Instant.ofEpochMilli(1L), settings, @@ -693,7 +694,7 @@ private Settings generateTsdbSettings(String mapping, Instant now) throws IOExce var result = provider.getAdditionalIndexSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - true, + IndexMode.TIME_SERIES, metadata, now, settings, diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java index b2ddab164b31b..d5c5193948213 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/LookAHeadTimeTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/LookAHeadTimeTests.java index e3d5ad0d63e84..199bc36d833aa 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/LookAHeadTimeTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/LookAHeadTimeTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataDataStreamRolloverServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataDataStreamRolloverServiceTests.java index 8156345b83b4c..96f70232e39de 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataDataStreamRolloverServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataDataStreamRolloverServiceTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java index d5356e371f497..29e49c8ddfa17 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; @@ -12,7 +13,6 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; -import org.elasticsearch.cluster.metadata.DataStreamFactoryRetention; import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; @@ -216,10 +216,7 @@ private MetadataIndexTemplateService getMetadataIndexTemplateService() { xContentRegistry(), EmptySystemIndices.INSTANCE, indexSettingProviders, - DataStreamGlobalRetentionSettings.create( - ClusterSettings.createBuiltInClusterSettings(), - DataStreamFactoryRetention.emptyFactoryRetention() - ) + DataStreamGlobalRetentionSettings.create(ClusterSettings.createBuiltInClusterSettings()) ); } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/TimestampFieldMapperServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/TimestampFieldMapperServiceTests.java index eb35c44d30331..fd51e0d12c1ee 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/TimestampFieldMapperServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/TimestampFieldMapperServiceTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeServiceTests.java index 4b0eaa6c46baf..f05fe518fa56d 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeServiceTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/CreateDataStreamRequestTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/CreateDataStreamRequestTests.java index 7d0359a946c2e..0867b30258fff 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/CreateDataStreamRequestTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/CreateDataStreamRequestTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.action; diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DataStreamsStatsResponseTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DataStreamsStatsResponseTests.java index 9874c19a78240..ec770ba51b232 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DataStreamsStatsResponseTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DataStreamsStatsResponseTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.action; diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DeleteDataStreamRequestTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DeleteDataStreamRequestTests.java index 056af78846f3e..5704d1d5510f5 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DeleteDataStreamRequestTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DeleteDataStreamRequestTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.action; diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportActionTests.java index cdfad60fcfcba..6d008b80a40a2 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportActionTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportActionTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.action; diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsRequestTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsRequestTests.java index 824bec21b16e9..3daf9121e0990 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsRequestTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsRequestTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.action; diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java index d4860de73213b..9414943cbb439 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.action; @@ -12,6 +13,7 @@ import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; +import org.elasticsearch.cluster.metadata.DataStreamOptions; import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.bytes.BytesReference; @@ -82,7 +84,7 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti .setAllowCustomRouting(true) .setIndexMode(IndexMode.STANDARD) .setLifecycle(new DataStreamLifecycle()) - .setFailureStoreEnabled(true) + .setDataStreamOptions(DataStreamOptions.FAILURE_STORE_ENABLED) .setFailureIndices(DataStream.DataStreamIndices.failureIndicesBuilder(failureStores).build()) .build(); @@ -100,6 +102,7 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti Response.DataStreamInfo dataStreamInfo = new Response.DataStreamInfo( logs, + true, ClusterHealthStatus.GREEN, "index-template", null, @@ -185,7 +188,7 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti .setAllowCustomRouting(true) .setIndexMode(IndexMode.STANDARD) .setLifecycle(new DataStreamLifecycle(null, null, false)) - .setFailureStoreEnabled(true) + .setDataStreamOptions(DataStreamOptions.FAILURE_STORE_ENABLED) .setFailureIndices(DataStream.DataStreamIndices.failureIndicesBuilder(failureStores).build()) .build(); @@ -203,6 +206,7 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti Response.DataStreamInfo dataStreamInfo = new Response.DataStreamInfo( logs, + true, ClusterHealthStatus.GREEN, "index-template", null, @@ -280,6 +284,7 @@ public void testManagedByDisplayValuesDontAccidentalyChange() { private Response.DataStreamInfo mutateInstance(Response.DataStreamInfo instance) { var dataStream = instance.getDataStream(); + var failureStoreEffectivelyEnabled = instance.isFailureStoreEffectivelyEnabled(); var status = instance.getDataStreamStatus(); var indexTemplate = instance.getIndexTemplate(); var ilmPolicyName = instance.getIlmPolicy(); @@ -287,7 +292,7 @@ private Response.DataStreamInfo mutateInstance(Response.DataStreamInfo instance) var indexSettings = instance.getIndexSettingsValues(); var templatePreferIlm = instance.templatePreferIlmValue(); var maximumTimestamp = instance.getMaximumTimestamp(); - switch (randomIntBetween(0, 7)) { + switch (randomIntBetween(0, 8)) { case 0 -> dataStream = randomValueOtherThan(dataStream, DataStreamTestHelper::randomInstance); case 1 -> status = randomValueOtherThan(status, () -> randomFrom(ClusterHealthStatus.values())); case 2 -> indexTemplate = randomBoolean() && indexTemplate != null ? null : randomAlphaOfLengthBetween(2, 10); @@ -312,9 +317,11 @@ private Response.DataStreamInfo mutateInstance(Response.DataStreamInfo instance) case 7 -> maximumTimestamp = (maximumTimestamp == null) ? randomNonNegativeLong() : (usually() ? randomValueOtherThan(maximumTimestamp, ESTestCase::randomNonNegativeLong) : null); + case 8 -> failureStoreEffectivelyEnabled = failureStoreEffectivelyEnabled ? false : true; } return new Response.DataStreamInfo( dataStream, + failureStoreEffectivelyEnabled, status, indexTemplate, ilmPolicyName, @@ -353,6 +360,7 @@ private Response.DataStreamInfo generateRandomDataStreamInfo() { List> timeSeries = randomBoolean() ? generateRandomTimeSeries() : null; return new Response.DataStreamInfo( DataStreamTestHelper.randomInstance(), + randomBoolean(), ClusterHealthStatus.GREEN, randomAlphaOfLengthBetween(2, 10), randomAlphaOfLengthBetween(2, 10), diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java index e167e15576240..e041fd1b45430 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.action; @@ -11,7 +12,7 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.DataStream; -import org.elasticsearch.cluster.metadata.DataStreamFactoryRetention; +import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings; import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings; import org.elasticsearch.cluster.metadata.DataStreamTestHelper; @@ -39,6 +40,8 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; public class TransportGetDataStreamsActionTests extends ESTestCase { @@ -46,8 +49,10 @@ public class TransportGetDataStreamsActionTests extends ESTestCase { private final IndexNameExpressionResolver resolver = TestIndexNameExpressionResolver.newInstance(); private final SystemIndices systemIndices = new SystemIndices(List.of()); private final DataStreamGlobalRetentionSettings dataStreamGlobalRetentionSettings = DataStreamGlobalRetentionSettings.create( - ClusterSettings.createBuiltInClusterSettings(), - DataStreamFactoryRetention.emptyFactoryRetention() + ClusterSettings.createBuiltInClusterSettings() + ); + private final DataStreamFailureStoreSettings emptyDataStreamFailureStoreSettings = DataStreamFailureStoreSettings.create( + ClusterSettings.createBuiltInClusterSettings() ); public void testGetDataStream() { @@ -167,6 +172,7 @@ public void testGetTimeSeriesDataStream() { systemIndices, ClusterSettings.createBuiltInClusterSettings(), dataStreamGlobalRetentionSettings, + emptyDataStreamFailureStoreSettings, null ); assertThat( @@ -198,6 +204,7 @@ public void testGetTimeSeriesDataStream() { systemIndices, ClusterSettings.createBuiltInClusterSettings(), dataStreamGlobalRetentionSettings, + emptyDataStreamFailureStoreSettings, null ); assertThat( @@ -249,6 +256,7 @@ public void testGetTimeSeriesDataStreamWithOutOfOrderIndices() { systemIndices, ClusterSettings.createBuiltInClusterSettings(), dataStreamGlobalRetentionSettings, + emptyDataStreamFailureStoreSettings, null ); assertThat( @@ -293,6 +301,7 @@ public void testGetTimeSeriesMixedDataStream() { systemIndices, ClusterSettings.createBuiltInClusterSettings(), dataStreamGlobalRetentionSettings, + emptyDataStreamFailureStoreSettings, null ); @@ -339,6 +348,7 @@ public void testPassingGlobalRetention() { systemIndices, ClusterSettings.createBuiltInClusterSettings(), dataStreamGlobalRetentionSettings, + emptyDataStreamFailureStoreSettings, null ); assertThat(response.getGlobalRetention(), nullValue()); @@ -355,8 +365,7 @@ public void testPassingGlobalRetention() { ) .put(DataStreamGlobalRetentionSettings.DATA_STREAMS_MAX_RETENTION_SETTING.getKey(), globalRetention.maxRetention()) .build() - ), - DataStreamFactoryRetention.emptyFactoryRetention() + ) ); response = TransportGetDataStreamsAction.innerOperation( state, @@ -365,8 +374,102 @@ public void testPassingGlobalRetention() { systemIndices, ClusterSettings.createBuiltInClusterSettings(), withGlobalRetentionSettings, + emptyDataStreamFailureStoreSettings, null ); assertThat(response.getGlobalRetention(), equalTo(globalRetention)); } + + public void testDataStreamIsFailureStoreEffectivelyEnabled_disabled() { + var metadata = new Metadata.Builder(); + DataStreamTestHelper.getClusterStateWithDataStreams( + metadata, + List.of(Tuple.tuple("data-stream-1", 2)), + List.of(), + System.currentTimeMillis(), + Settings.EMPTY, + 0, + false, + false + ); + ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(metadata).build(); + + var req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {}); + var response = TransportGetDataStreamsAction.innerOperation( + state, + req, + resolver, + systemIndices, + ClusterSettings.createBuiltInClusterSettings(), + dataStreamGlobalRetentionSettings, + emptyDataStreamFailureStoreSettings, + null + ); + assertThat(response.getDataStreams(), hasSize(1)); + assertThat(response.getDataStreams().get(0).isFailureStoreEffectivelyEnabled(), is(false)); + } + + public void testDataStreamIsFailureStoreEffectivelyEnabled_enabledExplicitly() { + var metadata = new Metadata.Builder(); + DataStreamTestHelper.getClusterStateWithDataStreams( + metadata, + List.of(Tuple.tuple("data-stream-1", 2)), + List.of(), + System.currentTimeMillis(), + Settings.EMPTY, + 0, + false, + true + ); + ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(metadata).build(); + + var req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {}); + var response = TransportGetDataStreamsAction.innerOperation( + state, + req, + resolver, + systemIndices, + ClusterSettings.createBuiltInClusterSettings(), + dataStreamGlobalRetentionSettings, + emptyDataStreamFailureStoreSettings, + null + ); + assertThat(response.getDataStreams(), hasSize(1)); + assertThat(response.getDataStreams().get(0).isFailureStoreEffectivelyEnabled(), is(true)); + } + + public void testDataStreamIsFailureStoreEffectivelyEnabled_enabledByClusterSetting() { + var metadata = new Metadata.Builder(); + DataStreamTestHelper.getClusterStateWithDataStreams( + metadata, + List.of(Tuple.tuple("data-stream-1", 2)), + List.of(), + System.currentTimeMillis(), + Settings.EMPTY, + 0, + false, + false + ); + ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(metadata).build(); + + var req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {}); + var response = TransportGetDataStreamsAction.innerOperation( + state, + req, + resolver, + systemIndices, + ClusterSettings.createBuiltInClusterSettings(), + dataStreamGlobalRetentionSettings, + DataStreamFailureStoreSettings.create( + ClusterSettings.createBuiltInClusterSettings( + Settings.builder() + .put(DataStreamFailureStoreSettings.DATA_STREAM_FAILURE_STORED_ENABLED_SETTING.getKey(), "data-stream-*") + .build() + ) + ), + null + ); + assertThat(response.getDataStreams(), hasSize(1)); + assertThat(response.getDataStreams().get(0).isFailureStoreEffectivelyEnabled(), is(true)); + } } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleErrorStoreTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleErrorStoreTests.java index 41e0f6578cb7d..c33bd8482710b 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleErrorStoreTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleErrorStoreTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle; diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java index 6a4a786e36103..0ab105a467ab3 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle; @@ -113,7 +114,12 @@ static void putComposableIndexTemplate( request.indexTemplate( ComposableIndexTemplate.builder() .indexPatterns(patterns) - .template(new Template(settings, mappings == null ? null : CompressedXContent.fromJSON(mappings), null, lifecycle)) + .template( + Template.builder() + .settings(settings) + .mappings(mappings == null ? null : CompressedXContent.fromJSON(mappings)) + .lifecycle(lifecycle) + ) .metadata(metadata) .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) .build() diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java index 8cb27fd9fd282..09ecb2c224e14 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle; @@ -36,11 +37,11 @@ import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.metadata.DataStream; -import org.elasticsearch.cluster.metadata.DataStreamFactoryRetention; import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.cluster.metadata.DataStreamLifecycle.Downsampling; import org.elasticsearch.cluster.metadata.DataStreamLifecycle.Downsampling.Round; +import org.elasticsearch.cluster.metadata.DataStreamOptions; import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexGraveyard; @@ -118,6 +119,7 @@ import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleService.TARGET_MERGE_FACTOR_VALUE; import static org.elasticsearch.test.ClusterServiceUtils.createClusterService; import static org.elasticsearch.test.ClusterServiceUtils.setState; +import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; @@ -139,8 +141,7 @@ public class DataStreamLifecycleServiceTests extends ESTestCase { private DoExecuteDelegate clientDelegate; private ClusterService clusterService; private final DataStreamGlobalRetentionSettings globalRetentionSettings = DataStreamGlobalRetentionSettings.create( - ClusterSettings.createBuiltInClusterSettings(), - DataStreamFactoryRetention.emptyFactoryRetention() + ClusterSettings.createBuiltInClusterSettings() ); @Before @@ -224,17 +225,11 @@ public void testOperationsExecutedOnce() { assertThat(clientSeenRequests.get(0), instanceOf(RolloverRequest.class)); RolloverRequest rolloverBackingIndexRequest = (RolloverRequest) clientSeenRequests.get(0); assertThat(rolloverBackingIndexRequest.getRolloverTarget(), is(dataStreamName)); - assertThat( - rolloverBackingIndexRequest.indicesOptions().failureStoreOptions(), - equalTo(new IndicesOptions.FailureStoreOptions(true, false)) - ); + assertThat(rolloverBackingIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.DATA)); assertThat(clientSeenRequests.get(1), instanceOf(RolloverRequest.class)); RolloverRequest rolloverFailureIndexRequest = (RolloverRequest) clientSeenRequests.get(1); assertThat(rolloverFailureIndexRequest.getRolloverTarget(), is(dataStreamName)); - assertThat( - rolloverFailureIndexRequest.indicesOptions().failureStoreOptions(), - equalTo(new IndicesOptions.FailureStoreOptions(false, true)) - ); + assertThat(rolloverFailureIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.FAILURES)); List deleteRequests = clientSeenRequests.subList(2, 5) .stream() .map(transportRequest -> (DeleteIndexRequest) transportRequest) @@ -325,7 +320,52 @@ public void testRetentionNotExecutedForTSIndicesWithinTimeBounds() { TransportRequest deleteIndexRequest = clientSeenRequests.get(1); assertThat(deleteIndexRequest, instanceOf(DeleteIndexRequest.class)); // only the first generation index should be eligible for retention - assertThat(((DeleteIndexRequest) deleteIndexRequest).indices(), is(new String[] { dataStream.getIndices().get(0).getName() })); + assertThat(((DeleteIndexRequest) deleteIndexRequest).indices(), arrayContaining(dataStream.getIndices().get(0).getName())); + } + + public void testMergePolicyNotExecutedForTSIndicesWithinTimeBounds() { + Instant currentTime = Instant.ofEpochMilli(now).truncatedTo(ChronoUnit.MILLIS); + // These ranges are on the edge of each other temporal boundaries. + Instant start1 = currentTime.minus(6, ChronoUnit.HOURS); + Instant end1 = currentTime.minus(4, ChronoUnit.HOURS); + Instant start2 = currentTime.minus(4, ChronoUnit.HOURS); + Instant end2 = currentTime.plus(2, ChronoUnit.HOURS); + Instant start3 = currentTime.plus(2, ChronoUnit.HOURS); + Instant end3 = currentTime.plus(4, ChronoUnit.HOURS); + + String dataStreamName = "logs_my-app_prod"; + var clusterState = DataStreamTestHelper.getClusterStateWithDataStream( + dataStreamName, + List.of(Tuple.tuple(start1, end1), Tuple.tuple(start2, end2), Tuple.tuple(start3, end3)) + ); + Metadata.Builder builder = Metadata.builder(clusterState.metadata()); + DataStream dataStream = builder.dataStream(dataStreamName); + // Overwrite the data stream in the cluster state to set the lifecycle policy, with no retention policy (i.e. infinite retention). + builder.put( + dataStream.copy() + .setName(dataStreamName) + .setGeneration(dataStream.getGeneration() + 1) + .setLifecycle(DataStreamLifecycle.newBuilder().build()) + .build() + ); + clusterState = ClusterState.builder(clusterState).metadata(builder).build(); + + dataStreamLifecycleService.run(clusterState); + // There should be two client requests: one rollover, and one to update the merge policy settings. N.B. The merge policy settings + // will always be updated before the force merge is done, see testMergePolicySettingsAreConfiguredBeforeForcemerge. + assertThat(clientSeenRequests.size(), is(2)); + assertThat(clientSeenRequests.get(0), instanceOf(RolloverRequest.class)); + TransportRequest updateSettingsRequest = clientSeenRequests.get(1); + assertThat(updateSettingsRequest, instanceOf(UpdateSettingsRequest.class)); + // Only the first generation index should be eligible for merging. The other have end dates in the future. + assertThat(((UpdateSettingsRequest) updateSettingsRequest).indices(), arrayContaining(dataStream.getIndices().get(0).getName())); + assertThat( + ((UpdateSettingsRequest) updateSettingsRequest).settings().keySet(), + containsInAnyOrder( + MergePolicyConfig.INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING.getKey(), + MergePolicyConfig.INDEX_MERGE_POLICY_MERGE_FACTOR_SETTING.getKey() + ) + ); } public void testRetentionSkippedWhilstDownsamplingInProgress() { @@ -1445,6 +1485,13 @@ public void testTargetIndices() { String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); int numBackingIndices = 3; int numFailureIndices = 2; + int mutationBranch = randomIntBetween(0, 2); + DataStreamOptions dataStreamOptions = switch (mutationBranch) { + case 0 -> DataStreamOptions.EMPTY; + case 1 -> DataStreamOptions.FAILURE_STORE_ENABLED; + case 2 -> DataStreamOptions.FAILURE_STORE_DISABLED; + default -> throw new IllegalStateException("Unexpected value: " + mutationBranch); + }; Metadata.Builder builder = Metadata.builder(); DataStream dataStream = createDataStream( builder, @@ -1454,7 +1501,7 @@ public void testTargetIndices() { settings(IndexVersion.current()), new DataStreamLifecycle(), now - ).copy().setFailureStoreEnabled(randomBoolean()).build(); // failure store is managed even when disabled + ).copy().setDataStreamOptions(dataStreamOptions).build(); // failure store is managed even when disabled builder.put(dataStream); Metadata metadata = builder.build(); Set indicesToExclude = Set.of(dataStream.getIndices().get(0), dataStream.getFailureIndices().getIndices().get(0)); @@ -1486,7 +1533,7 @@ public void testFailureStoreIsManagedEvenWhenDisabled() { settings(IndexVersion.current()), DataStreamLifecycle.newBuilder().dataRetention(0).build(), now - ).copy().setFailureStoreEnabled(false).build(); // failure store is managed even when it is disabled + ).copy().setDataStreamOptions(DataStreamOptions.FAILURE_STORE_DISABLED).build(); // failure store is managed even when disabled builder.put(dataStream); ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(builder).build(); @@ -1496,17 +1543,11 @@ public void testFailureStoreIsManagedEvenWhenDisabled() { assertThat(clientSeenRequests.get(0), instanceOf(RolloverRequest.class)); RolloverRequest rolloverBackingIndexRequest = (RolloverRequest) clientSeenRequests.get(0); assertThat(rolloverBackingIndexRequest.getRolloverTarget(), is(dataStreamName)); - assertThat( - rolloverBackingIndexRequest.indicesOptions().failureStoreOptions(), - equalTo(new IndicesOptions.FailureStoreOptions(true, false)) - ); + assertThat(rolloverBackingIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.DATA)); assertThat(clientSeenRequests.get(1), instanceOf(RolloverRequest.class)); RolloverRequest rolloverFailureIndexRequest = (RolloverRequest) clientSeenRequests.get(1); assertThat(rolloverFailureIndexRequest.getRolloverTarget(), is(dataStreamName)); - assertThat( - rolloverFailureIndexRequest.indicesOptions().failureStoreOptions(), - equalTo(new IndicesOptions.FailureStoreOptions(false, true)) - ); + assertThat(rolloverFailureIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.FAILURES)); assertThat( ((DeleteIndexRequest) clientSeenRequests.get(2)).indices()[0], is(dataStream.getFailureIndices().getIndices().get(0).getName()) diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/DataStreamLifecycleStatsResponseTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/DataStreamLifecycleStatsResponseTests.java index 111d1b61da8c9..de2cc6644aa68 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/DataStreamLifecycleStatsResponseTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/DataStreamLifecycleStatsResponseTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle.action; diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsActionTests.java index 8c423107ea2f4..5bfd9fb25ca63 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsActionTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsActionTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle.action; diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/downsampling/DeleteSourceAndAddDownsampleIndexExecutorTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/downsampling/DeleteSourceAndAddDownsampleIndexExecutorTests.java index e21fda0fe579b..38656fbc51ab8 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/downsampling/DeleteSourceAndAddDownsampleIndexExecutorTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/downsampling/DeleteSourceAndAddDownsampleIndexExecutorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle.downsampling; diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/downsampling/DeleteSourceAndAddDownsampleToDSTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/downsampling/DeleteSourceAndAddDownsampleToDSTests.java index 062d502ee7029..de89500680efc 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/downsampling/DeleteSourceAndAddDownsampleToDSTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/downsampling/DeleteSourceAndAddDownsampleToDSTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle.downsampling; diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthIndicatorServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthIndicatorServiceTests.java index 79596cfced99a..7275e0253713c 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthIndicatorServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthIndicatorServiceTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle.health; diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthInfoPublisherTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthInfoPublisherTests.java index 31393a3fc18ed..cff6127e0729e 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthInfoPublisherTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthInfoPublisherTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.lifecycle.health; diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java index 89625ee8a312e..e009db7209eab 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.mapper; @@ -47,7 +48,8 @@ protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerConflictCheck( "enabled", timestampMapping(true, b -> b.startObject("@timestamp").field("type", "date").endObject()), - timestampMapping(false, b -> b.startObject("@timestamp").field("type", "date").endObject()) + timestampMapping(false, b -> b.startObject("@timestamp").field("type", "date").endObject()), + dm -> {} ); checker.registerUpdateCheck( timestampMapping(false, b -> b.startObject("@timestamp").field("type", "date").endObject()), diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/MetadataCreateDataStreamServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/MetadataCreateDataStreamServiceTests.java index 4d0a957428e81..cd9e44f1febd8 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/MetadataCreateDataStreamServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/MetadataCreateDataStreamServiceTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams.mapper; diff --git a/modules/data-streams/src/yamlRestTest/java/org/elasticsearch/datastreams/DataStreamsClientYamlTestSuiteIT.java b/modules/data-streams/src/yamlRestTest/java/org/elasticsearch/datastreams/DataStreamsClientYamlTestSuiteIT.java index 37a83deeb3550..22f2a9fa394fb 100644 --- a/modules/data-streams/src/yamlRestTest/java/org/elasticsearch/datastreams/DataStreamsClientYamlTestSuiteIT.java +++ b/modules/data-streams/src/yamlRestTest/java/org/elasticsearch/datastreams/DataStreamsClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.datastreams; @@ -49,6 +50,9 @@ private static ElasticsearchCluster createCluster() { .setting("xpack.security.enabled", "true") .keystore("bootstrap.password", "x-pack-test-password") .user("x_pack_rest_user", "x-pack-test-password"); + if (initTestSeed().nextBoolean()) { + clusterBuilder.setting("xpack.license.self_generated.type", "trial"); + } boolean setNodes = Boolean.parseBoolean(System.getProperty("yaml.rest.tests.set_num_nodes", "true")); if (setNodes) { clusterBuilder.nodes(2); diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml index 7d375d14f8d3a..0d19f555d10a4 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml @@ -115,8 +115,10 @@ setup: "Create hidden data stream": - requires: cluster_features: ["gte_v7.11.0"] + test_runner_features: ["warnings", "headers"] reason: "hidden data streams only available in 7.11" - do: + headers: { X-elastic-product-origin: elastic } allowed_warnings: - "index template [my-template3] has index patterns [.hidden-data-stream, hidden-data-stream] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template3] will take precedence during new index creation" indices.put_index_template: @@ -210,8 +212,12 @@ setup: --- "Create data stream with failure store": - requires: - cluster_features: ["gte_v8.15.0"] - reason: "data stream failure stores default settings changed in 8.15+" + test_runner_features: [ capabilities, allowed_warnings ] + reason: "Data stream failure stores config in templates was added in 8.18+" + capabilities: + - method: POST + path: /_index_template/{template} + capabilities: [ 'failure_store_in_template' ] - do: ingest.put_pipeline: @@ -254,8 +260,7 @@ setup: name: my-template4 body: index_patterns: [ failure-data-stream1, failure-data-stream2 ] - data_stream: - failure_store: true + data_stream: {} template: settings: index: @@ -267,6 +272,9 @@ setup: type: date count: type: long + data_stream_options: + failure_store: + enabled: true - do: indices.create_data_stream: @@ -630,8 +638,12 @@ setup: --- "Delete data stream with failure store": - requires: - cluster_features: ["gte_v8.15.0"] - reason: "data stream failure stores REST structure changed in 8.15+" + reason: "Data stream failure stores config in templates was added in 8.18+" + test_runner_features: [ allowed_warnings, capabilities ] + capabilities: + - method: POST + path: /_index_template/{template} + capabilities: [ 'failure_store_in_template' ] - do: allowed_warnings: @@ -640,8 +652,7 @@ setup: name: my-template4 body: index_patterns: [ failure-data-stream1 ] - data_stream: - failure_store: true + data_stream: {} template: mappings: properties: @@ -649,6 +660,9 @@ setup: type: date count: type: long + data_stream_options: + failure_store: + enabled: true - do: indices.create_data_stream: @@ -720,8 +734,12 @@ setup: --- "Delete data stream with failure store uninitialized": - requires: - cluster_features: ["gte_v8.15.0"] - reason: "data stream failure stores REST structure changed in 8.15+" + reason: "Data stream failure stores config in templates was added in 8.18+" + test_runner_features: [ capabilities, allowed_warnings ] + capabilities: + - method: POST + path: /_index_template/{template} + capabilities: [ 'failure_store_in_template' ] - do: allowed_warnings: @@ -730,8 +748,11 @@ setup: name: my-template4 body: index_patterns: [ failure-data-stream1 ] - data_stream: - failure_store: true + data_stream: {} + template: + data_stream_options: + failure_store: + enabled: true - do: indices.create_data_stream: diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index d20231a6d6cf2..9ea3bfefabdf8 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -182,6 +182,112 @@ index without timestamp: body: - '{"metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' +--- +TSDB failures go to failure store: + - requires: + reason: "Data stream failure stores config in templates was added in 8.18+" + test_runner_features: [ capabilities, allowed_warnings ] + capabilities: + - method: POST + path: /_index_template/{template} + capabilities: [ 'failure_store_in_template' ] + - do: + allowed_warnings: + - "index template [my-template2] has index patterns [fs-k8s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation" + indices.put_index_template: + name: my-template2 + body: + index_patterns: [ "fs-k8s*" ] + data_stream: {} + template: + settings: + index: + mode: time_series + number_of_replicas: 1 + number_of_shards: 2 + routing_path: [ metricset, time_series_dimension ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + data_stream_options: + failure_store: + enabled: true + mappings: + properties: + "@timestamp": + type: date + metricset: + type: keyword + time_series_dimension: true + k8s: + properties: + pod: + properties: + uid: + type: keyword + time_series_dimension: true + name: + type: keyword + ip: + type: ip + network: + properties: + tx: + type: long + rx: + type: long + - do: + index: + index: fs-k8s + body: + - '{"metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' + - match: { result : "created"} + - match: { failure_store : "used"} + + - do: + bulk: + refresh: true + body: + - '{ "create": { "_index": "fs-k8s"} }' + - '{"@timestamp":"2021-04-28T01:00:00Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' + - '{ "create": { "_index": "k8s"} }' + - '{ "@timestamp": "2021-04-28T01:00:00Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' + - '{ "create": { "_index": "fs-k8s"} }' + - '{ "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' + - '{ "create": { "_index": "fs-k8s"} }' + - '{ "@timestamp":"2000-04-28T01:00:00Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' + - '{ "create": { "_index": "k8s"} }' + - '{"metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' + - '{ "create": { "_index": "k8s"} }' + - '{ "@timestamp":"2000-04-28T01:00:00Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' + - is_true: errors + + # Successfully indexed to backing index + - match: { items.0.create._index: '/\.ds-fs-k8s-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { items.0.create.status: 201 } + - is_false: items.0.create.failure_store + - match: { items.1.create._index: '/\.ds-k8s-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { items.1.create.status: 201 } + - is_false: items.1.create.failure_store + + # Successfully indexed to failure store + - match: { items.2.create._index: '/\.fs-fs-k8s-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + - match: { items.2.create.status: 201 } + - match: { items.2.create.failure_store: used } + - match: { items.3.create._index: '/\.fs-fs-k8s-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + - match: { items.3.create.status: 201 } + - match: { items.3.create.failure_store: used } + + # Rejected, eligible to go to failure store, but failure store not enabled + - match: { items.4.create._index: 'k8s' } + - match: { items.4.create.status: 400 } + - match: { items.4.create.error.type: timestamp_error } + - match: { items.4.create.failure_store: not_enabled } + - match: { items.4.create._index: 'k8s' } + - match: { items.4.create.status: 400 } + - match: { items.4.create.error.type: timestamp_error } + - match: { items.4.create.failure_store: not_enabled } + --- index without timestamp with pipeline: - do: @@ -1230,3 +1336,83 @@ non string dimension fields: - match: { .$idx0name.mappings.properties.attributes.properties.double.time_series_dimension: true } - match: { .$idx0name.mappings.properties.attributes.properties.host\.ip.type: 'ip' } - match: { .$idx0name.mappings.properties.attributes.properties.host\.ip.time_series_dimension: true } + +--- +multi value dimensions: + - requires: + cluster_features: ["routing.multi_value_routing_path"] + reason: support for multi-value dimensions + + - do: + allowed_warnings: + - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation" + indices.put_index_template: + name: my-dynamic-template + body: + index_patterns: [k9s*] + data_stream: {} + template: + settings: + index: + number_of_shards: 1 + mode: time_series + time_series: + start_time: 2023-08-31T13:03:08.138Z + + mappings: + properties: + attributes: + type: passthrough + dynamic: true + time_series_dimension: true + priority: 1 + dynamic_templates: + - counter_metric: + mapping: + type: integer + time_series_metric: counter + + - do: + bulk: + index: k9s + refresh: true + body: + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "attributes": { "dim1": ["a" , "b"], "dim2": [1, 2] } }' + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "20", "attributes": { "dim1": ["b" , "a"], "dim2": [1, 2] } }' + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "20", "attributes": { "dim1": ["c" , "b"], "dim2": [1, 2] } }' + - is_false: errors + + - do: + search: + index: k9s + body: + size: 0 + aggs: + tsids: + terms: + field: _tsid + + - length: { aggregations.tsids.buckets: 3 } # only the order of the dim1 attribute is different, yet we expect to have two distinct time series + + - do: + search: + index: k9s + body: + size: 0 + aggs: + dims: + terms: + field: dim1 + order: + _key: asc + + - length: { aggregations.dims.buckets: 3 } + - match: { aggregations.dims.buckets.0.key: a } + - match: { aggregations.dims.buckets.0.doc_count: 2 } + - match: { aggregations.dims.buckets.1.key: b } + - match: { aggregations.dims.buckets.1.doc_count: 3 } + - match: { aggregations.dims.buckets.2.key: c } + - match: { aggregations.dims.buckets.2.doc_count: 1 } diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/170_modify_data_stream.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/170_modify_data_stream.yml index 3c6d29d939226..13f79e95d99f4 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/170_modify_data_stream.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/170_modify_data_stream.yml @@ -92,9 +92,12 @@ --- "Modify a data stream's failure store": - requires: - cluster_features: ["gte_v8.15.0"] - reason: "data stream failure stores REST structure changed in 8.15+" - test_runner_features: [ "allowed_warnings" ] + reason: "Data stream failure stores config in templates was added in 8.18+" + test_runner_features: [ capabilities, allowed_warnings ] + capabilities: + - method: POST + path: /_index_template/{template} + capabilities: [ 'failure_store_in_template' ] - do: allowed_warnings: @@ -103,8 +106,7 @@ name: my-template body: index_patterns: [data-*] - data_stream: - failure_store: true + data_stream: {} template: mappings: properties: @@ -112,6 +114,9 @@ type: date count: type: long + data_stream_options: + failure_store: + enabled: true - do: indices.create_data_stream: diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml index af3204ed443ab..2f6b7a0bff34b 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml @@ -1,3 +1,15 @@ +setup: + - requires: + reason: "Data stream options was added in 8.18+" + test_runner_features: [ capabilities, allowed_warnings, contains ] + capabilities: + - method: POST + path: /{index}/_doc + capabilities: [ 'failure_store_status' ] + - method: POST + path: /_index_template/{template} + capabilities: [ 'failure_store_in_template' ] + --- teardown: - do: @@ -32,11 +44,6 @@ teardown: --- "Redirect ingest failure in data stream to failure store": - - requires: - cluster_features: ["gte_v8.15.0"] - reason: "data stream failure stores REST structure changed in 8.15+" - test_runner_features: [allowed_warnings, contains] - - do: ingest.put_pipeline: id: "failing_pipeline" @@ -76,14 +83,16 @@ teardown: name: generic_logs_template body: index_patterns: logs-* - data_stream: - failure_store: true + data_stream: {} template: settings: number_of_shards: 1 number_of_replicas: 1 index: default_pipeline: "parent_failing_pipeline" + data_stream_options: + failure_store: + enabled: true - do: index: @@ -92,6 +101,8 @@ teardown: body: '@timestamp': '2020-12-12' foo: bar + - match: { failure_store: used} + - match: { _index: '/\.fs-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/'} - do: indices.get_data_stream: @@ -143,11 +154,6 @@ teardown: --- "Redirect shard failure in data stream to failure store": - - requires: - cluster_features: ["gte_v8.14.0"] - reason: "data stream failure stores only redirect shard failures in 8.14+" - test_runner_features: [allowed_warnings, contains] - - do: allowed_warnings: - "index template [generic_logs_template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [generic_logs_template] will take precedence during new index creation" @@ -155,8 +161,7 @@ teardown: name: generic_logs_template body: index_patterns: logs-* - data_stream: - failure_store: true + data_stream: {} template: settings: number_of_shards: 1 @@ -167,7 +172,9 @@ teardown: type: date count: type: long - + data_stream_options: + failure_store: + enabled: true - do: index: @@ -176,6 +183,8 @@ teardown: body: '@timestamp': '2020-12-12' count: 'invalid value' + - match: { _index: '/\.fs-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000002/'} + - match: { failure_store: used} - do: indices.get_data_stream: @@ -222,14 +231,6 @@ teardown: --- "Ensure failure is redirected to correct failure store after a reroute processor": - - skip: - known_issues: - - cluster_feature: "gte_v8.15.0" - fixed_by: "gte_v8.16.0" - reason: "Failure store documents contained the original index name rather than the rerouted one before v8.16.0" - - requires: - test_runner_features: [allowed_warnings] - - do: ingest.put_pipeline: id: "failing_pipeline" @@ -254,14 +255,16 @@ teardown: name: destination_template body: index_patterns: destination-data-stream - data_stream: - failure_store: true + data_stream: {} template: settings: number_of_shards: 1 number_of_replicas: 1 index: default_pipeline: "failing_pipeline" + data_stream_options: + failure_store: + enabled: true - do: indices.create_data_stream: @@ -307,6 +310,7 @@ teardown: body: '@timestamp': '2020-12-12' foo: bar + - match: { failure_store: used} - do: search: @@ -322,11 +326,6 @@ teardown: --- "Failure redirects to original failure store during index change if self referenced": - - requires: - cluster_features: [ "gte_v8.15.0" ] - reason: "data stream failure stores REST structure changed in 8.15+" - test_runner_features: [ allowed_warnings, contains ] - - do: ingest.put_pipeline: id: "failing_pipeline" @@ -356,14 +355,16 @@ teardown: name: generic_logs_template body: index_patterns: logs-* - data_stream: - failure_store: true + data_stream: {} template: settings: number_of_shards: 1 number_of_replicas: 1 index: default_pipeline: "failing_pipeline" + data_stream_options: + failure_store: + enabled: true - do: index: @@ -421,11 +422,6 @@ teardown: --- "Failure redirects to original failure store during index change if final pipeline changes target": - - requires: - cluster_features: [ "gte_v8.15.0" ] - reason: "data stream failure stores REST structure changed in 8.15+" - test_runner_features: [ allowed_warnings, contains ] - - do: ingest.put_pipeline: id: "change_index_pipeline" @@ -450,14 +446,16 @@ teardown: name: generic_logs_template body: index_patterns: logs-* - data_stream: - failure_store: true + data_stream: {} template: settings: number_of_shards: 1 number_of_replicas: 1 index: final_pipeline: "change_index_pipeline" + data_stream_options: + failure_store: + enabled: true - do: index: @@ -466,6 +464,7 @@ teardown: body: '@timestamp': '2020-12-12' foo: bar + - match: { failure_store: used} - do: indices.get_data_stream: @@ -513,11 +512,6 @@ teardown: --- "Failure redirects to correct failure store when index loop is detected": - - requires: - cluster_features: [ "gte_v8.15.0" ] - reason: "data stream failure stores REST structure changed in 8.15+" - test_runner_features: [ allowed_warnings, contains ] - - do: ingest.put_pipeline: id: "send_to_destination" @@ -559,14 +553,16 @@ teardown: name: generic_logs_template body: index_patterns: logs-* - data_stream: - failure_store: true + data_stream: {} template: settings: number_of_shards: 1 number_of_replicas: 1 index: default_pipeline: "send_to_destination" + data_stream_options: + failure_store: + enabled: true - do: allowed_warnings: @@ -575,14 +571,16 @@ teardown: name: destination_logs_template body: index_patterns: destination-* - data_stream: - failure_store: true + data_stream: {} template: settings: number_of_shards: 1 number_of_replicas: 1 index: default_pipeline: "send_back_to_original" + data_stream_options: + failure_store: + enabled: true - do: index: @@ -591,6 +589,8 @@ teardown: body: '@timestamp': '2020-12-12' foo: bar + - match: { _index: '/\.fs-destination-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { failure_store: used} - do: @@ -639,11 +639,6 @@ teardown: --- "Failure redirects to correct failure store when pipeline loop is detected": - - requires: - cluster_features: [ "gte_v8.15.0" ] - reason: "data stream failure stores REST structure changed in 8.15+" - test_runner_features: [ allowed_warnings, contains ] - - do: ingest.put_pipeline: id: "step_1" @@ -685,14 +680,16 @@ teardown: name: generic_logs_template body: index_patterns: logs-* - data_stream: - failure_store: true + data_stream: {} template: settings: number_of_shards: 1 number_of_replicas: 1 index: default_pipeline: "step_1" + data_stream_options: + failure_store: + enabled: true - do: index: @@ -701,6 +698,7 @@ teardown: body: '@timestamp': '2020-12-12' foo: bar + - match: { failure_store: used} - do: indices.get_data_stream: @@ -751,11 +749,44 @@ teardown: --- "Version conflicts are not redirected to failure store": - - requires: - cluster_features: ["gte_v8.16.0"] - reason: "Redirecting version conflicts to the failure store is considered a bug fixed in 8.16" - test_runner_features: [allowed_warnings, contains] + - do: + allowed_warnings: + - "index template [generic_logs_template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [generic_logs_template] will take precedence during new index creation" + indices.put_index_template: + name: generic_logs_template + body: + index_patterns: logs-* + data_stream: {} + template: + settings: + number_of_shards: 1 + number_of_replicas: 1 + mappings: + properties: + '@timestamp': + type: date + count: + type: long + data_stream_options: + failure_store: + enabled: true + - do: + bulk: + refresh: true + body: + - '{ "create": { "_index": "logs-foobar", "_id": "1" } }' + - '{ "@timestamp": "2022-01-01", "baz": "quick", "a": "brown", "b": "fox" }' + - '{ "create": { "_index": "logs-foobar", "_id": "1" } }' + - '{ "@timestamp": "2022-01-01", "baz": "lazy", "a": "dog" }' + - is_true: errors + - match: { items.1.create._index: '/\.ds-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { items.1.create.status: 409 } + - match: { items.1.create.error.type: version_conflict_engine_exception} + - is_false: items.1.create.failure_store + +--- +"Test failure store status with bulk request": - do: allowed_warnings: - "index template [generic_logs_template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [generic_logs_template] will take precedence during new index creation" @@ -763,8 +794,7 @@ teardown: name: generic_logs_template body: index_patterns: logs-* - data_stream: - failure_store: true + data_stream: {} template: settings: number_of_shards: 1 @@ -775,6 +805,31 @@ teardown: type: date count: type: long + data_stream_options: + failure_store: + enabled: true + - do: + allowed_warnings: + - "index template [no-fs] has index patterns [no-fs*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [no-fs] will take precedence during new index creation" + indices.put_index_template: + name: no-fs + body: + index_patterns: no-fs* + data_stream: {} + template: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + '@timestamp': + type: date + count: + type: long + data_stream_options: + failure_store: + enabled: false + - do: bulk: @@ -784,7 +839,29 @@ teardown: - '{ "@timestamp": "2022-01-01", "baz": "quick", "a": "brown", "b": "fox" }' - '{ "create": { "_index": "logs-foobar", "_id": "1" } }' - '{ "@timestamp": "2022-01-01", "baz": "lazy", "a": "dog" }' + - '{ "create": { "_index": "logs-foobar", "_id": "1" } }' + - '{ "@timestamp": "2022-01-01", "count": "invalid" }' + - '{ "create": { "_index": "no-fs", "_id": "1" } }' + - '{ "@timestamp": "2022-01-01", "count": "invalid" }' - is_true: errors + # Successfully indexed to backing index + - match: { items.0.create._index: '/\.ds-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { items.0.create.status: 201 } + - is_false: items.0.create.failure_store + + # Rejected but not eligible to go to failure store - match: { items.1.create._index: '/\.ds-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - match: { items.1.create.status: 409 } - match: { items.1.create.error.type: version_conflict_engine_exception} + - is_false: items.1.create.failure_store + + # Successfully indexed to failure store + - match: { items.2.create._index: '/\.fs-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + - match: { items.2.create.status: 201 } + - match: { items.2.create.failure_store: used } + + # Rejected, eligible to go to failure store, but failure store not enabled + - match: { items.3.create._index: '/\.ds-no-fs-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { items.3.create.status: 400 } + - match: { items.3.create.error.type: document_parsing_exception } + - match: { items.3.create.failure_store: not_enabled } diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_require_data_stream.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/200_require_data_stream.yml similarity index 100% rename from modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_require_data_stream.yml rename to modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/200_require_data_stream.yml diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/200_rollover_failure_store.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/200_rollover_failure_store.yml deleted file mode 100644 index 0742435f045fb..0000000000000 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/200_rollover_failure_store.yml +++ /dev/null @@ -1,477 +0,0 @@ ---- -setup: - - requires: - cluster_features: ["gte_v8.15.0"] - reason: "data stream failure stores REST structure changed in 8.15+" - test_runner_features: [allowed_warnings, contains, capabilities] - - - do: - allowed_warnings: - - "index template [my-template] has index patterns [data-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" - indices.put_index_template: - name: my-template - body: - index_patterns: [data-*] - data_stream: - failure_store: true - template: - mappings: - properties: - '@timestamp': - type: date - count: - type: long - - - do: - indices.create_data_stream: - name: data-stream-for-rollover - ---- -teardown: - - do: - indices.delete_data_stream: - name: data-stream-for-lazy-rollover - ignore: 404 - - - do: - ingest.delete_pipeline: - id: failing_pipeline - ignore: 404 ---- -"Roll over a data stream's failure store without conditions": - # Initialize failure store - - do: - index: - index: data-stream-for-rollover - refresh: true - body: - '@timestamp': '2020-12-12' - count: 'invalid value' - - - do: - indices.rollover: - alias: "data-stream-for-rollover" - target_failure_store: true - - - match: { acknowledged: true } - - match: { old_index: "/\\.fs-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" } - - match: { new_index: "/\\.fs-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000003/" } - - match: { rolled_over: true } - - match: { dry_run: false } - - - do: - indices.get_data_stream: - name: "*" - - match: { data_streams.0.name: data-stream-for-rollover } - - match: { data_streams.0.timestamp_field.name: '@timestamp' } - # Both backing and failure indices use the same generation field. - - match: { data_streams.0.generation: 3 } - - length: { data_streams.0.indices: 1 } - - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - - length: { data_streams.0.failure_store.indices: 2 } - - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } - - match: { data_streams.0.failure_store.indices.1.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000003/' } - ---- -"Roll over a data stream's failure store with conditions": - - do: - index: - index: data-stream-for-rollover - refresh: true - body: - '@timestamp': '2020-12-12' - count: 'invalid value' - - - do: - indices.rollover: - alias: "data-stream-for-rollover" - target_failure_store: true - body: - conditions: - max_docs: 1 - - - match: { acknowledged: true } - - match: { old_index: "/\\.fs-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" } - - match: { new_index: "/\\.fs-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000003/" } - - match: { rolled_over: true } - - match: { dry_run: false } - - - do: - indices.get_data_stream: - name: "*" - - match: { data_streams.0.name: data-stream-for-rollover } - - match: { data_streams.0.timestamp_field.name: '@timestamp' } - # Both backing and failure indices use the same generation field. - - match: { data_streams.0.generation: 3 } - - length: { data_streams.0.indices: 1 } - - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - - length: { data_streams.0.failure_store.indices: 2 } - - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } - - match: { data_streams.0.failure_store.indices.1.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000003/' } - ---- -"Don't roll over a data stream's failure store when conditions aren't met": - # Initialize failure store - - do: - index: - index: data-stream-for-rollover - refresh: true - body: - '@timestamp': '2020-12-12' - count: 'invalid value' - - - do: - indices.rollover: - alias: "data-stream-for-rollover" - target_failure_store: true - body: - conditions: - max_primary_shard_docs: 2 - - - match: { acknowledged: false } - - match: { rolled_over: false } - - match: { dry_run: false } - - - do: - indices.get_data_stream: - name: "*" - - match: { data_streams.0.name: data-stream-for-rollover } - - match: { data_streams.0.timestamp_field.name: '@timestamp' } - - match: { data_streams.0.generation: 2 } - - length: { data_streams.0.indices: 1 } - - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - - length: { data_streams.0.failure_store.indices: 1 } - - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } - ---- -"Lazily roll over a data stream's failure store after a shard failure": - - requires: - reason: "data stream failure store lazy rollover only supported in 8.15+" - test_runner_features: [allowed_warnings, capabilities] - capabilities: - - method: POST - path: /{index}/_rollover - capabilities: [lazy-rollover-failure-store] - - # Initialize failure store - - do: - index: - index: data-stream-for-rollover - refresh: true - body: - '@timestamp': '2020-12-12' - count: 'invalid value' - - # Mark the failure store for lazy rollover - - do: - indices.rollover: - alias: "data-stream-for-rollover" - target_failure_store: true - lazy: true - - - match: { acknowledged: true } - - match: { rolled_over: false } - - match: { dry_run: false } - - - do: - indices.get_data_stream: - name: "*" - - match: { data_streams.0.name: data-stream-for-rollover } - - match: { data_streams.0.timestamp_field.name: '@timestamp' } - - match: { data_streams.0.generation: 2 } - - length: { data_streams.0.indices: 1 } - - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - - length: { data_streams.0.failure_store.indices: 1 } - - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } - - - do: - index: - index: data-stream-for-rollover - refresh: true - body: - '@timestamp': '2020-12-12' - count: 'invalid value' - - - do: - indices.get_data_stream: - name: "*" - - match: { data_streams.0.name: data-stream-for-rollover } - - match: { data_streams.0.timestamp_field.name: '@timestamp' } - # Both backing and failure indices use the same generation field. - - match: { data_streams.0.generation: 3 } - - length: { data_streams.0.indices: 1 } - - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - - length: { data_streams.0.failure_store.indices: 2 } - - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } - - match: { data_streams.0.failure_store.indices.1.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000003/' } - - # Ensure failure got redirected to new index (after rollover). - - do: - search: - index: .fs-data-stream-for-rollover-* - - length: { hits.hits: 2 } - - match: { hits.hits.0._index: "/\\.fs-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" } - - match: { hits.hits.1._index: "/\\.fs-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000003/" } - ---- -"Lazily roll over a data stream's failure store after an ingest failure": - - requires: - reason: "data stream failure store lazy rollover only supported in 8.15+" - test_runner_features: [allowed_warnings, capabilities] - capabilities: - - method: POST - path: /{index}/_rollover - capabilities: [lazy-rollover-failure-store] - - - do: - ingest.put_pipeline: - id: "failing_pipeline" - body: > - { - "description": "_description", - "processors": [ - { - "fail" : { - "message" : "error_message" - } - } - ] - } - - match: { acknowledged: true } - - - do: - allowed_warnings: - - "index template [my-template] has index patterns [data-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" - indices.put_index_template: - name: my-template - body: - index_patterns: [data-*] - data_stream: - failure_store: true - template: - settings: - index: - default_pipeline: "failing_pipeline" - - - do: - indices.create_data_stream: - name: data-stream-for-lazy-rollover - - # Initialize failure store - - do: - index: - index: data-stream-for-lazy-rollover - refresh: true - body: - '@timestamp': '2020-12-12' - count: 'invalid value' - - # Mark the failure store for lazy rollover - - do: - indices.rollover: - alias: data-stream-for-lazy-rollover - target_failure_store: true - lazy: true - - - match: { acknowledged: true } - - match: { rolled_over: false } - - match: { dry_run: false } - - - do: - indices.get_data_stream: - name: "*" - - match: { data_streams.0.name: data-stream-for-lazy-rollover } - - match: { data_streams.0.timestamp_field.name: '@timestamp' } - - match: { data_streams.0.generation: 2 } - - length: { data_streams.0.indices: 1 } - - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-lazy-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - - length: { data_streams.0.failure_store.indices: 1 } - - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-lazy-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } - - - do: - index: - index: data-stream-for-lazy-rollover - refresh: true - body: - '@timestamp': '2020-12-12' - count: 1 - - - do: - indices.get_data_stream: - name: "*" - - match: { data_streams.0.name: data-stream-for-lazy-rollover } - - match: { data_streams.0.timestamp_field.name: '@timestamp' } - # Both backing and failure indices use the same generation field. - - match: { data_streams.0.generation: 3 } - - length: { data_streams.0.indices: 1 } - - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-lazy-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - - length: { data_streams.0.failure_store.indices: 2 } - - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-lazy-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } - - match: { data_streams.0.failure_store.indices.1.index_name: '/\.fs-data-stream-for-lazy-rollover-(\d{4}\.\d{2}\.\d{2}-)?000003/' } - - # Ensure failure got redirected to new index (after rollover). - - do: - search: - index: .fs-data-stream-for-lazy-rollover-* - - length: { hits.hits: 2 } - - match: { hits.hits.0._index: "/\\.fs-data-stream-for-lazy-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" } - - match: { hits.hits.1._index: "/\\.fs-data-stream-for-lazy-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000003/" } ---- -"A failure store marked for lazy rollover should only be rolled over when there is a failure": - - requires: - reason: "data stream failure store lazy rollover only supported in 8.15+" - test_runner_features: [allowed_warnings, capabilities] - capabilities: - - method: POST - path: /{index}/_rollover - capabilities: [lazy-rollover-failure-store] - - # Initialize failure store - - do: - index: - index: data-stream-for-rollover - refresh: true - body: - '@timestamp': '2020-12-12' - count: 'invalid value' - - # Mark the failure store for lazy rollover - - do: - indices.rollover: - alias: "data-stream-for-rollover" - target_failure_store: true - lazy: true - - - match: { acknowledged: true } - - match: { rolled_over: false } - - match: { dry_run: false } - - - do: - indices.get_data_stream: - name: "*" - - match: { data_streams.0.name: data-stream-for-rollover } - - match: { data_streams.0.timestamp_field.name: '@timestamp' } - - match: { data_streams.0.generation: 2 } - - length: { data_streams.0.indices: 1 } - - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - - length: { data_streams.0.failure_store.indices: 1 } - - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } - - - do: - index: - index: data-stream-for-rollover - refresh: true - body: - '@timestamp': '2020-12-12' - count: 3 - - - do: - indices.get_data_stream: - name: "*" - - match: { data_streams.0.name: data-stream-for-rollover } - - match: { data_streams.0.timestamp_field.name: '@timestamp' } - # Both backing and failure indices use the same generation field. - - match: { data_streams.0.generation: 2 } - - length: { data_streams.0.indices: 1 } - - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - - length: { data_streams.0.failure_store.indices: 1 } - - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } - ---- -"Rolling over an uninitialized failure store should initialize it": - # Initializing with conditions is not allowed. - - do: - catch: /Rolling over\/initializing an empty failure store is only supported without conditions\./ - indices.rollover: - alias: "data-stream-for-rollover" - target_failure_store: true - body: - conditions: - max_docs: 1 - - - do: - indices.rollover: - alias: "data-stream-for-rollover" - target_failure_store: true - - - match: { acknowledged: true } - - match: { old_index: "_none_" } - - match: { new_index: "/\\.fs-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" } - - match: { rolled_over: true } - - match: { dry_run: false } - - - do: - indices.get_data_stream: - name: "*" - - match: { data_streams.0.name: data-stream-for-rollover } - - match: { data_streams.0.timestamp_field.name: '@timestamp' } - # Both backing and failure indices use the same generation field. - - match: { data_streams.0.generation: 2 } - - length: { data_streams.0.indices: 1 } - - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - - length: { data_streams.0.failure_store.indices: 1 } - - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } - ---- -"Rolling over a failure store on a data stream without the failure store enabled should work": - - do: - allowed_warnings: - - "index template [my-other-template] has index patterns [other-data-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-other-template] will take precedence during new index creation" - indices.put_index_template: - name: my-other-template - body: - index_patterns: [other-data-*] - data_stream: {} - - - do: - indices.create_data_stream: - name: other-data-stream-for-rollover - - # Initializing should work - - do: - indices.rollover: - alias: "other-data-stream-for-rollover" - target_failure_store: true - - - match: { acknowledged: true } - - match: { old_index: "_none_" } - - match: { new_index: "/\\.fs-other-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" } - - match: { rolled_over: true } - - match: { dry_run: false } - - - do: - indices.get_data_stream: - name: other-data-stream-for-rollover - - match: { data_streams.0.name: other-data-stream-for-rollover } - - match: { data_streams.0.timestamp_field.name: '@timestamp' } - # Both backing and failure indices use the same generation field. - - match: { data_streams.0.generation: 2 } - - length: { data_streams.0.indices: 1 } - - match: { data_streams.0.indices.0.index_name: '/\.ds-other-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - - length: { data_streams.0.failure_store.indices: 1 } - - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-other-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } - - # And "regular" rollover should work - - do: - indices.rollover: - alias: "other-data-stream-for-rollover" - target_failure_store: true - - - match: { acknowledged: true } - - match: { old_index: "/\\.fs-other-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" } - - match: { new_index: "/\\.fs-other-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000003/" } - - match: { rolled_over: true } - - match: { dry_run: false } - - - do: - indices.get_data_stream: - name: other-data-stream-for-rollover - - match: { data_streams.0.name: other-data-stream-for-rollover } - - match: { data_streams.0.timestamp_field.name: '@timestamp' } - # Both backing and failure indices use the same generation field. - - match: { data_streams.0.generation: 3 } - - length: { data_streams.0.indices: 1 } - - match: { data_streams.0.indices.0.index_name: '/\.ds-other-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - - length: { data_streams.0.failure_store.indices: 2 } - - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-other-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } - - match: { data_streams.0.failure_store.indices.1.index_name: '/\.fs-other-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000003/' } diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/210_rollover_failure_store.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/210_rollover_failure_store.yml new file mode 100644 index 0000000000000..cc3a11ffde5e8 --- /dev/null +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/210_rollover_failure_store.yml @@ -0,0 +1,471 @@ +--- +setup: + - requires: + reason: "Data stream failure stores config in templates was added in 8.16+" + test_runner_features: [ capabilities, allowed_warnings ] + capabilities: + - method: POST + path: /_index_template/{template} + capabilities: [ 'failure_store_in_template' ] + - method: POST + path: /{index}/_rollover + capabilities: [ 'lazy-rollover-failure-store' ] + + - do: + allowed_warnings: + - "index template [my-template] has index patterns [data-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" + indices.put_index_template: + name: my-template + body: + index_patterns: [data-*] + data_stream: {} + template: + mappings: + properties: + '@timestamp': + type: date + count: + type: long + data_stream_options: + failure_store: + enabled: true + + - do: + indices.create_data_stream: + name: data-stream-for-rollover + +--- +teardown: + - do: + indices.delete_data_stream: + name: data-stream-for-lazy-rollover + ignore: 404 + + - do: + ingest.delete_pipeline: + id: failing_pipeline + ignore: 404 +--- +"Roll over a data stream's failure store without conditions": + # Initialize failure store + - do: + index: + index: data-stream-for-rollover + refresh: true + body: + '@timestamp': '2020-12-12' + count: 'invalid value' + + - do: + indices.rollover: + alias: "data-stream-for-rollover" + target_failure_store: true + + - match: { acknowledged: true } + - match: { old_index: "/\\.fs-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" } + - match: { new_index: "/\\.fs-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000003/" } + - match: { rolled_over: true } + - match: { dry_run: false } + + - do: + indices.get_data_stream: + name: "*" + - match: { data_streams.0.name: data-stream-for-rollover } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + # Both backing and failure indices use the same generation field. + - match: { data_streams.0.generation: 3 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - length: { data_streams.0.failure_store.indices: 2 } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + - match: { data_streams.0.failure_store.indices.1.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000003/' } + +--- +"Roll over a data stream's failure store with conditions": + - do: + index: + index: data-stream-for-rollover + refresh: true + body: + '@timestamp': '2020-12-12' + count: 'invalid value' + + - do: + indices.rollover: + alias: "data-stream-for-rollover" + target_failure_store: true + body: + conditions: + max_docs: 1 + + - match: { acknowledged: true } + - match: { old_index: "/\\.fs-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" } + - match: { new_index: "/\\.fs-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000003/" } + - match: { rolled_over: true } + - match: { dry_run: false } + + - do: + indices.get_data_stream: + name: "*" + - match: { data_streams.0.name: data-stream-for-rollover } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + # Both backing and failure indices use the same generation field. + - match: { data_streams.0.generation: 3 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - length: { data_streams.0.failure_store.indices: 2 } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + - match: { data_streams.0.failure_store.indices.1.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000003/' } + +--- +"Don't roll over a data stream's failure store when conditions aren't met": + # Initialize failure store + - do: + index: + index: data-stream-for-rollover + refresh: true + body: + '@timestamp': '2020-12-12' + count: 'invalid value' + + - do: + indices.rollover: + alias: "data-stream-for-rollover" + target_failure_store: true + body: + conditions: + max_primary_shard_docs: 2 + + - match: { acknowledged: false } + - match: { rolled_over: false } + - match: { dry_run: false } + + - do: + indices.get_data_stream: + name: "*" + - match: { data_streams.0.name: data-stream-for-rollover } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + - match: { data_streams.0.generation: 2 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - length: { data_streams.0.failure_store.indices: 1 } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + +--- +"Lazily roll over a data stream's failure store after a shard failure": + # Initialize failure store + - do: + index: + index: data-stream-for-rollover + refresh: true + body: + '@timestamp': '2020-12-12' + count: 'invalid value' + + # Mark the failure store for lazy rollover + - do: + indices.rollover: + alias: "data-stream-for-rollover" + target_failure_store: true + lazy: true + + - match: { acknowledged: true } + - match: { rolled_over: false } + - match: { dry_run: false } + + - do: + indices.get_data_stream: + name: "*" + - match: { data_streams.0.name: data-stream-for-rollover } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + - match: { data_streams.0.generation: 2 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - length: { data_streams.0.failure_store.indices: 1 } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + + - do: + index: + index: data-stream-for-rollover + refresh: true + body: + '@timestamp': '2020-12-12' + count: 'invalid value' + + - do: + indices.get_data_stream: + name: "*" + - match: { data_streams.0.name: data-stream-for-rollover } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + # Both backing and failure indices use the same generation field. + - match: { data_streams.0.generation: 3 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - length: { data_streams.0.failure_store.indices: 2 } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + - match: { data_streams.0.failure_store.indices.1.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000003/' } + + # Ensure failure got redirected to new index (after rollover). + - do: + search: + index: .fs-data-stream-for-rollover-* + - length: { hits.hits: 2 } + - match: { hits.hits.0._index: "/\\.fs-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" } + - match: { hits.hits.1._index: "/\\.fs-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000003/" } + +--- +"Lazily roll over a data stream's failure store after an ingest failure": + - do: + ingest.put_pipeline: + id: "failing_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "fail" : { + "message" : "error_message" + } + } + ] + } + - match: { acknowledged: true } + + - do: + allowed_warnings: + - "index template [my-template] has index patterns [data-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" + indices.put_index_template: + name: my-template + body: + index_patterns: [data-*] + data_stream: {} + template: + settings: + index: + default_pipeline: "failing_pipeline" + data_stream_options: + failure_store: + enabled: true + + - do: + indices.create_data_stream: + name: data-stream-for-lazy-rollover + + # Initialize failure store + - do: + index: + index: data-stream-for-lazy-rollover + refresh: true + body: + '@timestamp': '2020-12-12' + count: 'invalid value' + + # Mark the failure store for lazy rollover + - do: + indices.rollover: + alias: data-stream-for-lazy-rollover + target_failure_store: true + lazy: true + + - match: { acknowledged: true } + - match: { rolled_over: false } + - match: { dry_run: false } + + - do: + indices.get_data_stream: + name: "*" + - match: { data_streams.0.name: data-stream-for-lazy-rollover } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + - match: { data_streams.0.generation: 2 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-lazy-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - length: { data_streams.0.failure_store.indices: 1 } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-lazy-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + + - do: + index: + index: data-stream-for-lazy-rollover + refresh: true + body: + '@timestamp': '2020-12-12' + count: 1 + + - do: + indices.get_data_stream: + name: "*" + - match: { data_streams.0.name: data-stream-for-lazy-rollover } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + # Both backing and failure indices use the same generation field. + - match: { data_streams.0.generation: 3 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-lazy-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - length: { data_streams.0.failure_store.indices: 2 } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-lazy-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + - match: { data_streams.0.failure_store.indices.1.index_name: '/\.fs-data-stream-for-lazy-rollover-(\d{4}\.\d{2}\.\d{2}-)?000003/' } + + # Ensure failure got redirected to new index (after rollover). + - do: + search: + index: .fs-data-stream-for-lazy-rollover-* + - length: { hits.hits: 2 } + - match: { hits.hits.0._index: "/\\.fs-data-stream-for-lazy-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" } + - match: { hits.hits.1._index: "/\\.fs-data-stream-for-lazy-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000003/" } +--- +"A failure store marked for lazy rollover should only be rolled over when there is a failure": + - requires: + reason: "data stream failure store lazy rollover only supported in 8.15+" + test_runner_features: [allowed_warnings, capabilities] + capabilities: + - method: POST + path: /{index}/_rollover + capabilities: [lazy-rollover-failure-store] + + # Initialize failure store + - do: + index: + index: data-stream-for-rollover + refresh: true + body: + '@timestamp': '2020-12-12' + count: 'invalid value' + + # Mark the failure store for lazy rollover + - do: + indices.rollover: + alias: "data-stream-for-rollover" + target_failure_store: true + lazy: true + + - match: { acknowledged: true } + - match: { rolled_over: false } + - match: { dry_run: false } + + - do: + indices.get_data_stream: + name: "*" + - match: { data_streams.0.name: data-stream-for-rollover } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + - match: { data_streams.0.generation: 2 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - length: { data_streams.0.failure_store.indices: 1 } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + + - do: + index: + index: data-stream-for-rollover + refresh: true + body: + '@timestamp': '2020-12-12' + count: 3 + + - do: + indices.get_data_stream: + name: "*" + - match: { data_streams.0.name: data-stream-for-rollover } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + # Both backing and failure indices use the same generation field. + - match: { data_streams.0.generation: 2 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - length: { data_streams.0.failure_store.indices: 1 } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + +--- +"Rolling over an uninitialized failure store should initialize it": + # Initializing with conditions is not allowed. + - do: + catch: /Rolling over\/initializing an empty failure store is only supported without conditions\./ + indices.rollover: + alias: "data-stream-for-rollover" + target_failure_store: true + body: + conditions: + max_docs: 1 + + - do: + indices.rollover: + alias: "data-stream-for-rollover" + target_failure_store: true + + - match: { acknowledged: true } + - match: { old_index: "_none_" } + - match: { new_index: "/\\.fs-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" } + - match: { rolled_over: true } + - match: { dry_run: false } + + - do: + indices.get_data_stream: + name: "*" + - match: { data_streams.0.name: data-stream-for-rollover } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + # Both backing and failure indices use the same generation field. + - match: { data_streams.0.generation: 2 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - length: { data_streams.0.failure_store.indices: 1 } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + +--- +"Rolling over a failure store on a data stream without the failure store enabled should work": + - do: + allowed_warnings: + - "index template [my-other-template] has index patterns [other-data-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-other-template] will take precedence during new index creation" + indices.put_index_template: + name: my-other-template + body: + index_patterns: [other-data-*] + data_stream: {} + + - do: + indices.create_data_stream: + name: other-data-stream-for-rollover + + # Initializing should work + - do: + indices.rollover: + alias: "other-data-stream-for-rollover" + target_failure_store: true + + - match: { acknowledged: true } + - match: { old_index: "_none_" } + - match: { new_index: "/\\.fs-other-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" } + - match: { rolled_over: true } + - match: { dry_run: false } + + - do: + indices.get_data_stream: + name: other-data-stream-for-rollover + - match: { data_streams.0.name: other-data-stream-for-rollover } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + # Both backing and failure indices use the same generation field. + - match: { data_streams.0.generation: 2 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-other-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - length: { data_streams.0.failure_store.indices: 1 } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-other-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + + # And "regular" rollover should work + - do: + indices.rollover: + alias: "other-data-stream-for-rollover" + target_failure_store: true + + - match: { acknowledged: true } + - match: { old_index: "/\\.fs-other-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" } + - match: { new_index: "/\\.fs-other-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000003/" } + - match: { rolled_over: true } + - match: { dry_run: false } + + - do: + indices.get_data_stream: + name: other-data-stream-for-rollover + - match: { data_streams.0.name: other-data-stream-for-rollover } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + # Both backing and failure indices use the same generation field. + - match: { data_streams.0.generation: 3 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-other-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - length: { data_streams.0.failure_store.indices: 2 } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-other-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + - match: { data_streams.0.failure_store.indices.1.index_name: '/\.fs-other-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000003/' } diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/220_failure_store_cluster_setting.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/220_failure_store_cluster_setting.yml new file mode 100644 index 0000000000000..90bd6fe406b57 --- /dev/null +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/220_failure_store_cluster_setting.yml @@ -0,0 +1,222 @@ +setup: + - requires: + reason: "Data stream options was added in 8.18+" + test_runner_features: [ capabilities, allowed_warnings, contains ] + capabilities: + - method: POST + path: /{index}/_doc + capabilities: [ 'failure_store_status' ] + - method: POST + path: /_index_template/{template} + capabilities: [ 'failure_store_in_template' ] + - method: PUT + path: /_cluster/settings + capabilities: [ 'data_stream_failure_store_cluster_setting' ] + + - do: + cluster.put_settings: + body: + persistent: + data_streams.failure_store.enabled: '*-matching' + + - do: + ingest.put_pipeline: + id: "failing_pipeline" + body: > + { + "processors": [ + { + "fail": { + "message" : "pipeline go boom" + } + } + ] + } + + - do: + indices.put_index_template: + name: index_template_default_fs + body: + index_patterns: default-fs-* + data_stream: {} + template: + settings: + number_of_shards: 1 + number_of_replicas: 1 + + - do: + cluster.put_component_template: + name: component_template_disabled_fs + body: + template: + data_stream_options: + failure_store: + enabled: false + + - do: + indices.put_index_template: + name: index_template_disabled_fs + body: + index_patterns: disabled-fs-* + data_stream: {} + composed_of: + - component_template_disabled_fs + template: + settings: + number_of_shards: 1 + number_of_replicas: 1 + +--- +teardown: + - do: + indices.delete_data_stream: + name: default-fs-matching + ignore: 404 + + - do: + indices.delete_data_stream: + name: disabled-fs-matching + ignore: 404 + + - do: + indices.delete_index_template: + name: index_template_disabled_fs + ignore: 404 + + - do: + cluster.delete_component_template: + name: component_template_disabled_fs + ignore: 404 + + - do: + indices.delete_index_template: + name: index_template_default_fs + ignore: 404 + + - do: + ingest.delete_pipeline: + id: "failing_pipeline" + ignore: 404 + + - do: + cluster.put_settings: + body: + persistent: + data_streams.failure_store.enabled: null + +--- +"Redirect ingest failure when auto-creating data stream to failure store when enabled by setting": + - do: + index: + index: default-fs-matching + refresh: true + pipeline: 'failing_pipeline' + body: + '@timestamp': '2020-12-12' + foo: bar + - match: { failure_store: used } + - match: { _index: '/\.fs-default-fs-matching-(\d{4}\.\d{2}\.\d{2}-)?\d{6}/' } + +--- +"Redirect ingest failure into pre-existing data stream to failure store when enabled by setting": + - do: + indices.create_data_stream: + name: default-fs-matching + + - do: + index: + index: default-fs-matching + refresh: true + pipeline: 'failing_pipeline' + body: + '@timestamp': '2020-12-12' + foo: bar + - match: { failure_store: used } + - match: { _index: '/\.fs-default-fs-matching-(\d{4}\.\d{2}\.\d{2}-)?\d{6}/' } + +--- +"Do not redirect ingest failure when auto-creating data stream to failure store when enabled by setting but disabled in template": + - do: + index: + index: disabled-fs-matching + refresh: true + pipeline: 'failing_pipeline' + body: + '@timestamp': '2020-12-12' + foo: bar + catch: '/pipeline go boom/' + +--- +"Do not redirect ingest failure into pre-existing data stream to failure store when enabled by setting but disabled in template": + - do: + indices.create_data_stream: + name: disabled-fs-matching + + - do: + index: + index: disabled-fs-matching + refresh: true + pipeline: 'failing_pipeline' + body: + '@timestamp': '2020-12-12' + foo: bar + catch: '/pipeline go boom/' + +--- +"Redirect mapping failure when auto-creating data stream to failure store when enabled by setting": + - do: + index: + index: default-fs-matching + refresh: true + body: + '@timestamp': 'not a timestamp' + foo: bar + - match: { failure_store: used } + - match: { _index: '/\.fs-default-fs-matching-(\d{4}\.\d{2}\.\d{2}-)?\d{6}/' } + +--- +"Redirect mapping failure into pre-existing data stream to failure store when enabled by setting": + - do: + indices.create_data_stream: + name: default-fs-matching + + - do: + index: + index: default-fs-matching + refresh: true + body: + '@timestamp': 'not a timestamp' + foo: bar + - match: { failure_store: used } + - match: { _index: '/\.fs-default-fs-matching-(\d{4}\.\d{2}\.\d{2}-)?\d{6}/' } + +--- +"Do not redirect mapping failure when auto-creating data stream to failure store when enabled by setting but disabled in template": + - do: + index: + index: disabled-fs-matching + refresh: true + body: + '@timestamp': 'not a timestamp' + foo: bar + catch: '/failed to parse field/' + +--- +"Do not redirect mapping failure into pre-existing data stream to failure store when enabled by setting but disabled in template": + - do: + indices.create_data_stream: + name: disabled-fs-matching + + - do: + index: + index: disabled-fs-matching + refresh: true + body: + '@timestamp': 'not a timestamp' + foo: bar + catch: '/failed to parse field/' + +# See also DataStreamOptionsIT for tests of the interaction between the failure store cluster setting and using +# the /_data_stream/{name}/_options API to explicitly enable and disable the failure store. (At time of writing, these +# can only be done in a Java REST test case, not a YAML one, because the failure store is behind a feature gate and so +# the REST API spec has not been added.) diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/30_auto_create_data_stream.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/30_auto_create_data_stream.yml index 61d17c3d675cf..60500767213af 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/30_auto_create_data_stream.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/30_auto_create_data_stream.yml @@ -50,9 +50,12 @@ --- "Don't initialize failure store during data stream auto-creation on successful index": - requires: - cluster_features: ["gte_v8.15.0"] - reason: "data stream failure stores REST structure changed in 8.15+" - test_runner_features: allowed_warnings + reason: "Data stream failure stores config in templates was added in 8.18+" + test_runner_features: [allowed_warnings, capabilities] + capabilities: + - method: POST + path: /_index_template/{template} + capabilities: [ 'failure_store_in_template' ] - do: allowed_warnings: @@ -61,12 +64,14 @@ name: generic_logs_template body: index_patterns: logs-* - data_stream: - failure_store: true + data_stream: {} template: settings: number_of_shards: 1 number_of_replicas: 1 + data_stream_options: + failure_store: + enabled: true - do: index: diff --git a/modules/dot-prefix-validation/build.gradle b/modules/dot-prefix-validation/build.gradle new file mode 100644 index 0000000000000..037a962846a7f --- /dev/null +++ b/modules/dot-prefix-validation/build.gradle @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' +apply plugin: 'elasticsearch.internal-cluster-test' + +esplugin { + description = 'Validation for dot-prefixed indices for non-operator users' + classname = 'org.elasticsearch.validation.DotPrefixValidationPlugin' +} + +restResources { + restApi { + include '_common', 'indices', 'index', 'cluster', 'nodes', 'get', 'ingest', 'bulk', 'reindex' + } +} + +tasks.named("yamlRestTestV7CompatTest").configure { enabled = false }; + +tasks.named('yamlRestTest') { + usesDefaultDistribution() +} diff --git a/modules/dot-prefix-validation/src/main/java/module-info.java b/modules/dot-prefix-validation/src/main/java/module-info.java new file mode 100644 index 0000000000000..4d0c92f255e7d --- /dev/null +++ b/modules/dot-prefix-validation/src/main/java/module-info.java @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +module org.elasticsearch.validation { + requires org.elasticsearch.server; + requires org.elasticsearch.base; +} diff --git a/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/AutoCreateDotValidator.java b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/AutoCreateDotValidator.java new file mode 100644 index 0000000000000..ec3c22620ca46 --- /dev/null +++ b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/AutoCreateDotValidator.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.validation; + +import org.elasticsearch.action.admin.indices.create.AutoCreateAction; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.ThreadContext; + +import java.util.Set; + +public class AutoCreateDotValidator extends DotPrefixValidator { + public AutoCreateDotValidator(ThreadContext threadContext, ClusterService clusterService) { + super(threadContext, clusterService); + } + + @Override + protected Set getIndicesFromRequest(CreateIndexRequest request) { + return Set.of(request.index()); + } + + @Override + public String actionName() { + return AutoCreateAction.NAME; + } +} diff --git a/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/CreateIndexDotValidator.java b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/CreateIndexDotValidator.java new file mode 100644 index 0000000000000..f39a1d09fa07c --- /dev/null +++ b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/CreateIndexDotValidator.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.validation; + +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.ThreadContext; + +import java.util.Set; + +public class CreateIndexDotValidator extends DotPrefixValidator { + public CreateIndexDotValidator(ThreadContext threadContext, ClusterService clusterService) { + super(threadContext, clusterService); + } + + @Override + protected Set getIndicesFromRequest(CreateIndexRequest request) { + return Set.of(request.index()); + } + + @Override + public String actionName() { + return TransportCreateIndexAction.TYPE.name(); + } +} diff --git a/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidationPlugin.java b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidationPlugin.java new file mode 100644 index 0000000000000..c462dbdcf6c40 --- /dev/null +++ b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidationPlugin.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.validation; + +import org.elasticsearch.action.support.MappedActionFilter; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.Plugin; + +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +public class DotPrefixValidationPlugin extends Plugin implements ActionPlugin { + private final AtomicReference> actionFilters = new AtomicReference<>(); + + public DotPrefixValidationPlugin() {} + + @Override + public Collection createComponents(PluginServices services) { + ThreadContext context = services.threadPool().getThreadContext(); + ClusterService clusterService = services.clusterService(); + + actionFilters.set( + List.of( + new CreateIndexDotValidator(context, clusterService), + new AutoCreateDotValidator(context, clusterService), + new IndexTemplateDotValidator(context, clusterService) + ) + ); + + return Set.of(); + } + + @Override + public Collection getMappedActionFilters() { + return actionFilters.get(); + } + + @Override + public List> getSettings() { + return List.of(DotPrefixValidator.VALIDATE_DOT_PREFIXES, DotPrefixValidator.IGNORED_INDEX_PATTERNS_SETTING); + } +} diff --git a/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidator.java b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidator.java new file mode 100644 index 0000000000000..4f5d4cf15d7f1 --- /dev/null +++ b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidator.java @@ -0,0 +1,189 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.validation; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.ActionFilterChain; +import org.elasticsearch.action.support.MappedActionFilter; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.logging.DeprecationCategory; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.tasks.Task; + +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; +import java.util.stream.Collectors; + +/** + * DotPrefixValidator provides an abstract class implementing a mapped action filter. + * + * This class then implements the {@link #apply(Task, String, ActionRequest, ActionListener, ActionFilterChain)} + * method which checks for indices in the request that begin with a dot, emitting a deprecation + * warning if they do. If the request is performed by a non-external user (operator, internal product, etc.) + * as defined by {@link #isInternalRequest()} then the deprecation is emitted. Otherwise, it is skipped. + * + * The indices for consideration are returned by the abstract {@link #getIndicesFromRequest(Object)} + * method, which subclasses must implement. + * + * Some built-in index names and patterns are also elided from the check, as defined in + * {@link #IGNORED_INDEX_NAMES} and {@link #IGNORED_INDEX_PATTERNS_SETTING}. + */ +public abstract class DotPrefixValidator implements MappedActionFilter { + public static final Setting VALIDATE_DOT_PREFIXES = Setting.boolSetting( + "cluster.indices.validate_dot_prefixes", + true, + Setting.Property.NodeScope + ); + + /** + * Names and patterns for indexes where no deprecation should be emitted. + * Normally we would want to transition these to either system indices, or + * to use an internal origin for the client. These are shorter-term + * workarounds until that work can be completed. + * + * .elastic-connectors-* is used by enterprise search + * .ml-* is used by ML + * .slo-observability-* is used by Observability + */ + private static Set IGNORED_INDEX_NAMES = Set.of( + ".elastic-connectors-v1", + ".elastic-connectors-sync-jobs-v1", + ".ml-state", + ".ml-anomalies-unrelated" + ); + public static Setting> IGNORED_INDEX_PATTERNS_SETTING = Setting.listSetting( + "cluster.indices.validate_ignored_dot_patterns", + List.of( + "\\.ml-state-\\d+", + "\\.slo-observability\\.sli-v\\d+.*", + "\\.slo-observability\\.summary-v\\d+.*", + "\\.entities\\.v\\d+\\.latest\\..*" + ), + Function.identity(), + (patternList) -> patternList.forEach(pattern -> { + try { + Pattern.compile(pattern); + } catch (PatternSyntaxException e) { + throw new IllegalArgumentException("invalid dot validation exception pattern: [" + pattern + "]", e); + } + }), + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(DotPrefixValidator.class); + + private final ThreadContext threadContext; + private final boolean isEnabled; + private volatile Set ignoredIndexPatterns; + + public DotPrefixValidator(ThreadContext threadContext, ClusterService clusterService) { + this.threadContext = threadContext; + this.isEnabled = VALIDATE_DOT_PREFIXES.get(clusterService.getSettings()); + this.ignoredIndexPatterns = IGNORED_INDEX_PATTERNS_SETTING.get(clusterService.getSettings()) + .stream() + .map(Pattern::compile) + .collect(Collectors.toSet()); + clusterService.getClusterSettings().addSettingsUpdateConsumer(IGNORED_INDEX_PATTERNS_SETTING, this::updateIgnoredIndexPatterns); + } + + private void updateIgnoredIndexPatterns(List patterns) { + this.ignoredIndexPatterns = patterns.stream().map(Pattern::compile).collect(Collectors.toSet()); + } + + protected abstract Set getIndicesFromRequest(RequestType request); + + @SuppressWarnings("unchecked") + @Override + public void apply( + Task task, + String action, + Request request, + ActionListener listener, + ActionFilterChain chain + ) { + Set indices = getIndicesFromRequest((RequestType) request); + if (isEnabled) { + validateIndices(indices); + } + chain.proceed(task, action, request, listener); + } + + void validateIndices(@Nullable Set indices) { + if (indices != null && isInternalRequest() == false) { + for (String index : indices) { + if (Strings.hasLength(index)) { + char c = getFirstChar(index); + if (c == '.') { + final String strippedName = stripDateMath(index); + if (IGNORED_INDEX_NAMES.contains(strippedName)) { + return; + } + if (this.ignoredIndexPatterns.stream().anyMatch(p -> p.matcher(strippedName).matches())) { + return; + } + deprecationLogger.warn( + DeprecationCategory.INDICES, + "dot-prefix", + "Index [{}] name begins with a dot (.), which is deprecated, " + + "and will not be allowed in a future Elasticsearch version.", + index + ); + } + } + } + } + } + + private static char getFirstChar(String index) { + char c = index.charAt(0); + if (c == '<') { + // Date-math is being used for the index, we need to + // consider it by stripping the first '<' before we + // check for a dot-prefix + String strippedLeading = index.substring(1); + if (Strings.hasLength(strippedLeading)) { + c = strippedLeading.charAt(0); + } + } + return c; + } + + private static String stripDateMath(String index) { + char c = index.charAt(0); + if (c == '<') { + assert index.charAt(index.length() - 1) == '>' + : "expected index name with date math to start with < and end with >, how did this pass request validation? " + index; + return index.substring(1, index.length() - 1); + } else { + return index; + } + } + + boolean isInternalRequest() { + final String actionOrigin = threadContext.getTransient(ThreadContext.ACTION_ORIGIN_TRANSIENT_NAME); + final boolean isSystemContext = threadContext.isSystemContext(); + final boolean isInternalOrigin = Optional.ofNullable(actionOrigin).map(Strings::hasText).orElse(false); + final boolean hasElasticOriginHeader = Optional.ofNullable(threadContext.getHeader(Task.X_ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER)) + .map(Strings::hasText) + .orElse(false); + return isSystemContext || isInternalOrigin || hasElasticOriginHeader; + } +} diff --git a/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/IndexTemplateDotValidator.java b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/IndexTemplateDotValidator.java new file mode 100644 index 0000000000000..dd9b0feeab388 --- /dev/null +++ b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/IndexTemplateDotValidator.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.validation; + +import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.ThreadContext; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +public class IndexTemplateDotValidator extends DotPrefixValidator { + public IndexTemplateDotValidator(ThreadContext threadContext, ClusterService clusterService) { + super(threadContext, clusterService); + } + + @Override + protected Set getIndicesFromRequest(TransportPutComposableIndexTemplateAction.Request request) { + return new HashSet<>(Arrays.asList(request.indices())); + } + + @Override + public String actionName() { + return TransportPutComposableIndexTemplateAction.TYPE.name(); + } +} diff --git a/modules/dot-prefix-validation/src/test/java/org/elasticsearch/validation/DotPrefixValidatorTests.java b/modules/dot-prefix-validation/src/test/java/org/elasticsearch/validation/DotPrefixValidatorTests.java new file mode 100644 index 0000000000000..6953f1cfc63df --- /dev/null +++ b/modules/dot-prefix-validation/src/test/java/org/elasticsearch/validation/DotPrefixValidatorTests.java @@ -0,0 +1,125 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.validation; + +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.BeforeClass; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class DotPrefixValidatorTests extends ESTestCase { + private final OperatorValidator opV = new OperatorValidator<>(); + private final NonOperatorValidator nonOpV = new NonOperatorValidator<>(); + + private static ClusterService clusterService; + + @BeforeClass + public static void beforeClass() { + List allowed = new ArrayList<>(DotPrefixValidator.IGNORED_INDEX_PATTERNS_SETTING.getDefault(Settings.EMPTY)); + // Add a new allowed pattern for testing + allowed.add("\\.potato\\d+"); + Settings settings = Settings.builder() + .put(DotPrefixValidator.IGNORED_INDEX_PATTERNS_SETTING.getKey(), Strings.collectionToCommaDelimitedString(allowed)) + .build(); + clusterService = mock(ClusterService.class); + ClusterSettings clusterSettings = new ClusterSettings( + settings, + Sets.newHashSet(DotPrefixValidator.VALIDATE_DOT_PREFIXES, DotPrefixValidator.IGNORED_INDEX_PATTERNS_SETTING) + ); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + when(clusterService.getSettings()).thenReturn(settings); + when(clusterService.threadPool()).thenReturn(mock(ThreadPool.class)); + } + + public void testValidation() { + + nonOpV.validateIndices(Set.of("regular")); + opV.validateIndices(Set.of("regular")); + assertFails(Set.of(".regular")); + opV.validateIndices(Set.of(".regular")); + assertFails(Set.of("first", ".second")); + assertFails(Set.of("<.regular-{MM-yy-dd}>")); + + // Test ignored names + nonOpV.validateIndices(Set.of(".elastic-connectors-v1")); + nonOpV.validateIndices(Set.of(".elastic-connectors-sync-jobs-v1")); + nonOpV.validateIndices(Set.of(".ml-state")); + nonOpV.validateIndices(Set.of(".ml-anomalies-unrelated")); + + // Test ignored patterns + nonOpV.validateIndices(Set.of(".ml-state-21309")); + nonOpV.validateIndices(Set.of(">.ml-state-21309>")); + nonOpV.validateIndices(Set.of(".slo-observability.sli-v2")); + nonOpV.validateIndices(Set.of(".slo-observability.sli-v2.3")); + nonOpV.validateIndices(Set.of(".slo-observability.sli-v2.3-2024-01-01")); + nonOpV.validateIndices(Set.of("<.slo-observability.sli-v3.3.{2024-10-16||/M{yyyy-MM-dd|UTC}}>")); + nonOpV.validateIndices(Set.of(".slo-observability.summary-v2")); + nonOpV.validateIndices(Set.of(".slo-observability.summary-v2.3")); + nonOpV.validateIndices(Set.of(".slo-observability.summary-v2.3-2024-01-01")); + nonOpV.validateIndices(Set.of("<.slo-observability.summary-v3.3.{2024-10-16||/M{yyyy-MM-dd|UTC}}>")); + nonOpV.validateIndices(Set.of(".entities.v1.latest.builtin_services_from_ecs_data")); + nonOpV.validateIndices(Set.of(".entities.v92.latest.eggplant.potato")); + nonOpV.validateIndices(Set.of("<.entities.v12.latest.eggplant-{M{yyyy-MM-dd|UTC}}>")); + + // Test pattern added to the settings + nonOpV.validateIndices(Set.of(".potato5")); + nonOpV.validateIndices(Set.of("<.potato5>")); + } + + private void assertFails(Set indices) { + nonOpV.validateIndices(indices); + assertWarnings( + "Index [" + + indices.stream().filter(i -> i.startsWith(".") || i.startsWith("<.")).toList().get(0) + + "] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version." + ); + } + + private static class NonOperatorValidator extends DotPrefixValidator { + + private NonOperatorValidator() { + super(new ThreadContext(Settings.EMPTY), clusterService); + } + + @Override + protected Set getIndicesFromRequest(Object request) { + return Set.of(); + } + + @Override + public String actionName() { + return ""; + } + + @Override + boolean isInternalRequest() { + return false; + } + } + + private static class OperatorValidator extends NonOperatorValidator { + @Override + boolean isInternalRequest() { + return true; + } + } +} diff --git a/modules/dot-prefix-validation/src/yamlRestTest/java/org/elasticsearch/validation/DotPrefixClientYamlTestSuiteIT.java b/modules/dot-prefix-validation/src/yamlRestTest/java/org/elasticsearch/validation/DotPrefixClientYamlTestSuiteIT.java new file mode 100644 index 0000000000000..9ecf6b9ba7d83 --- /dev/null +++ b/modules/dot-prefix-validation/src/yamlRestTest/java/org/elasticsearch/validation/DotPrefixClientYamlTestSuiteIT.java @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.validation; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.LocalClusterSpecBuilder; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; + +import static org.elasticsearch.test.cluster.FeatureFlag.FAILURE_STORE_ENABLED; + +public class DotPrefixClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + public DotPrefixClientYamlTestSuiteIT(final ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return createParameters(); + } + + private static final String BASIC_AUTH_VALUE = basicAuthHeaderValue("x_pack_rest_user", new SecureString("x-pack-test-password")); + + @Override + protected Settings restClientSettings() { + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE).build(); + } + + @ClassRule + public static ElasticsearchCluster cluster = createCluster(); + + private static ElasticsearchCluster createCluster() { + LocalClusterSpecBuilder clusterBuilder = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .feature(FAILURE_STORE_ENABLED) + .setting("xpack.security.enabled", "true") + .keystore("bootstrap.password", "x-pack-test-password") + .user("x_pack_rest_user", "x-pack-test-password"); + boolean setNodes = Boolean.parseBoolean(System.getProperty("yaml.rest.tests.set_num_nodes", "true")); + if (setNodes) { + clusterBuilder.nodes(2); + } + return clusterBuilder.build(); + } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + +} diff --git a/modules/dot-prefix-validation/src/yamlRestTest/resources/rest-api-spec/test/dot_prefix/10_basic.yml b/modules/dot-prefix-validation/src/yamlRestTest/resources/rest-api-spec/test/dot_prefix/10_basic.yml new file mode 100644 index 0000000000000..ae256daeb8abb --- /dev/null +++ b/modules/dot-prefix-validation/src/yamlRestTest/resources/rest-api-spec/test/dot_prefix/10_basic.yml @@ -0,0 +1,196 @@ +--- +teardown: + - do: + indices.delete: + index: .* + +--- +"Index creation with a dot-prefix is deprecated unless x-elastic-product-origin set": + - requires: + test_runner_features: ["warnings", "warnings_regex", "headers"] + + - do: + warnings: + - "Index [.myindex] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version." + indices.create: + index: .myindex + body: + settings: + index.hidden: true + + - do: + headers: { X-elastic-product-origin: kibana } + indices.create: + index: .myindex2 + body: + settings: + index.hidden: true + + - do: + warnings_regex: + - "Index \\[.*\\] name begins with a dot \\(\\.\\), which is deprecated, and will not be allowed in a future Elasticsearch version\\." + indices.create: + index: <.myindex-{now/d}> + body: + settings: + index.hidden: true + + - do: + headers: { X-elastic-product-origin: kibana } + indices.create: + index: <.myindex2-{now/d}> + body: + settings: + index.hidden: true + +--- +"Deprecated auto-creation of dot-prefixed indices": + - requires: + test_runner_features: ["warnings", "headers"] + + - do: + warnings: + - "Index [.myindex] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version." + - "index name [.myindex] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" + index: + index: .myindex + id: "1" + body: {foo: bar} + + - do: + headers: { X-elastic-product-origin: kibana } + warnings: + - "index name [.myindex2] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" + index: + index: .myindex2 + id: "1" + body: {foo: bar} + + - do: + warnings: + - "Index [.myindex3] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version." + - "index name [.myindex3] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" + bulk: + body: + - index: + _index: .myindex3 + - message: foo + +--- +"Deprecated auto-creation of dot-prefixed indices through pipelines": + - requires: + test_runner_features: ["warnings", "headers"] + + - do: + ingest.put_pipeline: + id: mypipeline + body: > + { + "processors": [ + { + "set" : { + "field" : "_index", + "value": "{{redirect_to}}" + } + } + ] + } + - do: + warnings: + - "Index [.other] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version." + - "index name [.other] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" + index: + index: myindex + id: "1" + body: {redirect_to: ".other"} + pipeline: mypipeline + + - do: + warnings: + - "Index [.other2] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version." + - "index name [.other2] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" + bulk: + body: + - index: + _index: myindex + pipeline: mypipeline + - redirect_to: .other2 + + - do: + index: + index: original + id: "1" + body: { "redirect_to": ".other3" } + - do: + indices.refresh: {} + - do: + warnings: + - "Index [.other3] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version." + - "index name [.other3] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" + reindex: + body: + source: + index: original + dest: + index: newindex + pipeline: mypipeline + + - do: + warnings: + - "Index [.reindex] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version." + - "index name [.reindex] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" + reindex: + body: + source: + index: original + dest: + index: .reindex + + - do: + headers: { X-elastic-product-origin: kibana } + warnings: + - "index name [.reindex2] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" + reindex: + body: + source: + index: original + dest: + index: .reindex2 + + - do: + ingest.delete_pipeline: + id: mypipeline + +--- +"Deprecated index template with a dot prefix index pattern": + - requires: + test_runner_features: ["warnings", "headers", "allowed_warnings"] + + - do: + warnings: + - "Index [.data-*] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version." + allowed_warnings: + - "index template [my-template] has index patterns [regular, .data-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" + indices.put_index_template: + name: my-template + body: + index_patterns: [regular, .data-*] + data_stream: {} + + - do: + headers: { X-elastic-product-origin: kibana } + allowed_warnings: + - "index template [my-template2] has index patterns [other, .data2-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation" + indices.put_index_template: + name: my-template2 + body: + index_patterns: [other, .data2-*] + data_stream: {} + + - do: + indices.delete_index_template: + name: my-template + + - do: + indices.delete_index_template: + name: my-template2 diff --git a/modules/health-shards-availability/build.gradle b/modules/health-shards-availability/build.gradle index 6c7cf5a19c8ac..1e2502601b2b7 100644 --- a/modules/health-shards-availability/build.gradle +++ b/modules/health-shards-availability/build.gradle @@ -1,17 +1,18 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.internal-yaml-rest-test' apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { - description 'Health report API extension providing the shards_availability output' - classname 'org.elasticsearch.health.plugin.ShardsAvailabilityPlugin' + description = 'Health report API extension providing the shards_availability output' + classname = 'org.elasticsearch.health.plugin.ShardsAvailabilityPlugin' } restResources { diff --git a/modules/health-shards-availability/src/main/java/module-info.java b/modules/health-shards-availability/src/main/java/module-info.java index 4ee4cafeb5f96..3f09c5d28df0f 100644 --- a/modules/health-shards-availability/src/main/java/module-info.java +++ b/modules/health-shards-availability/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.shardhealth { diff --git a/modules/health-shards-availability/src/main/java/org/elasticsearch/health/plugin/ShardsAvailabilityPlugin.java b/modules/health-shards-availability/src/main/java/org/elasticsearch/health/plugin/ShardsAvailabilityPlugin.java index eac72c36bef18..33a31d48b3d32 100644 --- a/modules/health-shards-availability/src/main/java/org/elasticsearch/health/plugin/ShardsAvailabilityPlugin.java +++ b/modules/health-shards-availability/src/main/java/org/elasticsearch/health/plugin/ShardsAvailabilityPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.health.plugin; diff --git a/modules/ingest-attachment/build.gradle b/modules/ingest-attachment/build.gradle index 89f0b530713c6..456c13725e7cd 100644 --- a/modules/ingest-attachment/build.gradle +++ b/modules/ingest-attachment/build.gradle @@ -2,17 +2,18 @@ import org.elasticsearch.gradle.internal.info.BuildParams /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.internal-yaml-rest-test' apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { - description 'Ingest processor that uses Apache Tika to extract contents' - classname 'org.elasticsearch.ingest.attachment.IngestAttachmentPlugin' + description = 'Ingest processor that uses Apache Tika to extract contents' + classname = 'org.elasticsearch.ingest.attachment.IngestAttachmentPlugin' } // this overwrites the 'versions' map from Elasticsearch itself, but indeed we want that -- we're interested in managing our dependencies @@ -150,7 +151,7 @@ tasks.named("thirdPartyAudit").configure { ignoreMissingClasses() } -if (BuildParams.inFipsJvm) { +if (buildParams.inFipsJvm) { tasks.named("test").configure { enabled = false } tasks.named("yamlRestTest").configure { enabled = false }; tasks.named("yamlRestTestV7CompatTest").configure { enabled = false }; diff --git a/modules/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java b/modules/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java index 48cb155ac2970..c35f516b4e1eb 100644 --- a/modules/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java +++ b/modules/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.attachment; @@ -195,7 +196,7 @@ public IngestDocument execute(IngestDocument ingestDocument) { * @param property property to add * @param value value to add */ - private void addAdditionalField(Map additionalFields, Property property, String value) { + private void addAdditionalField(Map additionalFields, Property property, String value) { if (properties.contains(property) && Strings.hasLength(value)) { additionalFields.put(property.toLowerCase(), value); } @@ -232,7 +233,7 @@ public AttachmentProcessor create( String processorTag, String description, Map config - ) throws Exception { + ) { String field = readStringProperty(TYPE, processorTag, config, "field"); String resourceName = readOptionalStringProperty(TYPE, processorTag, config, "resource_name"); String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "attachment"); diff --git a/modules/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/IngestAttachmentPlugin.java b/modules/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/IngestAttachmentPlugin.java index e829934285e2a..0229e13f97b53 100644 --- a/modules/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/IngestAttachmentPlugin.java +++ b/modules/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/IngestAttachmentPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.attachment; diff --git a/modules/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/TikaImpl.java b/modules/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/TikaImpl.java index c8d0e0f96f85b..02d85ef0ecfbf 100644 --- a/modules/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/TikaImpl.java +++ b/modules/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/TikaImpl.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.attachment; diff --git a/modules/ingest-attachment/src/main/plugin-metadata/plugin-security.policy b/modules/ingest-attachment/src/main/plugin-metadata/plugin-security.policy index e1a244e50e0ad..06074c01737ba 100644 --- a/modules/ingest-attachment/src/main/plugin-metadata/plugin-security.policy +++ b/modules/ingest-attachment/src/main/plugin-metadata/plugin-security.policy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ // NOTE: when modifying this file, look at restrictions in TikaImpl too diff --git a/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java b/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java index 0a18504e4505c..02d8b71afb856 100644 --- a/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java +++ b/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.attachment; diff --git a/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java b/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java index 20b0f19e9549e..4916f578903ce 100644 --- a/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java +++ b/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.attachment; diff --git a/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaDocTests.java b/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaDocTests.java index 93182941160a1..c17570f3df0da 100644 --- a/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaDocTests.java +++ b/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaDocTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.attachment; diff --git a/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaImplTests.java b/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaImplTests.java index 40ff0a8eaf095..baf51484245d4 100644 --- a/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaImplTests.java +++ b/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaImplTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.attachment; diff --git a/modules/ingest-attachment/src/yamlRestTest/java/org/elasticsearch/ingest/attachment/IngestAttachmentClientYamlTestSuiteIT.java b/modules/ingest-attachment/src/yamlRestTest/java/org/elasticsearch/ingest/attachment/IngestAttachmentClientYamlTestSuiteIT.java index fb3ebebfae769..6cf91c4b65db1 100644 --- a/modules/ingest-attachment/src/yamlRestTest/java/org/elasticsearch/ingest/attachment/IngestAttachmentClientYamlTestSuiteIT.java +++ b/modules/ingest-attachment/src/yamlRestTest/java/org/elasticsearch/ingest/attachment/IngestAttachmentClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.attachment; diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle index d7100745680ba..029f4698a3e0e 100644 --- a/modules/ingest-common/build.gradle +++ b/modules/ingest-common/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask @@ -12,15 +13,15 @@ apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { - description 'Module for ingest processors that do not require additional security permissions or have large dependencies and resources' - classname 'org.elasticsearch.ingest.common.IngestCommonPlugin' + description = 'Module for ingest processors that do not require additional security permissions or have large dependencies and resources' + classname ='org.elasticsearch.ingest.common.IngestCommonPlugin' extendedPlugins = ['lang-painless'] } dependencies { compileOnly project(':modules:lang-painless:spi') - api project(':libs:elasticsearch-grok') - api project(':libs:elasticsearch-dissect') + api project(':libs:grok') + api project(':libs:dissect') implementation "org.apache.httpcomponents:httpclient:${versions.httpclient}" implementation "org.apache.httpcomponents:httpcore:${versions.httpcore}" } @@ -50,6 +51,22 @@ tasks.named("thirdPartyAudit").configure { ) } +tasks.named("yamlRestTest").configure { + systemProperty 'tests.rest.blacklist', [ + // for some reason, allowed_warnings on the test isn't working here + 'ingest/30_date_processor/Test date processor with no timezone configured', + 'ingest/30_date_processor/Test week based date parsing', + ].join(',') +} + +tasks.named("yamlRestTestV7CompatTest").configure { + systemProperty 'tests.rest.blacklist', [ + // for some reason, allowed_warnings on the test isn't working here + 'ingest/30_date_processor/Test date processor with no timezone configured', + 'ingest/30_date_processor/Test week based date parsing', + ].join(',') +} + tasks.named("yamlRestTestV7CompatTransform").configure { task -> task.addAllowedWarningRegex("\\[types removal\\].*") } diff --git a/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/IngestRestartIT.java b/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/IngestRestartIT.java index f1c592e6e8345..6b43d7a1a3ada 100644 --- a/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/IngestRestartIT.java +++ b/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/IngestRestartIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/ManyNestedPipelinesIT.java b/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/ManyNestedPipelinesIT.java index c9f3f023b43ef..6ae283ef9efde 100644 --- a/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/ManyNestedPipelinesIT.java +++ b/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/ManyNestedPipelinesIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/plugins/internal/XContentMeteringParserDecoratorWithPipelinesIT.java b/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/plugins/internal/XContentMeteringParserDecoratorWithPipelinesIT.java deleted file mode 100644 index 7f0910ea5cc4d..0000000000000 --- a/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/plugins/internal/XContentMeteringParserDecoratorWithPipelinesIT.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.plugins.internal; - -import org.elasticsearch.action.DocWriteRequest; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.ingest.PutPipelineRequest; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.ParsedDocument; -import org.elasticsearch.ingest.common.IngestCommonPlugin; -import org.elasticsearch.plugins.IngestPlugin; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.xcontent.FilterXContentParserWrapper; -import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xcontent.XContentType; - -import java.io.IOException; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicLong; - -import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; -import static org.hamcrest.Matchers.equalTo; - -@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST) -public class XContentMeteringParserDecoratorWithPipelinesIT extends ESIntegTestCase { - - private static String TEST_INDEX_NAME = "test-index-name"; - // the assertions are done in plugin which is static and will be created by ES server. - // hence a static flag to make sure it is indeed used - public static volatile boolean hasWrappedParser; - public static AtomicLong providedFixedSize = new AtomicLong(); - - public void testDocumentIsReportedWithPipelines() throws Exception { - hasWrappedParser = false; - // pipeline adding fields, changing destination is not affecting reporting - final BytesReference pipelineBody = new BytesArray(""" - { - "processors": [ - { - "set": { - "field": "my-text-field", - "value": "xxxx" - } - }, - { - "set": { - "field": "my-boolean-field", - "value": true - } - } - ] - } - """); - clusterAdmin().putPipeline(new PutPipelineRequest("pipeline", pipelineBody, XContentType.JSON)).actionGet(); - - client().index( - new IndexRequest(TEST_INDEX_NAME).setPipeline("pipeline") - .id("1") - .source(jsonBuilder().startObject().field("test", "I am sam i am").endObject()) - ).actionGet(); - assertBusy(() -> { - // ingest node has used an observer that was counting #map operations - // and passed that info to newFixedSize observer in TransportShardBulkAction - assertTrue(hasWrappedParser); - assertThat(providedFixedSize.get(), equalTo(1L)); - }); - } - - @Override - protected Collection> nodePlugins() { - return List.of(TestDocumentParsingProviderPlugin.class, IngestCommonPlugin.class); - } - - public static class TestDocumentParsingProviderPlugin extends Plugin implements DocumentParsingProviderPlugin, IngestPlugin { - - public TestDocumentParsingProviderPlugin() {} - - @Override - public DocumentParsingProvider getDocumentParsingProvider() { - // returns a static instance, because we want to assert that the wrapping is called only once - return new DocumentParsingProvider() { - @Override - public XContentMeteringParserDecorator newMeteringParserDecorator(DocWriteRequest request) { - if (request instanceof IndexRequest indexRequest && indexRequest.getNormalisedBytesParsed() > 0) { - long normalisedBytesParsed = indexRequest.getNormalisedBytesParsed(); - providedFixedSize.set(normalisedBytesParsed); - return new TestXContentMeteringParserDecorator(normalisedBytesParsed); - } - return new TestXContentMeteringParserDecorator(0L); - } - - @Override - public DocumentSizeReporter newDocumentSizeReporter( - String indexName, - MapperService mapperService, - DocumentSizeAccumulator documentSizeAccumulator - ) { - return DocumentSizeReporter.EMPTY_INSTANCE; - } - }; - } - } - - public static class TestXContentMeteringParserDecorator implements XContentMeteringParserDecorator { - long mapCounter = 0; - - public TestXContentMeteringParserDecorator(long mapCounter) { - this.mapCounter = mapCounter; - } - - @Override - public XContentParser decorate(XContentParser xContentParser) { - hasWrappedParser = true; - return new FilterXContentParserWrapper(xContentParser) { - - @Override - public Map map() throws IOException { - mapCounter++; - return super.map(); - } - }; - } - - @Override - public ParsedDocument.DocumentSize meteredDocumentSize() { - return new ParsedDocument.DocumentSize(mapCounter, 0); - } - } - -} diff --git a/modules/ingest-common/src/main/java/module-info.java b/modules/ingest-common/src/main/java/module-info.java index 066f0094d973d..c3b3ab90892d9 100644 --- a/modules/ingest-common/src/main/java/module-info.java +++ b/modules/ingest-common/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.ingest.common { diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java index 6140d083f4eb6..695666a22a7c4 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java index 4f28c9bb14f80..0992983dd7d94 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/BytesProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/BytesProcessor.java index d63f1e60fa52d..892b73ebfc8dc 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/BytesProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/BytesProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CommunityIdProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CommunityIdProcessor.java index 0377da53846d5..ab158acb60fdf 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CommunityIdProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CommunityIdProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java index 3ba21de567a0c..5c72d1e444185 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CsvParser.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CsvParser.java index 5b41f8b406633..d4486d832a1b2 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CsvParser.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CsvParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CsvProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CsvProcessor.java index 36ed43ee98e49..d7abe08a7bfc5 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CsvProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CsvProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java index a7d3749836962..84d4569e5841f 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; @@ -96,6 +97,7 @@ Function getFunction(String format, ZoneId zoneId, Locale // fill the rest of the date up with the parsed date if (accessor.isSupported(ChronoField.YEAR) == false && accessor.isSupported(ChronoField.YEAR_OF_ERA) == false + && accessor.isSupported(WeekFields.ISO.weekBasedYear()) == false && accessor.isSupported(WeekFields.of(locale).weekBasedYear()) == false && accessor.isSupported(ChronoField.INSTANT_SECONDS) == false) { int year = LocalDate.now(ZoneOffset.UTC).getYear(); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java index 4aa76f63f76a1..a7da324899488 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java index ae3de1f1c446f..22db5a330fb45 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; @@ -97,7 +98,7 @@ private static ZoneId newDateTimeZone(String timezone) { } private static Locale newLocale(String locale) { - return locale == null ? Locale.ROOT : LocaleUtils.parse(locale); + return locale == null ? Locale.ENGLISH : LocaleUtils.parse(locale); } @Override diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java index 929f77192846a..d2d2f155dfa4d 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java index 5174ea614c0ee..93dc45bf67e6d 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java index 3d08c72f522b9..f09e1b3fc58c7 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessorException.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessorException.java index 6cfda53ab2e2f..6fad4cd25d628 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessorException.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessorException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FingerprintProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FingerprintProcessor.java index 701eab051dd1e..ed4d0b21a9119 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FingerprintProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FingerprintProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java index b86410d73c961..7831e743a712e 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java index f7d1c9f62305b..a7bcad63d3001 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java index 1e53ea5cda682..d93fb33302121 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GsubProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GsubProcessor.java index ea93a589f3c97..6fefbd9e2c083 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GsubProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GsubProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/HtmlStripProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/HtmlStripProcessor.java index b6d96d1b3f9af..07ea63fe92626 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/HtmlStripProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/HtmlStripProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java index 9bf3c17e1ee63..d585c6217202f 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; @@ -71,6 +72,7 @@ public Map getProcessors(Processor.Parameters paramet entry(SetProcessor.TYPE, new SetProcessor.Factory(parameters.scriptService)), entry(SortProcessor.TYPE, new SortProcessor.Factory()), entry(SplitProcessor.TYPE, new SplitProcessor.Factory()), + entry(TerminateProcessor.TYPE, new TerminateProcessor.Factory()), entry(TrimProcessor.TYPE, new TrimProcessor.Factory()), entry(URLDecodeProcessor.TYPE, new URLDecodeProcessor.Factory()), entry(UppercaseProcessor.TYPE, new UppercaseProcessor.Factory()), diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java index ca664206e575a..8c7ca42843842 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java index 3bf82be24330e..69c5699912f5d 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/KeyValueProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/KeyValueProcessor.java index a7b1efb52efc5..e51457adef3d6 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/KeyValueProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/KeyValueProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java index 6f71ef382a3aa..0115d28cab838 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java index 6d1809d871680..9551613b1e911 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Processors.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Processors.java index 7285983782e6e..44e3167e3c689 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Processors.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Processors.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ProcessorsWhitelistExtension.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ProcessorsWhitelistExtension.java index 9f736dc28ee2c..3c3e706cfc2b8 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ProcessorsWhitelistExtension.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ProcessorsWhitelistExtension.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java index acccdd92fc9b1..97e26a9961c20 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java index fa86bcda5047b..e92fd5767df38 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java index 85affa225b882..ce1783fa18a00 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RerouteProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RerouteProcessor.java index e44bb57f0b8e8..52e0aae1116cc 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RerouteProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RerouteProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java index b629f00545aec..a2798dbefd0d9 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java index 2d7db39f3738e..0ffef98580c4a 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java index 084a11d26a837..0614fa90b67f0 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java index 05fc3bc68c2fc..d46652f7a5a79 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TerminateProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TerminateProcessor.java new file mode 100644 index 0000000000000..5b6144ba8eab9 --- /dev/null +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TerminateProcessor.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.common; + +import org.elasticsearch.ingest.AbstractProcessor; +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.Processor; + +import java.util.Map; + +/** + * A {@link Processor} which simply prevents subsequent processors in the pipeline from running (without failing, like {@link FailProcessor} + * does). This will normally be run conditionally, using the {@code if} option. + */ +public class TerminateProcessor extends AbstractProcessor { + + static final String TYPE = "terminate"; + + TerminateProcessor(String tag, String description) { + super(tag, description); + } + + @Override + public IngestDocument execute(IngestDocument ingestDocument) { + ingestDocument.terminate(); + return ingestDocument; + } + + @Override + public String getType() { + return TYPE; + } + + public static final class Factory implements Processor.Factory { + + @Override + public Processor create( + Map processorFactories, + String tag, + String description, + Map config + ) { + return new TerminateProcessor(tag, description); + } + } +} diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java index 292b9098e7460..ceb277fb427a3 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/URLDecodeProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/URLDecodeProcessor.java index 24c32d123a244..2e0b3181b452e 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/URLDecodeProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/URLDecodeProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java index ec8a234b620b0..a84fb2cbe7411 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UriPartsProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UriPartsProcessor.java index c476c6a9d3b9d..40116ec99abbd 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UriPartsProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UriPartsProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/main/resources/org/elasticsearch/ingest/common/processors_whitelist.txt b/modules/ingest-common/src/main/resources/org/elasticsearch/ingest/common/processors_whitelist.txt index e1779aade5190..1c5bbb970282b 100644 --- a/modules/ingest-common/src/main/resources/org/elasticsearch/ingest/common/processors_whitelist.txt +++ b/modules/ingest-common/src/main/resources/org/elasticsearch/ingest/common/processors_whitelist.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # This file contains a whitelist of static processor methods that can be accessed from painless diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorFactoryTestCase.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorFactoryTestCase.java index 209375ac12eda..4bdeba65e4638 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorFactoryTestCase.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorFactoryTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorTestCase.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorTestCase.java index d052facbac74e..e4f697bcc4e5f 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorTestCase.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java index daccbc9559d48..790d9860de11e 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorTests.java index 3b5c2d39a3ca8..b7c4997e6dd6a 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorFactoryTests.java index 19ee0e4b1f434..dbb784162ce7d 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorTests.java index 8ca6e71b4a516..b88c5b0da2935 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorFactoryTests.java index 8d08040a40585..53fce7e2c0746 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorTests.java index 3848f4531adcb..dff9916093586 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java index df31f0ece6f6f..ad07526518173 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java index 28083aa0551ee..1d116c14c8985 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CsvProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CsvProcessorFactoryTests.java index 7053e079a87d7..22f69f88b7e1e 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CsvProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CsvProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CsvProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CsvProcessorTests.java index e111d5f18d210..8b2013b3a8a9b 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CsvProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CsvProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java index e564b4a4c0fd7..554e074bcc7e4 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; @@ -74,17 +75,17 @@ public void testParseJavaDefaultYear() { public void testParseWeekBasedYearAndWeek() { String format = "YYYY-ww"; ZoneId timezone = DateUtils.of("Europe/Amsterdam"); - Function javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.ROOT); + Function javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.ENGLISH); ZonedDateTime dateTime = javaFunction.apply("2020-33"); - assertThat(dateTime, equalTo(ZonedDateTime.of(2020, 8, 10, 0, 0, 0, 0, timezone))); + assertThat(dateTime, equalTo(ZonedDateTime.of(2020, 8, 9, 0, 0, 0, 0, timezone))); } public void testParseWeekBasedYear() { String format = "YYYY"; ZoneId timezone = DateUtils.of("Europe/Amsterdam"); - Function javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.ROOT); + Function javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.ENGLISH); ZonedDateTime dateTime = javaFunction.apply("2019"); - assertThat(dateTime, equalTo(ZonedDateTime.of(2018, 12, 31, 0, 0, 0, 0, timezone))); + assertThat(dateTime, equalTo(ZonedDateTime.of(2018, 12, 30, 0, 0, 0, 0, timezone))); } public void testParseWeekBasedWithLocale() { diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java index 1e247655c8751..1e558eaeb6fbb 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java index 0257bbcd46a73..1fa7b69ed0966 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java index 52aa66c37d6aa..76a219ec4458d 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java index 532bf575bda74..b9d9f84b15bb6 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorFactoryTests.java index b4bd9fea704be..8f5278ea17a5a 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorTests.java index 60f8a32e01bdf..6bfaf46a746f4 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorFactoryTests.java index a4f8a6b0a2b58..9b86111c3c40f 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java index 1cd0a0ead8785..89b680ec5d5d7 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java index ad73ef65c6c94..7df3cfb748dea 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorTests.java index db417dd68425c..56d10e6a8a18f 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorFactoryTests.java index f9f808a0fe84f..65d6eb96629dc 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorTests.java index 550bcc546b32b..a687bdd5cf41b 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java index 233d8fb2b860e..f5a38dab748f4 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java index fb580591ac8c7..04b60a6e25dd1 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java index 41154634b43e8..2f5dc9618d0c3 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java index 09801eeda6437..6f095a7a22062 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java index 0b20fbc22e1cc..71ffd12bf7518 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java index ce5a9e7987411..489e3860f510a 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorTests.java index 0f4964e6f2bd5..5572d81486822 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/HtmlStripProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/HtmlStripProcessorFactoryTests.java index 43e858fb6d2c0..001488b4d2f0b 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/HtmlStripProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/HtmlStripProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/HtmlStripProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/HtmlStripProcessorTests.java index 46e82c1a91d21..15bcd99ae58c1 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/HtmlStripProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/HtmlStripProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java index 8f768f9c01bb2..e1e5485173db2 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorTests.java index 5ee9ee383638b..516d262bf4563 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorFactoryTests.java index 02c936bb4b629..92474172a6612 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java index 068a93d438a12..42db55d14c2b7 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorFactoryTests.java index 5d1669a33b7bc..f4f39aa66d381 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorTests.java index af9d7d952f335..e8c554b3aea18 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorFactoryTests.java index bdaad9e1d152d..ade7df9092320 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorTests.java index 4c86f7ee8599b..8d327fb432622 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java index 59787e106307d..ed669a6687fa6 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java index 72e10e5ba3711..2f4e159dfcd0a 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorFactoryTests.java index 5dc20dd9e07b6..9928ed18bfbff 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorTests.java index 7d6fa99f81580..0a0666de9b014 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java index 49a9dd065184e..0eeb2f6af53f5 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java index 5bda0972401e6..b18a7ff26e30e 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java index 005cbc260f69a..d18ccace91424 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java index 9765320ef4d57..1ebc5f16a65d3 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RerouteProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RerouteProcessorFactoryTests.java index 580645a4ef46a..07e8ba405736b 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RerouteProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RerouteProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RerouteProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RerouteProcessorTests.java index bc51e1194d525..084be1c2b227d 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RerouteProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RerouteProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java index b3e4a870177bd..264ae5098203b 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java index 91dd0c861c02a..205d91d6de404 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java index 09c5c58e3664d..4d57967980cf1 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java index 6cef9d3ecde8a..9a5eb239fba18 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorFactoryTests.java index 0aeae9e6eed49..eaa720003579b 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorTests.java index f638f8c123152..d6806afe9c45e 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java index 10d96b8add96c..d44bad8b119d5 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java index 500debd79b966..8fc8dad61ae2b 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TerminateProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TerminateProcessorTests.java new file mode 100644 index 0000000000000..1888f8366edd3 --- /dev/null +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TerminateProcessorTests.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.common; + +import org.elasticsearch.ingest.CompoundProcessor; +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.Pipeline; +import org.elasticsearch.ingest.TestTemplateService; +import org.elasticsearch.ingest.ValueSource; +import org.elasticsearch.test.ESTestCase; + +import java.util.Map; + +import static org.elasticsearch.ingest.RandomDocumentPicks.randomIngestDocument; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; + +public class TerminateProcessorTests extends ESTestCase { + + public void testTerminateInPipeline() throws Exception { + Pipeline pipeline = new Pipeline( + "my-pipeline", + null, + null, + null, + new CompoundProcessor( + new SetProcessor( + "before-set", + "Sets before field to true", + new TestTemplateService.MockTemplateScript.Factory("before"), + ValueSource.wrap(true, TestTemplateService.instance()), + null + ), + new TerminateProcessor("terminate", "terminates the pipeline"), + new SetProcessor( + "after-set", + "Sets after field to true", + new TestTemplateService.MockTemplateScript.Factory("after"), + ValueSource.wrap(true, TestTemplateService.instance()), + null + ) + ) + ); + IngestDocument input = randomIngestDocument(random(), Map.of("foo", "bar")); + PipelineOutput output = new PipelineOutput(); + + pipeline.execute(input, output::set); + + assertThat(output.exception, nullValue()); + // We expect the before-set processor to have run, but not the after-set one: + assertThat(output.document.getSource(), is(Map.of("foo", "bar", "before", true))); + } + + private static class PipelineOutput { + IngestDocument document; + Exception exception; + + void set(IngestDocument document, Exception exception) { + this.document = document; + this.exception = exception; + } + } +} diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorFactoryTests.java index cb801d4c0a919..711b799bb0490 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorTests.java index 3e6c3b7cba821..2593196693cb7 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/URLDecodeProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/URLDecodeProcessorFactoryTests.java index c2384cffbe44a..f9e4e701be478 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/URLDecodeProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/URLDecodeProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/URLDecodeProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/URLDecodeProcessorTests.java index 6284e169f44b8..1c43c5c056e92 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/URLDecodeProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/URLDecodeProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorFactoryTests.java index 02af7116d86b0..b0998e1c6fe5e 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorTests.java index 16a295e2143f0..e107eb84a60ad 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorFactoryTests.java index 151aa1f5f387c..971a726869fc0 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorTests.java index e7552d23d659a..5441bfa581c23 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/yamlRestTest/java/org/elasticsearch/ingest/common/IngestCommonClientYamlTestSuiteIT.java b/modules/ingest-common/src/yamlRestTest/java/org/elasticsearch/ingest/common/IngestCommonClientYamlTestSuiteIT.java index 598d171bb328b..8150b7cdd176f 100644 --- a/modules/ingest-common/src/yamlRestTest/java/org/elasticsearch/ingest/common/IngestCommonClientYamlTestSuiteIT.java +++ b/modules/ingest-common/src/yamlRestTest/java/org/elasticsearch/ingest/common/IngestCommonClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.common; diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/30_date_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/30_date_processor.yml index 78fb895ea791c..a5caf7493340c 100644 --- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/30_date_processor.yml +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/30_date_processor.yml @@ -1,3 +1,6 @@ +setup: + - requires: + test_runner_features: allowed_warnings_regex --- teardown: - do: @@ -97,6 +100,8 @@ teardown: "Test date processor with no timezone configured": - do: + allowed_warnings_regex: + - 'Date format \[dd/MMM/yyyy:H:m:s Z] contains textual field specifiers that could change in JDK 23.*' ingest.put_pipeline: id: "my_pipeline" # sample formats from beats, featuring mongodb, icinga, apache @@ -165,6 +170,8 @@ teardown: - match: { acknowledged: true } - do: + allowed_warnings_regex: + - 'Date format \[dd/MMM/yyyy:H:m:s Z] contains textual field specifiers that could change in JDK 23.*' index: index: test id: "1" @@ -204,6 +211,8 @@ teardown: --- "Test week based date parsing": - do: + allowed_warnings_regex: + - 'Date format \[YYYY-ww] contains week-date field specifiers that are changing in JDK 23.*' indices.create: index: test body: @@ -214,6 +223,8 @@ teardown: format: YYYY-ww - do: + allowed_warnings_regex: + - 'Date format \[YYYY-ww] contains week-date field specifiers that are changing in JDK 23.*' ingest.put_pipeline: id: "my_pipeline" body: > @@ -263,69 +274,3 @@ teardown: id: "1" - match: { _source.date_source_field: "2020-33" } - match: { _source.date_target_field: "2020-08-10T00:00:00.000Z" } - ---- -"Test week based date parsing with locale": - #locale is used when parsing as well on a pipeline. As per US locale, start of the 33rd week 2020 is on 09August2020 (sunday) - - do: - indices.create: - index: test - body: - mappings: - properties: - date_source_field: - type: date - format: YYYY-ww - locale: en-US - - - do: - ingest.put_pipeline: - id: "my_pipeline" - body: > - { - "description": "_description", - "processors": [ - { - "date" : { - "field" : "date_source_field", - "target_field" : "date_target_field", - "formats" : ["YYYY-ww"], - "locale" : "en-US" - } - } - ] - } - - match: { acknowledged: true } - - - do: - ingest.simulate: - id: "my_pipeline" - body: > - { - "docs": [ - { - "_source": { - "date_source_field": "2020-33" - } - } - ] - } - - length: { docs: 1 } - - match: { docs.0.doc._source.date_source_field: "2020-33" } - - match: { docs.0.doc._source.date_target_field: "2020-08-09T00:00:00.000Z" } - - length: { docs.0.doc._ingest: 1 } - - is_true: docs.0.doc._ingest.timestamp - - - do: - index: - index: test - id: "1" - pipeline: "my_pipeline" - body: {date_source_field: "2020-33"} - - - do: - get: - index: test - id: "1" - - match: { _source.date_source_field: "2020-33" } - - match: { _source.date_target_field: "2020-08-09T00:00:00.000Z" } diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/310_reroute_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/310_reroute_processor.yml index 53229290da03e..5b7e6cff63b31 100644 --- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/310_reroute_processor.yml +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/310_reroute_processor.yml @@ -36,7 +36,10 @@ teardown: ingest.delete_pipeline: id: "pipeline-2" ignore: 404 - + - do: + indices.delete_data_stream: + name: "data-stream-*" + expand_wildcards: all --- "Test first matching router terminates pipeline": - skip: @@ -252,3 +255,322 @@ teardown: - match: { _source.existing-field : true } - match: { _source.added-in-pipeline-before-reroute : true } - match: { _source.added-in-pipeline-after-reroute : true } + +--- +"Test data stream with lazy rollover obtains pipeline from template": + # This test starts with chain of reroutes, from data-stream-1, to data-stream-2, to data-stream-3 + # We then add higher priority templates that remove the reroute processors. Then we show that + # after a lazy rollover on data-stream-2, a document written to data-stream-1 still gets rerouted + # to data-steam-2, but not on to data-stream-3. Finally, after a lazy rollover on data-stream-1 + # causes the new template to also take effect on data-stream-1, and the last write goes directly + # into data-stream-1. Multiple reroute steps are tested because pipeline resolution using a + # different code path for initial index and indices after a reroute. + + # start with pipeline that reroutes from ds-1 to ds-2 + - do: + ingest.put_pipeline: + id: "reroute-1" + body: > + { + "processors": [ + { + "reroute" : {"destination": "data-stream-2"} + } + ] + } + - match: { acknowledged: true } + + # and pipeline that reroutes from ds-2 to ds-3 + - do: + ingest.put_pipeline: + id: "reroute-2" + body: > + { + "processors": [ + { + "reroute" : {"destination": "data-stream-3"} + } + ] + } + - match: { acknowledged: true } + + # set pipelines in templates + - do: + indices.put_index_template: + name: template-1 + body: + index_patterns: [ "data-stream-1"] + priority: 1 + data_stream: { } + template: + settings: + index.default_pipeline: "reroute-1" + - match: { acknowledged: true } + - do: + indices.put_index_template: + name: template-2 + body: + index_patterns: [ "data-stream-2"] + priority: 1 + data_stream: { } + template: + settings: + index.default_pipeline: "reroute-2" + - match: { acknowledged: true } + - do: + indices.put_index_template: + name: template_3 + body: + index_patterns: [ "data-stream-3" ] + priority: 1 + data_stream: { } + - match: { acknowledged: true } + + - do: + indices.create_data_stream: + name: data-stream-1 + - match: { acknowledged: true } + - do: + indices.create_data_stream: + name: data-stream-2 + - match: { acknowledged: true } + - do: + indices.create_data_stream: + name: data-stream-3 + - match: { acknowledged: true } + + # write to ds-1 + - do: + index: + index: data-stream-1 + body: + '@timestamp': '2020-12-12' + some-field: 1 + - do: + indices.refresh: + index: data-stream-3 + + # document is rerouted to ds-3 + - do: + search: + index: data-stream-3 + body: { query: { match_all: { } } } + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.some-field: 1 } + + # add higher priority templates without reroute processors + - do: + indices.put_index_template: + name: template_4 + body: + index_patterns: [ "data-stream-1" ] + priority: 2 # higher priority + data_stream: { } + - match: { acknowledged: true } + - do: + indices.put_index_template: + name: template_5 + body: + index_patterns: [ "data-stream-2" ] + priority: 2 # higher priority + data_stream: { } + - match: { acknowledged: true } + + # write to ds-1 + - do: + index: + index: data-stream-1 + body: + '@timestamp': '2020-12-12' + some-field: 2 + - do: + indices.refresh: + index: data-stream-3 + + # still rerouted because ds-1 and ds-2 rolled over + - do: + search: + index: data-stream-3 + body: { query: { match_all: { } } } + - length: { hits.hits: 2 } + + # perform lazy rollover on ds-2 + - do: + indices.rollover: + alias: data-stream-2 + lazy: true + + # write to ds-1 + - do: + index: + index: data-stream-1 + body: + '@timestamp': '2020-12-12' + some-field: 3 + - do: + indices.refresh: + index: data-stream-2 + + # written to ds-2, as rerouted to ds-2, but not on to ds-3 + - do: + search: + index: data-stream-2 + body: { query: { match_all: { } } } + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.some-field: 3 } + + # perform lazy rollover on 1 + - do: + indices.rollover: + alias: data-stream-1 + lazy: true + + # write to ds-1 + - do: + index: + index: data-stream-1 + body: + '@timestamp': '2020-12-12' + some-field: 4 + - do: + indices.refresh: + index: data-stream-1 + + # written to ds-1, as not rerouted to ds-2 + - do: + search: + index: data-stream-1 + body: { query: { match_all: { } } } + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.some-field: 4 } + +--- +"Test remove then add reroute processor with and without lazy rollover": + # start with pipeline that reroutes from ds-1 to ds-2 + - do: + ingest.put_pipeline: + id: "reroute-1" + body: > + { + "processors": [ + { + "reroute" : {"destination": "data-stream-2"} + } + ] + } + - match: { acknowledged: true } + + # set pipelines in templates + - do: + indices.put_index_template: + name: template-1 + body: + index_patterns: [ "data-stream-1"] + priority: 1 + data_stream: { } + template: + settings: + index.default_pipeline: "reroute-1" + - match: { acknowledged: true } + - do: + indices.put_index_template: + name: template_2 + body: + index_patterns: [ "data-stream-2" ] + priority: 1 + data_stream: { } + - match: { acknowledged: true } + + - do: + indices.create_data_stream: + name: data-stream-1 + - match: { acknowledged: true } + + - do: + indices.create_data_stream: + name: data-stream-2 + - match: { acknowledged: true } + + # write to ds-1 + - do: + index: + index: data-stream-1 + body: + '@timestamp': '2020-12-12' + some-field: 1 + - do: + indices.refresh: + index: data-stream-2 + + # document is rerouted to ds-2 + - do: + search: + index: data-stream-2 + body: { query: { match_all: { } } } + - length: { hits.hits: 1 } + + # add higher priority templates without reroute processors + - do: + indices.put_index_template: + name: template_3 + body: + index_patterns: [ "data-stream-1" ] + priority: 2 # higher priority + data_stream: { } + - match: { acknowledged: true } + + # perform lazy rollover on ds-2 + - do: + indices.rollover: + alias: data-stream-1 + lazy: true + + # write to ds-1 + - do: + index: + index: data-stream-1 + body: + '@timestamp': '2020-12-12' + some-field: 2 + - do: + indices.refresh: + index: data-stream-1 + + # written to ds-1, as not rerouted to ds-2 + - do: + search: + index: data-stream-1 + body: { query: { match_all: { } } } + - length: { hits.hits: 1 } + + # add another higher priority templates with reroute processors + - do: + indices.put_index_template: + name: template-3 + body: + index_patterns: [ "data-stream-1" ] + priority: 3 + data_stream: { } + template: + settings: + index.default_pipeline: "reroute-1" + - match: { acknowledged: true } + + # don't do a lazy rollover + # write to ds-1 + - do: + index: + index: data-stream-1 + body: + '@timestamp': '2020-12-12' + some-field: 3 + - do: + indices.refresh: + index: data-stream-1 + + # because no lazy rollover, still no reroute processor + - do: + search: + index: data-stream-1 + body: { query: { match_all: { } } } + - length: { hits.hits: 2 } diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/330_terminate_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/330_terminate_processor.yml new file mode 100644 index 0000000000000..7a46d7bb272d8 --- /dev/null +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/330_terminate_processor.yml @@ -0,0 +1,138 @@ +--- +setup: + - do: + ingest.put_pipeline: + id: "test-pipeline" + body: > + { + "description": "Appends just 'before' to the steps field if the number field is less than 50, or both 'before' and 'after' if not", + "processors": [ + { + "append": { + "field": "steps", + "value": "before" + } + }, + { + "terminate": { + "if": "ctx.number < 50" + } + }, + { + "append": { + "field": "steps", + "value": "after" + } + } + ] + } + - do: + ingest.put_pipeline: + id: "test-final-pipeline" + body: > + { + "description": "Appends 'final' to the steps field", + "processors": [ + { + "append": { + "field": "steps", + "value": "final" + } + } + ] + } + - do: + ingest.put_pipeline: + id: "test-outer-pipeline" + body: > + { + "description": "Runs test-pipeline and then append 'outer' to the steps field", + "processors": [ + { + "pipeline": { + "name": "test-pipeline" + } + }, + { + "append": { + "field": "steps", + "value": "outer" + } + } + ] + } + - do: + indices.create: + index: "test-index-with-default-and-final-pipelines" + body: + settings: + index: + default_pipeline: "test-pipeline" + final_pipeline: "test-final-pipeline" + - do: + indices.create: + index: "test-vanilla-index" + +--- +teardown: + - do: + indices.delete: + index: "test-index-with-default-and-final-pipelines" + ignore_unavailable: true + - do: + indices.delete: + index: "test-vanilla-index" + ignore_unavailable: true + - do: + ingest.delete_pipeline: + id: "test-pipeline" + ignore: 404 + - do: + ingest.delete_pipeline: + id: "test-outer-pipeline" + ignore: 404 + +--- +"Test pipeline including conditional terminate pipeline": + + - do: + bulk: + refresh: true + body: + - '{ "index": {"_index": "test-index-with-default-and-final-pipelines" } }' + - '{ "comment": "should terminate", "number": 40, "steps": [] }' + - '{ "index": {"_index": "test-index-with-default-and-final-pipelines" } }' + - '{ "comment": "should continue to end", "number": 60, "steps": [] }' + + - do: + search: + rest_total_hits_as_int: true + index: "test-index-with-default-and-final-pipelines" + body: + sort: "number" + - match: { hits.total: 2 } + - match: { hits.hits.0._source.number: 40 } + - match: { hits.hits.1._source.number: 60 } + - match: { hits.hits.0._source.steps: ["before", "final"] } + - match: { hits.hits.1._source.steps: ["before", "after", "final"] } + +--- +"Test pipeline with terminate invoked from an outer pipeline": + + - do: + bulk: + refresh: true + pipeline: "test-outer-pipeline" + body: + - '{ "index": {"_index": "test-vanilla-index" } }' + - '{ "comment": "should terminate inner pipeline but not outer", "number": 40, "steps": [] }' + + - do: + search: + rest_total_hits_as_int: true + index: "test-vanilla-index" + body: + sort: "number" + - match: { hits.total: 1 } + - match: { hits.hits.0._source.number: 40 } + - match: { hits.hits.0._source.steps: ["before", "outer"] } diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle index bc5bb165cd0d2..0e3f597ce9767 100644 --- a/modules/ingest-geoip/build.gradle +++ b/modules/ingest-geoip/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.OS @@ -13,8 +14,8 @@ apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { - description 'Ingest processor that uses lookup geo data based on IP addresses using the MaxMind geo database' - classname 'org.elasticsearch.ingest.geoip.IngestGeoIpPlugin' + description = 'Ingest processor that uses lookup geo data based on IP addresses using the MaxMind geo database' + classname ='org.elasticsearch.ingest.geoip.IngestGeoIpPlugin' bundleSpec.from("${project.buildDir}/ingest-geoip") { into '/' @@ -92,3 +93,8 @@ tasks.named("yamlRestTestV7CompatTransform").configure { task -> artifacts { restTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test")) } + +tasks.named("yamlRestTestV7CompatTransform").configure({ task -> + task.skipTest("ingest_geoip/40_geoip_databases/Test adding, getting, and removing geoip databases", + "get databases behavior began returning more results in 8.16") +}) diff --git a/modules/ingest-geoip/qa/file-based-update/build.gradle b/modules/ingest-geoip/qa/file-based-update/build.gradle index 0e396d62eb16f..413a091726cdb 100644 --- a/modules/ingest-geoip/qa/file-based-update/build.gradle +++ b/modules/ingest-geoip/qa/file-based-update/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.legacy-java-rest-test' diff --git a/modules/ingest-geoip/qa/file-based-update/src/javaRestTest/java/org/elasticsearch/ingest/geoip/UpdateDatabasesIT.java b/modules/ingest-geoip/qa/file-based-update/src/javaRestTest/java/org/elasticsearch/ingest/geoip/UpdateDatabasesIT.java index 99f9e7af104c0..d75ce06b565ea 100644 --- a/modules/ingest-geoip/qa/file-based-update/src/javaRestTest/java/org/elasticsearch/ingest/geoip/UpdateDatabasesIT.java +++ b/modules/ingest-geoip/qa/file-based-update/src/javaRestTest/java/org/elasticsearch/ingest/geoip/UpdateDatabasesIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; diff --git a/modules/ingest-geoip/qa/full-cluster-restart/build.gradle b/modules/ingest-geoip/qa/full-cluster-restart/build.gradle index a97664923438b..c6f4e1667957d 100644 --- a/modules/ingest-geoip/qa/full-cluster-restart/build.gradle +++ b/modules/ingest-geoip/qa/full-cluster-restart/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.Version @@ -25,7 +26,7 @@ assert Version.fromString(VersionProperties.getVersions().get("elasticsearch")). // once we are ready to test migrations from 8.x to 9.x, we can set the compatible version to 8.0.0 // see https://github.com/elastic/elasticsearch/pull/93666 -BuildParams.bwcVersions.withWireCompatible(v -> v.before("7.0.0")) { bwcVersion, baseName -> +buildParams.bwcVersions.withWireCompatible(v -> v.before("7.0.0")) { bwcVersion, baseName -> tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) { usesBwcDistribution(bwcVersion) systemProperty("tests.old_cluster_version", bwcVersion) diff --git a/modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java b/modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java index 4f8abf4b82390..cf66a978c6b5f 100644 --- a/modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java +++ b/modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/AbstractGeoIpIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/AbstractGeoIpIT.java index 92ec911dbf451..84ada52f45306 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/AbstractGeoIpIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/AbstractGeoIpIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceIT.java new file mode 100644 index 0000000000000..136a9cd965e60 --- /dev/null +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceIT.java @@ -0,0 +1,235 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import com.maxmind.geoip2.model.CountryResponse; +import com.maxmind.geoip2.record.Country; + +import org.elasticsearch.action.admin.indices.flush.FlushRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.common.hash.MessageDigests; +import org.elasticsearch.xcontent.XContentType; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.zip.GZIPOutputStream; + +import static org.hamcrest.Matchers.equalTo; + +public class DatabaseNodeServiceIT extends AbstractGeoIpIT { + /* + * This test makes sure that if we index an ordinary mmdb file into the .geoip_databases index, it is correctly handled upon retrieval. + */ + public void testNonGzippedDatabase() throws Exception { + String databaseType = "GeoLite2-Country"; + String databaseFileName = databaseType + ".mmdb"; + // making the database name unique so we know we're not using another one: + String databaseName = randomAlphaOfLength(20) + "-" + databaseFileName; + byte[] mmdbBytes = getBytesForFile(databaseFileName); + final DatabaseNodeService databaseNodeService = internalCluster().getInstance(DatabaseNodeService.class); + assertNull(databaseNodeService.getDatabase(databaseName)); + int numChunks = indexData(databaseName, mmdbBytes); + /* + * If DatabaseNodeService::checkDatabases runs it will sometimes (rarely) remove the database we are using in this test while we + * are trying to assert things about it. So if it does then we 'just' try again. + */ + assertBusy(() -> { + retrieveDatabase(databaseNodeService, databaseName, mmdbBytes, numChunks); + assertNotNull(databaseNodeService.getDatabase(databaseName)); + assertValidDatabase(databaseNodeService, databaseName, databaseType); + }); + } + + /* + * This test makes sure that if we index a gzipped tar file wrapping an mmdb file into the .geoip_databases index, it is correctly + * handled upon retrieval. + */ + public void testGzippedDatabase() throws Exception { + String databaseType = "GeoLite2-Country"; + String databaseFileName = databaseType + ".mmdb"; + // making the database name unique so we know we're not using another one: + String databaseName = randomAlphaOfLength(20) + "-" + databaseFileName; + byte[] mmdbBytes = getBytesForFile(databaseFileName); + byte[] gzipBytes = gzipFileBytes(databaseName, mmdbBytes); + final DatabaseNodeService databaseNodeService = internalCluster().getInstance(DatabaseNodeService.class); + assertNull(databaseNodeService.getDatabase(databaseName)); + int numChunks = indexData(databaseName, gzipBytes); + /* + * If DatabaseNodeService::checkDatabases runs it will sometimes (rarely) remove the database we are using in this test while we + * are trying to assert things about it. So if it does then we 'just' try again. + */ + assertBusy(() -> { + retrieveDatabase(databaseNodeService, databaseName, gzipBytes, numChunks); + assertNotNull(databaseNodeService.getDatabase(databaseName)); + assertValidDatabase(databaseNodeService, databaseName, databaseType); + }); + } + + /* + * This makes sure that the database is generally usable + */ + private void assertValidDatabase(DatabaseNodeService databaseNodeService, String databaseFileName, String databaseType) + throws IOException { + IpDatabase database = databaseNodeService.getDatabase(databaseFileName); + assertNotNull(database); + assertThat(database.getDatabaseType(), equalTo(databaseType)); + CountryResponse countryResponse = database.getResponse("89.160.20.128", GeoIpTestUtils::getCountry); + assertNotNull(countryResponse); + Country country = countryResponse.getCountry(); + assertNotNull(country); + assertThat(country.getName(), equalTo("Sweden")); + } + + /* + * This has the databaseNodeService retrieve the database from the .geoip_databases index, making the database ready for use when + * databaseNodeService.getDatabase(databaseFileName) is called. + */ + private void retrieveDatabase(DatabaseNodeService databaseNodeService, String databaseFileName, byte[] expectedBytes, int numChunks) + throws IOException { + GeoIpTaskState.Metadata metadata = new GeoIpTaskState.Metadata(1, 0, numChunks - 1, getMd5(expectedBytes), 1); + databaseNodeService.retrieveAndUpdateDatabase(databaseFileName, metadata); + } + + private String getMd5(byte[] bytes) { + MessageDigest md = MessageDigests.md5(); + md.update(bytes); + return MessageDigests.toHexString(md.digest()); + } + + private byte[] gzipFileBytes(String databaseName, byte[] mmdbBytes) throws IOException { + final byte[] EMPTY_BUF = new byte[512]; + Path mmdbFile = createTempFile(); + Files.copy(new ByteArrayInputStream(mmdbBytes), mmdbFile, StandardCopyOption.REPLACE_EXISTING); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (OutputStream gos = new GZIPOutputStream(new BufferedOutputStream(baos))) { + long size = Files.size(mmdbFile); + gos.write(createTarHeader(databaseName, size)); + Files.copy(mmdbFile, gos); + if (size % 512 != 0) { + gos.write(EMPTY_BUF, 0, (int) (512 - (size % 512))); + } + gos.write(EMPTY_BUF); + gos.write(EMPTY_BUF); + } + return baos.toByteArray(); + } + + private static byte[] createTarHeader(String name, long size) { + byte[] buf = new byte[512]; + byte[] sizeBytes = String.format(Locale.ROOT, "%1$012o", size).getBytes(StandardCharsets.UTF_8); + byte[] nameBytes = name.substring(Math.max(0, name.length() - 100)).getBytes(StandardCharsets.US_ASCII); + byte[] id = "0001750".getBytes(StandardCharsets.UTF_8); + byte[] permission = "000644 ".getBytes(StandardCharsets.UTF_8); + byte[] time = String.format(Locale.ROOT, "%1$012o", System.currentTimeMillis() / 1000).getBytes(StandardCharsets.UTF_8); + System.arraycopy(nameBytes, 0, buf, 0, nameBytes.length); + System.arraycopy(permission, 0, buf, 100, 7); + System.arraycopy(id, 0, buf, 108, 7); + System.arraycopy(id, 0, buf, 116, 7); + System.arraycopy(sizeBytes, 0, buf, 124, 12); + System.arraycopy(time, 0, buf, 136, 12); + int checksum = 256; + for (byte b : buf) { + checksum += b & 0xFF; + } + byte[] checksumBytes = String.format(Locale.ROOT, "%1$07o", checksum).getBytes(StandardCharsets.UTF_8); + System.arraycopy(checksumBytes, 0, buf, 148, 7); + return buf; + } + + /* + * Finds the given databaseFileName on the classpath, and returns its bytes. + */ + private static byte[] getBytesForFile(String databaseFileName) throws IOException { + try (InputStream is = DatabaseNodeServiceIT.class.getResourceAsStream("/" + databaseFileName)) { + if (is == null) { + throw new FileNotFoundException("Resource [" + databaseFileName + "] not found in classpath"); + } + try (BufferedInputStream bis = new BufferedInputStream(is)) { + return bis.readAllBytes(); + } + } + } + + /* + * This indexes data into the .geoip_databases index in a random number of chunks. + */ + private static int indexData(String databaseFileName, byte[] content) throws IOException { + List chunks = chunkBytes(content, randomIntBetween(1, 100)); + indexChunks(databaseFileName, chunks); + return chunks.size(); + } + + /* + * This turns the given content bytes into the given number of chunks. + */ + private static List chunkBytes(byte[] content, int chunks) throws IOException { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + byteArrayOutputStream.write(content); + byteArrayOutputStream.close(); + + byte[] all = byteArrayOutputStream.toByteArray(); + int chunkSize = Math.max(1, all.length / chunks); + List data = new ArrayList<>(); + + for (int from = 0; from < all.length;) { + int to = from + chunkSize; + if (to > all.length) { + to = all.length; + } + data.add(Arrays.copyOfRange(all, from, to)); + from = to; + } + + while (data.size() > chunks) { + byte[] last = data.remove(data.size() - 1); + byte[] secondLast = data.remove(data.size() - 1); + byte[] merged = new byte[secondLast.length + last.length]; + System.arraycopy(secondLast, 0, merged, 0, secondLast.length); + System.arraycopy(last, 0, merged, secondLast.length, last.length); + data.add(merged); + } + return data; + } + + /* + * This writes the given chunks into the .geoip_databases index. + */ + private static void indexChunks(String name, List chunks) { + int chunk = 0; + for (byte[] buf : chunks) { + IndexRequest indexRequest = new IndexRequest(GeoIpDownloader.DATABASES_INDEX).id(name + "_" + chunk + "_" + 1) + .create(true) + .source(XContentType.SMILE, "name", name, "chunk", chunk, "data", buf); + client().index(indexRequest).actionGet(); + chunk++; + } + FlushRequest flushRequest = new FlushRequest(GeoIpDownloader.DATABASES_INDEX); + client().admin().indices().flush(flushRequest).actionGet(); + // Ensure that the chunk documents are visible: + RefreshRequest refreshRequest = new RefreshRequest(GeoIpDownloader.DATABASES_INDEX); + client().admin().indices().refresh(refreshRequest).actionGet(); + } +} diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java index cc757c413713d..3a1efabb38eca 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; @@ -22,6 +23,7 @@ import org.elasticsearch.action.ingest.PutPipelineRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Settings; @@ -43,19 +45,25 @@ import java.io.IOException; import java.util.Collection; +import java.util.List; import java.util.Map; import static org.elasticsearch.ingest.EnterpriseGeoIpTask.ENTERPRISE_GEOIP_DOWNLOADER; +import static org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloaderTaskExecutor.IPINFO_TOKEN_SETTING; import static org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloaderTaskExecutor.MAXMIND_LICENSE_KEY_SETTING; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; public class EnterpriseGeoIpDownloaderIT extends ESIntegTestCase { - private static final String DATABASE_TYPE = "GeoIP2-City"; + private static final String MAXMIND_DATABASE_TYPE = "GeoIP2-City"; + private static final String IPINFO_DATABASE_TYPE = "asn"; @ClassRule - public static final EnterpriseGeoIpHttpFixture fixture = new EnterpriseGeoIpHttpFixture(DATABASE_TYPE); + public static final EnterpriseGeoIpHttpFixture fixture = new EnterpriseGeoIpHttpFixture( + List.of(MAXMIND_DATABASE_TYPE), + List.of(IPINFO_DATABASE_TYPE) + ); protected String getEndpoint() { return fixture.getAddress(); @@ -65,6 +73,7 @@ protected String getEndpoint() { protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { MockSecureSettings secureSettings = new MockSecureSettings(); secureSettings.setString(MAXMIND_LICENSE_KEY_SETTING.getKey(), "license_key"); + secureSettings.setString(IPINFO_TOKEN_SETTING.getKey(), "token"); Settings.Builder builder = Settings.builder(); builder.setSecureSettings(secureSettings) .put(super.nodeSettings(nodeOrdinal, otherSettings)) @@ -91,22 +100,27 @@ public void testEnterpriseDownloaderTask() throws Exception { * Note that the "enterprise database" is actually just a geolite database being loaded by the GeoIpHttpFixture. */ EnterpriseGeoIpDownloader.DEFAULT_MAXMIND_ENDPOINT = getEndpoint(); - final String pipelineName = "enterprise_geoip_pipeline"; + EnterpriseGeoIpDownloader.DEFAULT_IPINFO_ENDPOINT = getEndpoint(); final String indexName = "enterprise_geoip_test_index"; + final String geoipPipelineName = "enterprise_geoip_pipeline"; + final String iplocationPipelineName = "enterprise_iplocation_pipeline"; final String sourceField = "ip"; - final String targetField = "ip-city"; + final String targetField = "ip-result"; startEnterpriseGeoIpDownloaderTask(); - configureDatabase(DATABASE_TYPE); - createGeoIpPipeline(pipelineName, DATABASE_TYPE, sourceField, targetField); + configureMaxmindDatabase(MAXMIND_DATABASE_TYPE); + configureIpinfoDatabase(IPINFO_DATABASE_TYPE); + waitAround(); + createPipeline(geoipPipelineName, "geoip", MAXMIND_DATABASE_TYPE, sourceField, targetField); + createPipeline(iplocationPipelineName, "ip_location", IPINFO_DATABASE_TYPE, sourceField, targetField); + /* + * We know that the databases index has been populated (because we waited around, :wink:), but we don't know for sure that + * the databases have been pulled down and made available on all nodes. So we run these ingest-and-check steps in assertBusy blocks. + */ assertBusy(() -> { - /* - * We know that the .geoip_databases index has been populated, but we don't know for sure that the database has been pulled - * down and made available on all nodes. So we run this ingest-and-check step in an assertBusy. - */ logger.info("Ingesting a test document"); - String documentId = ingestDocument(indexName, pipelineName, sourceField); + String documentId = ingestDocument(indexName, geoipPipelineName, sourceField, "89.160.20.128"); GetResponse getResponse = client().get(new GetRequest(indexName, documentId)).actionGet(); Map returnedSource = getResponse.getSource(); assertNotNull(returnedSource); @@ -114,6 +128,16 @@ public void testEnterpriseDownloaderTask() throws Exception { assertNotNull(targetFieldValue); assertThat(((Map) targetFieldValue).get("organization_name"), equalTo("Bredband2 AB")); }); + assertBusy(() -> { + logger.info("Ingesting another test document"); + String documentId = ingestDocument(indexName, iplocationPipelineName, sourceField, "12.10.66.1"); + GetResponse getResponse = client().get(new GetRequest(indexName, documentId)).actionGet(); + Map returnedSource = getResponse.getSource(); + assertNotNull(returnedSource); + Object targetFieldValue = returnedSource.get(targetField); + assertNotNull(targetFieldValue); + assertThat(((Map) targetFieldValue).get("organization_name"), equalTo("OAKLAWN JOCKEY CLUB, INC.")); + }); } private void startEnterpriseGeoIpDownloaderTask() { @@ -132,29 +156,46 @@ private void startEnterpriseGeoIpDownloaderTask() { ); } - private void configureDatabase(String databaseType) throws Exception { + private void configureMaxmindDatabase(String databaseType) { admin().cluster() .execute( PutDatabaseConfigurationAction.INSTANCE, new PutDatabaseConfigurationAction.Request( TimeValue.MAX_VALUE, TimeValue.MAX_VALUE, - new DatabaseConfiguration("test", databaseType, new DatabaseConfiguration.Maxmind("test_account")) + new DatabaseConfiguration("test-1", databaseType, new DatabaseConfiguration.Maxmind("test_account")) ) ) .actionGet(); + } + + private void configureIpinfoDatabase(String databaseType) { + admin().cluster() + .execute( + PutDatabaseConfigurationAction.INSTANCE, + new PutDatabaseConfigurationAction.Request( + TimeValue.MAX_VALUE, + TimeValue.MAX_VALUE, + new DatabaseConfiguration("test-2", databaseType, new DatabaseConfiguration.Ipinfo()) + ) + ) + .actionGet(); + } + + private void waitAround() throws Exception { ensureGreen(GeoIpDownloader.DATABASES_INDEX); assertBusy(() -> { SearchResponse searchResponse = client().search(new SearchRequest(GeoIpDownloader.DATABASES_INDEX)).actionGet(); try { - assertThat(searchResponse.getHits().getHits().length, equalTo(1)); + assertThat(searchResponse.getHits().getHits().length, equalTo(2)); } finally { searchResponse.decRef(); } }); } - private void createGeoIpPipeline(String pipelineName, String databaseType, String sourceField, String targetField) throws IOException { + private void createPipeline(String pipelineName, String processorType, String databaseType, String sourceField, String targetField) + throws IOException { final BytesReference bytes; try (XContentBuilder builder = JsonXContent.contentBuilder()) { builder.startObject(); @@ -164,7 +205,7 @@ private void createGeoIpPipeline(String pipelineName, String databaseType, Strin { builder.startObject(); { - builder.startObject("geoip"); + builder.startObject(processorType); { builder.field("field", sourceField); builder.field("target_field", targetField); @@ -182,11 +223,11 @@ private void createGeoIpPipeline(String pipelineName, String databaseType, Strin assertAcked(clusterAdmin().putPipeline(new PutPipelineRequest(pipelineName, bytes, XContentType.JSON)).actionGet()); } - private String ingestDocument(String indexName, String pipelineName, String sourceField) { + private String ingestDocument(String indexName, String pipelineName, String sourceField, String value) { BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add( - new IndexRequest(indexName).source("{\"" + sourceField + "\": \"89.160.20.128\"}", XContentType.JSON).setPipeline(pipelineName) - ); + bulkRequest.add(new IndexRequest(indexName).source(Strings.format(""" + { "%s": "%s"} + """, sourceField, value), XContentType.JSON).setPipeline(pipelineName)); BulkResponse response = client().bulk(bulkRequest).actionGet(); BulkItemResponse[] bulkItemResponses = response.getItems(); assertThat(bulkItemResponses.length, equalTo(1)); diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderCliIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderCliIT.java index 90506616a7d31..abe653d735233 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderCliIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderCliIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java index d994bd70eb7a0..0fa32e801c9cb 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java @@ -1,14 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; -import com.maxmind.geoip2.DatabaseReader; +import com.maxmind.db.Reader; import org.apache.lucene.search.TotalHits; import org.elasticsearch.action.ingest.SimulateDocumentBaseResult; @@ -751,8 +752,8 @@ private void deleteDatabasesInConfigDirectory() throws Exception { @SuppressForbidden(reason = "Maxmind API requires java.io.File") private void parseDatabase(Path tempFile) throws IOException { - try (DatabaseReader databaseReader = new DatabaseReader.Builder(tempFile.toFile()).build()) { - assertNotNull(databaseReader.getMetadata()); + try (Reader reader = new Reader(tempFile.toFile())) { + assertNotNull(reader.getMetadata()); } } diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderStatsIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderStatsIT.java index ec54317e144d1..a95297738d415 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderStatsIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderStatsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskIT.java index 473f1fb498e76..d3f36dc555beb 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpProcessorNonIngestNodeIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpProcessorNonIngestNodeIT.java index f34f647a01e05..650b835d70b60 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpProcessorNonIngestNodeIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpProcessorNonIngestNodeIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java index 87daefab7b428..0499b0f94106b 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; @@ -12,7 +13,6 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.core.IOUtils; import org.elasticsearch.index.VersionType; @@ -32,6 +32,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import static org.elasticsearch.ingest.geoip.GeoIpProcessor.GEOIP_TYPE; import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase; import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases; import static org.hamcrest.Matchers.equalTo; @@ -66,7 +67,7 @@ public void test() throws Exception { ClusterService clusterService = mock(ClusterService.class); when(clusterService.state()).thenReturn(ClusterState.EMPTY_STATE); DatabaseNodeService databaseNodeService = createRegistry(geoIpConfigDir, geoIpTmpDir, clusterService); - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(GEOIP_TYPE, databaseNodeService); copyDatabase("GeoLite2-City-Test.mmdb", geoIpTmpDir.resolve("GeoLite2-City.mmdb")); copyDatabase("GeoLite2-City-Test.mmdb", geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb")); databaseNodeService.updateDatabase("GeoLite2-City.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City.mmdb")); @@ -205,10 +206,10 @@ private static DatabaseNodeService createRegistry(Path geoIpConfigDir, Path geoI private static void lazyLoadReaders(DatabaseNodeService databaseNodeService) throws IOException { if (databaseNodeService.get("GeoLite2-City.mmdb") != null) { databaseNodeService.get("GeoLite2-City.mmdb").getDatabaseType(); - databaseNodeService.get("GeoLite2-City.mmdb").getCity(InetAddresses.forString("2.125.160.216")); + databaseNodeService.get("GeoLite2-City.mmdb").getResponse("2.125.160.216", GeoIpTestUtils::getCity); } databaseNodeService.get("GeoLite2-City-Test.mmdb").getDatabaseType(); - databaseNodeService.get("GeoLite2-City-Test.mmdb").getCity(InetAddresses.forString("2.125.160.216")); + databaseNodeService.get("GeoLite2-City-Test.mmdb").getResponse("2.125.160.216", GeoIpTestUtils::getCity); } } diff --git a/modules/ingest-geoip/src/main/java/module-info.java b/modules/ingest-geoip/src/main/java/module-info.java index 4d0acefcb6c9f..0703d9fb449aa 100644 --- a/modules/ingest-geoip/src/main/java/module-info.java +++ b/modules/ingest-geoip/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.ingest.geoip { @@ -17,4 +18,6 @@ exports org.elasticsearch.ingest.geoip.direct to org.elasticsearch.server; exports org.elasticsearch.ingest.geoip.stats to org.elasticsearch.server; + + exports org.elasticsearch.ingest.geoip to com.maxmind.db; } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/ConfigDatabases.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/ConfigDatabases.java index 8711e014b840a..3d2b54b04695f 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/ConfigDatabases.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/ConfigDatabases.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; @@ -72,17 +73,17 @@ void updateDatabase(Path file, boolean update) { String databaseFileName = file.getFileName().toString(); try { if (update) { - logger.info("database file changed [{}], reload database...", file); + logger.info("database file changed [{}], reloading database...", file); DatabaseReaderLazyLoader loader = new DatabaseReaderLazyLoader(cache, file, null); DatabaseReaderLazyLoader existing = configDatabases.put(databaseFileName, loader); if (existing != null) { - existing.close(); + existing.shutdown(); } } else { - logger.info("database file removed [{}], close database...", file); + logger.info("database file removed [{}], closing database...", file); DatabaseReaderLazyLoader existing = configDatabases.remove(databaseFileName); assert existing != null; - existing.close(); + existing.shutdown(); } } catch (Exception e) { logger.error(() -> "failed to update database [" + databaseFileName + "]", e); @@ -115,7 +116,7 @@ Map initConfigDatabases() throws IOException { @Override public void close() throws IOException { for (DatabaseReaderLazyLoader lazyLoader : configDatabases.values()) { - lazyLoader.close(); + lazyLoader.shutdown(); } } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java index aa8656dc14d91..fd88e2e71f0c9 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; @@ -17,18 +18,21 @@ import java.util.Set; /** - * A high-level representation of a kind of geoip database that is supported by the {@link GeoIpProcessor}. + * A high-level representation of a kind of ip location database that is supported by the {@link GeoIpProcessor}. *

    * A database has a set of properties that are valid to use with it (see {@link Database#properties()}), * as well as a list of default properties to use if no properties are specified (see {@link Database#defaultProperties()}). *

    - * See especially {@link Database#getDatabase(String, String)} which is used to obtain instances of this class. + * Some database providers have similar concepts but might have slightly different properties associated with those types. + * This can be accommodated, for example, by having a Foo value and a separate FooV2 value where the 'V' should be read as + * 'variant' or 'variation'. A V-less Database type is inherently the first variant/variation (i.e. V1). */ enum Database { City( Set.of( Property.IP, + Property.COUNTRY_IN_EUROPEAN_UNION, Property.COUNTRY_ISO_CODE, Property.CONTINENT_CODE, Property.COUNTRY_NAME, @@ -37,7 +41,12 @@ enum Database { Property.REGION_NAME, Property.CITY_NAME, Property.TIMEZONE, - Property.LOCATION + Property.LOCATION, + Property.POSTAL_CODE, + Property.ACCURACY_RADIUS, + Property.REGISTERED_COUNTRY_IN_EUROPEAN_UNION, + Property.REGISTERED_COUNTRY_ISO_CODE, + Property.REGISTERED_COUNTRY_NAME ), Set.of( Property.COUNTRY_ISO_CODE, @@ -50,7 +59,17 @@ enum Database { ) ), Country( - Set.of(Property.IP, Property.CONTINENT_CODE, Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE), + Set.of( + Property.IP, + Property.CONTINENT_CODE, + Property.CONTINENT_NAME, + Property.COUNTRY_NAME, + Property.COUNTRY_IN_EUROPEAN_UNION, + Property.COUNTRY_ISO_CODE, + Property.REGISTERED_COUNTRY_IN_EUROPEAN_UNION, + Property.REGISTERED_COUNTRY_ISO_CODE, + Property.REGISTERED_COUNTRY_NAME + ), Set.of(Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE) ), Asn( @@ -81,12 +100,15 @@ enum Database { Enterprise( Set.of( Property.IP, + Property.COUNTRY_CONFIDENCE, + Property.COUNTRY_IN_EUROPEAN_UNION, Property.COUNTRY_ISO_CODE, Property.COUNTRY_NAME, Property.CONTINENT_CODE, Property.CONTINENT_NAME, Property.REGION_ISO_CODE, Property.REGION_NAME, + Property.CITY_CONFIDENCE, Property.CITY_NAME, Property.TIMEZONE, Property.LOCATION, @@ -105,7 +127,13 @@ enum Database { Property.MOBILE_COUNTRY_CODE, Property.MOBILE_NETWORK_CODE, Property.USER_TYPE, - Property.CONNECTION_TYPE + Property.CONNECTION_TYPE, + Property.POSTAL_CODE, + Property.POSTAL_CONFIDENCE, + Property.ACCURACY_RADIUS, + Property.REGISTERED_COUNTRY_IN_EUROPEAN_UNION, + Property.REGISTERED_COUNTRY_ISO_CODE, + Property.REGISTERED_COUNTRY_NAME ), Set.of( Property.COUNTRY_ISO_CODE, @@ -138,55 +166,40 @@ enum Database { Property.MOBILE_COUNTRY_CODE, Property.MOBILE_NETWORK_CODE ) + ), + AsnV2( + Set.of( + Property.IP, + Property.ASN, + Property.ORGANIZATION_NAME, + Property.NETWORK, + Property.DOMAIN, + Property.COUNTRY_ISO_CODE, + Property.TYPE + ), + Set.of(Property.IP, Property.ASN, Property.ORGANIZATION_NAME, Property.NETWORK) + ), + CityV2( + Set.of( + Property.IP, + Property.COUNTRY_ISO_CODE, + Property.REGION_NAME, + Property.CITY_NAME, + Property.TIMEZONE, + Property.LOCATION, + Property.POSTAL_CODE + ), + Set.of(Property.COUNTRY_ISO_CODE, Property.REGION_NAME, Property.CITY_NAME, Property.LOCATION) + ), + CountryV2( + Set.of(Property.IP, Property.CONTINENT_CODE, Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE), + Set.of(Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE) + ), + PrivacyDetection( + Set.of(Property.IP, Property.HOSTING, Property.PROXY, Property.RELAY, Property.TOR, Property.VPN, Property.SERVICE), + Set.of(Property.HOSTING, Property.PROXY, Property.RELAY, Property.TOR, Property.VPN, Property.SERVICE) ); - private static final String CITY_DB_SUFFIX = "-City"; - private static final String COUNTRY_DB_SUFFIX = "-Country"; - private static final String ASN_DB_SUFFIX = "-ASN"; - private static final String ANONYMOUS_IP_DB_SUFFIX = "-Anonymous-IP"; - private static final String CONNECTION_TYPE_DB_SUFFIX = "-Connection-Type"; - private static final String DOMAIN_DB_SUFFIX = "-Domain"; - private static final String ENTERPRISE_DB_SUFFIX = "-Enterprise"; - private static final String ISP_DB_SUFFIX = "-ISP"; - - /** - * Parses the passed-in databaseType (presumably from the passed-in databaseFile) and return the Database instance that is - * associated with that databaseType. - * - * @param databaseType the database type String from the metadata of the database file - * @param databaseFile the database file from which the database type was obtained - * @throws IllegalArgumentException if the databaseType is not associated with a Database instance - * @return the Database instance that is associated with the databaseType - */ - public static Database getDatabase(final String databaseType, final String databaseFile) { - Database database = null; - if (databaseType != null) { - if (databaseType.endsWith(Database.CITY_DB_SUFFIX)) { - database = Database.City; - } else if (databaseType.endsWith(Database.COUNTRY_DB_SUFFIX)) { - database = Database.Country; - } else if (databaseType.endsWith(Database.ASN_DB_SUFFIX)) { - database = Database.Asn; - } else if (databaseType.endsWith(Database.ANONYMOUS_IP_DB_SUFFIX)) { - database = Database.AnonymousIp; - } else if (databaseType.endsWith(Database.CONNECTION_TYPE_DB_SUFFIX)) { - database = Database.ConnectionType; - } else if (databaseType.endsWith(Database.DOMAIN_DB_SUFFIX)) { - database = Database.Domain; - } else if (databaseType.endsWith(Database.ENTERPRISE_DB_SUFFIX)) { - database = Database.Enterprise; - } else if (databaseType.endsWith(Database.ISP_DB_SUFFIX)) { - database = Database.Isp; - } - } - - if (database == null) { - throw new IllegalArgumentException("Unsupported database type [" + databaseType + "] for file [" + databaseFile + "]"); - } - - return database; - } - private final Set properties; private final Set defaultProperties; @@ -235,12 +248,15 @@ public Set parseProperties(@Nullable final List propertyNames) enum Property { IP, + COUNTRY_CONFIDENCE, + COUNTRY_IN_EUROPEAN_UNION, COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_CODE, CONTINENT_NAME, REGION_ISO_CODE, REGION_NAME, + CITY_CONFIDENCE, CITY_NAME, TIMEZONE, LOCATION, @@ -259,7 +275,20 @@ enum Property { MOBILE_COUNTRY_CODE, MOBILE_NETWORK_CODE, CONNECTION_TYPE, - USER_TYPE; + USER_TYPE, + TYPE, + POSTAL_CODE, + POSTAL_CONFIDENCE, + ACCURACY_RADIUS, + HOSTING, + TOR, + PROXY, + RELAY, + VPN, + SERVICE, + REGISTERED_COUNTRY_IN_EUROPEAN_UNION, + REGISTERED_COUNTRY_ISO_CODE, + REGISTERED_COUNTRY_NAME; /** * Parses a string representation of a property into an actual Property instance. Not all properties that exist are diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java index dcb882ede230c..940231b12c894 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; @@ -19,11 +20,13 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.hash.MessageDigests; import org.elasticsearch.common.logging.HeaderWarning; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; import org.elasticsearch.env.Environment; import org.elasticsearch.gateway.GatewayService; @@ -35,8 +38,8 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.watcher.ResourceWatcherService; -import java.io.BufferedInputStream; import java.io.Closeable; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.UncheckedIOException; import java.nio.file.FileAlreadyExistsException; @@ -51,8 +54,10 @@ import java.security.MessageDigest; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -86,7 +91,7 @@ * if there is an old instance of this database then that is closed. * 4) Cleanup locally loaded databases that are no longer mentioned in {@link GeoIpTaskState}. */ -public final class DatabaseNodeService implements GeoIpDatabaseProvider, Closeable { +public final class DatabaseNodeService implements IpDatabaseProvider { private static final Logger logger = LogManager.getLogger(DatabaseNodeService.class); @@ -220,7 +225,7 @@ DatabaseReaderLazyLoader getDatabaseReaderLazyLoader(String name) { } @Override - public GeoIpDatabase getDatabase(String name) { + public IpDatabase getDatabase(String name) { return getDatabaseReaderLazyLoader(name); } @@ -235,9 +240,21 @@ DatabaseReaderLazyLoader get(String key) { return databases.get(key); } - @Override - public void close() throws IOException { - IOUtils.close(databases.values()); + public void shutdown() throws IOException { + // this is a little 'fun' looking, but it's just adapting IOUtils.close() into something + // that can call a bunch of shutdown methods (rather than close methods) + final var loadersToShutdown = databases.values().stream().map(ShutdownCloseable::new).toList(); + databases.clear(); + IOUtils.close(loadersToShutdown); + } + + private record ShutdownCloseable(DatabaseReaderLazyLoader loader) implements Closeable { + @Override + public void close() throws IOException { + if (loader != null) { + loader.shutdown(); + } + } } void checkDatabases(ClusterState state) { @@ -339,15 +356,15 @@ void checkDatabases(ClusterState state) { } void retrieveAndUpdateDatabase(String databaseName, GeoIpTaskState.Metadata metadata) throws IOException { - logger.trace("Retrieving database {}", databaseName); + logger.trace("retrieving database [{}]", databaseName); final String recordedMd5 = metadata.md5(); // This acts as a lock, if this method for a specific db is executed later and downloaded for this db is still ongoing then // FileAlreadyExistsException is thrown and this method silently returns. // (this method is never invoked concurrently and is invoked by a cluster state applier thread) - final Path databaseTmpGzFile; + final Path retrievedFile; try { - databaseTmpGzFile = Files.createFile(geoipTmpDirectory.resolve(databaseName + ".tmp.gz")); + retrievedFile = Files.createFile(geoipTmpDirectory.resolve(databaseName + ".tmp.retrieved")); } catch (FileAlreadyExistsException e) { logger.debug("database update [{}] already in progress, skipping...", databaseName); return; @@ -362,28 +379,21 @@ void retrieveAndUpdateDatabase(String databaseName, GeoIpTaskState.Metadata meta DatabaseReaderLazyLoader lazyLoader = databases.get(databaseName); if (lazyLoader != null && recordedMd5.equals(lazyLoader.getMd5())) { logger.debug("deleting tmp file because database [{}] has already been updated.", databaseName); - Files.delete(databaseTmpGzFile); + Files.delete(retrievedFile); return; } final Path databaseTmpFile = Files.createFile(geoipTmpDirectory.resolve(databaseName + ".tmp")); - logger.debug("retrieve geoip database [{}] from [{}] to [{}]", databaseName, GeoIpDownloader.DATABASES_INDEX, databaseTmpGzFile); - retrieveDatabase( - databaseName, - recordedMd5, - metadata, - bytes -> Files.write(databaseTmpGzFile, bytes, StandardOpenOption.APPEND), - () -> { - logger.debug("decompressing [{}]", databaseTmpGzFile.getFileName()); - - Path databaseFile = geoipTmpDirectory.resolve(databaseName); + logger.debug("retrieving database [{}] from [{}] to [{}]", databaseName, GeoIpDownloader.DATABASES_INDEX, retrievedFile); + retrieveDatabase(databaseName, recordedMd5, metadata, bytes -> Files.write(retrievedFile, bytes, StandardOpenOption.APPEND), () -> { + final Path databaseFile = geoipTmpDirectory.resolve(databaseName); + + boolean isTarGz = MMDBUtil.isGzip(retrievedFile); + if (isTarGz) { // tarball contains .mmdb, LICENSE.txt, COPYRIGHTS.txt and optional README.txt files. // we store mmdb file as is and prepend database name to all other entries to avoid conflicts - try ( - TarInputStream is = new TarInputStream( - new GZIPInputStream(new BufferedInputStream(Files.newInputStream(databaseTmpGzFile)), 8192) - ) - ) { + logger.debug("decompressing [{}]", retrievedFile.getFileName()); + try (TarInputStream is = new TarInputStream(new GZIPInputStream(Files.newInputStream(retrievedFile), 8192))) { TarInputStream.TarEntry entry; while ((entry = is.getNextEntry()) != null) { // there might be ./ entry in tar, we should skip it @@ -399,32 +409,37 @@ void retrieveAndUpdateDatabase(String databaseName, GeoIpTaskState.Metadata meta } } } - - logger.debug("moving database from [{}] to [{}]", databaseTmpFile, databaseFile); - Files.move(databaseTmpFile, databaseFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); - updateDatabase(databaseName, recordedMd5, databaseFile); - Files.delete(databaseTmpGzFile); - }, - failure -> { - logger.error(() -> "failed to retrieve database [" + databaseName + "]", failure); - try { - Files.deleteIfExists(databaseTmpFile); - Files.deleteIfExists(databaseTmpGzFile); - } catch (IOException ioe) { - ioe.addSuppressed(failure); - logger.error("Unable to delete tmp database file after failure", ioe); - } + } else { + /* + * Given that this is not code that will be called extremely frequently, we copy the file to the expected location here in + * order to avoid making the rest of the code more complex to avoid this. + */ + Files.copy(retrievedFile, databaseTmpFile, StandardCopyOption.REPLACE_EXISTING); } - ); + // finally, atomically move some-database.mmdb.tmp to some-database.mmdb + logger.debug("moving database from [{}] to [{}]", databaseTmpFile, databaseFile); + Files.move(databaseTmpFile, databaseFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + updateDatabase(databaseName, recordedMd5, databaseFile); + Files.delete(retrievedFile); + }, failure -> { + logger.error(() -> "failed to retrieve database [" + databaseName + "]", failure); + try { + Files.deleteIfExists(databaseTmpFile); + Files.deleteIfExists(retrievedFile); + } catch (IOException ioe) { + ioe.addSuppressed(failure); + logger.error("unable to delete tmp database file after failure", ioe); + } + }); } void updateDatabase(String databaseFileName, String recordedMd5, Path file) { try { - logger.debug("starting reload of changed geoip database file [{}]", file); + logger.debug("starting reload of changed database file [{}]", file); DatabaseReaderLazyLoader loader = new DatabaseReaderLazyLoader(cache, file, recordedMd5); DatabaseReaderLazyLoader existing = databases.put(databaseFileName, loader); if (existing != null) { - existing.close(); + existing.shutdown(); } else { // Loaded a database for the first time, so reload pipelines for which a database was not available: Predicate predicate = p -> databaseFileName.equals(p.getDatabaseName()); @@ -450,7 +465,7 @@ void updateDatabase(String databaseFileName, String recordedMd5, Path file) { logger.debug("no pipelines found to reload"); } } - logger.info("successfully loaded geoip database file [{}]", file.getFileName()); + logger.info("successfully loaded database file [{}]", file.getFileName()); } catch (Exception e) { logger.error(() -> "failed to update database [" + databaseFileName + "]", e); } @@ -462,7 +477,7 @@ void removeStaleEntries(Collection staleEntries) { logger.debug("database [{}] no longer exists, cleaning up...", staleEntry); DatabaseReaderLazyLoader existing = databases.remove(staleEntry); assert existing != null; - existing.close(true); + existing.shutdown(true); } catch (Exception e) { logger.error(() -> "failed to clean database [" + staleEntry + "]", e); } @@ -531,6 +546,35 @@ public Set getConfigDatabases() { return configDatabases.getConfigDatabases().keySet(); } + public Map getConfigDatabasesDetail() { + Map allDatabases = new HashMap<>(); + for (Map.Entry entry : configDatabases.getConfigDatabases().entrySet()) { + DatabaseReaderLazyLoader databaseReaderLazyLoader = entry.getValue(); + try { + allDatabases.put( + entry.getKey(), + new ConfigDatabaseDetail( + entry.getKey(), + databaseReaderLazyLoader.getMd5(), + databaseReaderLazyLoader.getBuildDateMillis(), + databaseReaderLazyLoader.getDatabaseType() + ) + ); + } catch (FileNotFoundException e) { + /* + * Since there is nothing to prevent a database from being deleted while this method is running, it is possible we get an + * exception here because the file no longer exists. We just log it and move on -- it's preferable to synchronization. + */ + logger.trace(Strings.format("Unable to get metadata for config database %s", entry.getKey()), e); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return allDatabases; + } + + public record ConfigDatabaseDetail(String name, @Nullable String md5, @Nullable Long buildDateInMillis, @Nullable String type) {} + public Set getFilesInTemp() { try (Stream files = Files.list(geoipTmpDirectory)) { return files.map(Path::getFileName).map(Path::toString).collect(Collectors.toSet()); diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java index 72873efd0d73f..120afe0e9e815 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java @@ -1,29 +1,21 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; import com.maxmind.db.NoCache; import com.maxmind.db.Reader; -import com.maxmind.geoip2.DatabaseReader; -import com.maxmind.geoip2.model.AbstractResponse; -import com.maxmind.geoip2.model.AnonymousIpResponse; -import com.maxmind.geoip2.model.AsnResponse; -import com.maxmind.geoip2.model.CityResponse; -import com.maxmind.geoip2.model.ConnectionTypeResponse; -import com.maxmind.geoip2.model.CountryResponse; -import com.maxmind.geoip2.model.DomainResponse; -import com.maxmind.geoip2.model.EnterpriseResponse; -import com.maxmind.geoip2.model.IspResponse; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.SetOnce; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.CheckedBiFunction; import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.core.Booleans; @@ -31,22 +23,18 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.SuppressForbidden; -import java.io.Closeable; +import java.io.File; import java.io.IOException; -import java.io.InputStream; -import java.net.InetAddress; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.Objects; -import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; /** * Facilitates lazy loading of the database reader, so that when the geoip plugin is installed, but not used, * no memory is being wasted on the database reader. */ -class DatabaseReaderLazyLoader implements GeoIpDatabase, Closeable { +public class DatabaseReaderLazyLoader implements IpDatabase { private static final boolean LOAD_DATABASE_ON_HEAP = Booleans.parseBoolean(System.getProperty("es.geoip.load_db_on_heap", "false")); @@ -55,33 +43,28 @@ class DatabaseReaderLazyLoader implements GeoIpDatabase, Closeable { private final String md5; private final GeoIpCache cache; private final Path databasePath; - private final CheckedSupplier loader; - final SetOnce databaseReader; + private final CheckedSupplier loader; + final SetOnce databaseReader; // cache the database type so that we do not re-read it on every pipeline execution final SetOnce databaseType; + final SetOnce buildDate; - private volatile boolean deleteDatabaseFileOnClose; + private volatile boolean deleteDatabaseFileOnShutdown; private final AtomicInteger currentUsages = new AtomicInteger(0); DatabaseReaderLazyLoader(GeoIpCache cache, Path databasePath, String md5) { - this(cache, databasePath, md5, createDatabaseLoader(databasePath)); - } - - DatabaseReaderLazyLoader(GeoIpCache cache, Path databasePath, String md5, CheckedSupplier loader) { this.cache = cache; this.databasePath = Objects.requireNonNull(databasePath); this.md5 = md5; - this.loader = Objects.requireNonNull(loader); + this.loader = createDatabaseLoader(databasePath); this.databaseReader = new SetOnce<>(); this.databaseType = new SetOnce<>(); + this.buildDate = new SetOnce<>(); } /** - * Read the database type from the database. We do this manually instead of relying on the built-in mechanism to avoid reading the - * entire database into memory merely to read the type. This is especially important to maintain on master nodes where pipelines are - * validated. If we read the entire database into memory, we could potentially run into low-memory constraints on such nodes where - * loading this data would otherwise be wasteful if they are not also ingest nodes. + * Read the database type from the database and cache it for future calls. * * @return the database type * @throws IOException if an I/O exception occurs reading the database type @@ -91,127 +74,21 @@ public final String getDatabaseType() throws IOException { if (databaseType.get() == null) { synchronized (databaseType) { if (databaseType.get() == null) { - final long fileSize = databaseFileSize(); - if (fileSize <= 512) { - throw new IOException("unexpected file length [" + fileSize + "] for [" + databasePath + "]"); - } - final int[] databaseTypeMarker = { 'd', 'a', 't', 'a', 'b', 'a', 's', 'e', '_', 't', 'y', 'p', 'e' }; - try (InputStream in = databaseInputStream()) { - // read the last 512 bytes - final long skipped = in.skip(fileSize - 512); - if (skipped != fileSize - 512) { - throw new IOException("failed to skip [" + (fileSize - 512) + "] bytes while reading [" + databasePath + "]"); - } - final byte[] tail = new byte[512]; - int read = 0; - do { - final int actualBytesRead = in.read(tail, read, 512 - read); - if (actualBytesRead == -1) { - throw new IOException("unexpected end of stream [" + databasePath + "] after reading [" + read + "] bytes"); - } - read += actualBytesRead; - } while (read != 512); - - // find the database_type header - int metadataOffset = -1; - int markerOffset = 0; - for (int i = 0; i < tail.length; i++) { - byte b = tail[i]; - - if (b == databaseTypeMarker[markerOffset]) { - markerOffset++; - } else { - markerOffset = 0; - } - if (markerOffset == databaseTypeMarker.length) { - metadataOffset = i + 1; - break; - } - } - - if (metadataOffset == -1) { - throw new IOException("database type marker not found"); - } - - // read the database type - final int offsetByte = tail[metadataOffset] & 0xFF; - final int type = offsetByte >>> 5; - if (type != 2) { - throw new IOException("type must be UTF-8 string"); - } - int size = offsetByte & 0x1f; - databaseType.set(new String(tail, metadataOffset + 1, size, StandardCharsets.UTF_8)); - } + databaseType.set(MMDBUtil.getDatabaseType(databasePath)); } } } return databaseType.get(); } - long databaseFileSize() throws IOException { - return Files.size(databasePath); - } - - InputStream databaseInputStream() throws IOException { - return Files.newInputStream(databasePath); - } - - @Nullable - @Override - public CityResponse getCity(InetAddress ipAddress) { - return getResponse(ipAddress, DatabaseReader::tryCity); - } - - @Nullable - @Override - public CountryResponse getCountry(InetAddress ipAddress) { - return getResponse(ipAddress, DatabaseReader::tryCountry); - } - - @Nullable - @Override - public AsnResponse getAsn(InetAddress ipAddress) { - return getResponse(ipAddress, DatabaseReader::tryAsn); - } - - @Nullable - @Override - public AnonymousIpResponse getAnonymousIp(InetAddress ipAddress) { - return getResponse(ipAddress, DatabaseReader::tryAnonymousIp); - } - - @Nullable - @Override - public ConnectionTypeResponse getConnectionType(InetAddress ipAddress) { - return getResponse(ipAddress, DatabaseReader::tryConnectionType); - } - - @Nullable - @Override - public DomainResponse getDomain(InetAddress ipAddress) { - return getResponse(ipAddress, DatabaseReader::tryDomain); - } - - @Nullable - @Override - public EnterpriseResponse getEnterprise(InetAddress ipAddress) { - return getResponse(ipAddress, DatabaseReader::tryEnterprise); - } - - @Nullable - @Override - public IspResponse getIsp(InetAddress ipAddress) { - return getResponse(ipAddress, DatabaseReader::tryIsp); - } - boolean preLookup() { return currentUsages.updateAndGet(current -> current < 0 ? current : current + 1) > 0; } @Override - public void release() throws IOException { + public void close() throws IOException { if (currentUsages.updateAndGet(current -> current > 0 ? current - 1 : current + 1) == -1) { - doClose(); + doShutdown(); } } @@ -219,21 +96,19 @@ int current() { return currentUsages.get(); } + @Override @Nullable - private T getResponse( - InetAddress ipAddress, - CheckedBiFunction, Exception> responseProvider - ) { + public RESPONSE getResponse(String ipAddress, CheckedBiFunction responseProvider) { return cache.putIfAbsent(ipAddress, databasePath.toString(), ip -> { try { - return responseProvider.apply(get(), ipAddress).orElse(null); + return responseProvider.apply(get(), ipAddress); } catch (Exception e) { - throw new RuntimeException(e); + throw ExceptionsHelper.convertToRuntime(e); } }); } - DatabaseReader get() throws IOException { + Reader get() throws IOException { if (databaseReader.get() == null) { synchronized (databaseReader) { if (databaseReader.get() == null) { @@ -249,44 +124,48 @@ String getMd5() { return md5; } - public void close(boolean shouldDeleteDatabaseFileOnClose) throws IOException { - this.deleteDatabaseFileOnClose = shouldDeleteDatabaseFileOnClose; - close(); + public void shutdown(boolean shouldDeleteDatabaseFileOnShutdown) throws IOException { + this.deleteDatabaseFileOnShutdown = shouldDeleteDatabaseFileOnShutdown; + shutdown(); } - @Override - public void close() throws IOException { + public void shutdown() throws IOException { if (currentUsages.updateAndGet(u -> -1 - u) == -1) { - doClose(); + doShutdown(); } } // Visible for Testing - protected void doClose() throws IOException { + protected void doShutdown() throws IOException { IOUtils.close(databaseReader.get()); int numEntriesEvicted = cache.purgeCacheEntriesForDatabase(databasePath); logger.info("evicted [{}] entries from cache after reloading database [{}]", numEntriesEvicted, databasePath); - if (deleteDatabaseFileOnClose) { + if (deleteDatabaseFileOnShutdown) { logger.info("deleting [{}]", databasePath); Files.delete(databasePath); } } - private static CheckedSupplier createDatabaseLoader(Path databasePath) { + private static CheckedSupplier createDatabaseLoader(Path databasePath) { return () -> { - DatabaseReader.Builder builder = createDatabaseBuilder(databasePath).withCache(NoCache.getInstance()); - if (LOAD_DATABASE_ON_HEAP) { - builder.fileMode(Reader.FileMode.MEMORY); - } else { - builder.fileMode(Reader.FileMode.MEMORY_MAPPED); - } - return builder.build(); + Reader.FileMode mode = LOAD_DATABASE_ON_HEAP ? Reader.FileMode.MEMORY : Reader.FileMode.MEMORY_MAPPED; + return new Reader(pathToFile(databasePath), mode, NoCache.getInstance()); }; } @SuppressForbidden(reason = "Maxmind API requires java.io.File") - private static DatabaseReader.Builder createDatabaseBuilder(Path databasePath) { - return new DatabaseReader.Builder(databasePath.toFile()); + private static File pathToFile(Path databasePath) { + return databasePath.toFile(); } + long getBuildDateMillis() throws IOException { + if (buildDate.get() == null) { + synchronized (buildDate) { + if (buildDate.get() == null) { + buildDate.set(loader.get().getMetadata().getBuildDate().getTime()); + } + } + } + return buildDate.get(); + } } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloader.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloader.java index 9645e34751642..e04014ff693be 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloader.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloader.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; @@ -19,9 +20,9 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.Strings; import org.elasticsearch.common.hash.MessageDigests; -import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.query.BoolQueryBuilder; @@ -37,11 +38,13 @@ import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.Scheduler; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; +import java.io.Closeable; import java.io.IOException; import java.io.InputStream; -import java.net.PasswordAuthentication; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.util.Arrays; @@ -55,6 +58,7 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; +import static org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloaderTaskExecutor.IPINFO_SETTINGS_PREFIX; import static org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloaderTaskExecutor.MAXMIND_SETTINGS_PREFIX; /** @@ -66,31 +70,28 @@ public class EnterpriseGeoIpDownloader extends AllocatedPersistentTask { private static final Logger logger = LogManager.getLogger(EnterpriseGeoIpDownloader.class); - private static final Pattern CHECKSUM_PATTERN = Pattern.compile("(\\w{64})\\s\\s(.*)"); + + // a sha256 checksum followed by two spaces followed by an (ignored) file name + private static final Pattern SHA256_CHECKSUM_PATTERN = Pattern.compile("(\\w{64})\\s\\s(.*)"); + + // an md5 checksum + private static final Pattern MD5_CHECKSUM_PATTERN = Pattern.compile("(\\w{32})"); // for overriding in tests static String DEFAULT_MAXMIND_ENDPOINT = System.getProperty( - MAXMIND_SETTINGS_PREFIX + "endpoint.default", + MAXMIND_SETTINGS_PREFIX + "endpoint.default", // "https://download.maxmind.com/geoip/databases" ); // n.b. a future enhancement might be to allow for a MAXMIND_ENDPOINT_SETTING, but // at the moment this is an unsupported system property for use in tests (only) - static String downloadUrl(final String name, final String suffix) { - String endpointPattern = DEFAULT_MAXMIND_ENDPOINT; - if (endpointPattern.contains("%")) { - throw new IllegalArgumentException("Invalid endpoint [" + endpointPattern + "]"); - } - if (endpointPattern.endsWith("/") == false) { - endpointPattern += "/"; - } - endpointPattern += "%s/download?suffix=%s"; - - // at this point the pattern looks like this (in the default case): - // https://download.maxmind.com/geoip/databases/%s/download?suffix=%s - - return Strings.format(endpointPattern, name, suffix); - } + // for overriding in tests + static String DEFAULT_IPINFO_ENDPOINT = System.getProperty( + IPINFO_SETTINGS_PREFIX + "endpoint.default", // + "https://ipinfo.io/data" + ); + // n.b. a future enhancement might be to allow for an IPINFO_ENDPOINT_SETTING, but + // at the moment this is an unsupported system property for use in tests (only) static final String DATABASES_INDEX = ".geoip_databases"; static final int MAX_CHUNK_SIZE = 1024 * 1024; @@ -104,7 +105,7 @@ static String downloadUrl(final String name, final String suffix) { protected volatile EnterpriseGeoIpTaskState state; private volatile Scheduler.ScheduledCancellable scheduled; private final Supplier pollIntervalSupplier; - private final Function credentialsBuilder; + private final Function tokenProvider; EnterpriseGeoIpDownloader( Client client, @@ -118,7 +119,7 @@ static String downloadUrl(final String name, final String suffix) { TaskId parentTask, Map headers, Supplier pollIntervalSupplier, - Function credentialsBuilder + Function tokenProvider ) { super(id, type, action, description, parentTask, headers); this.client = client; @@ -126,7 +127,7 @@ static String downloadUrl(final String name, final String suffix) { this.clusterService = clusterService; this.threadPool = threadPool; this.pollIntervalSupplier = pollIntervalSupplier; - this.credentialsBuilder = credentialsBuilder; + this.tokenProvider = tokenProvider; } void setState(EnterpriseGeoIpTaskState state) { @@ -155,7 +156,7 @@ void updateDatabases() throws IOException { } } - logger.trace("Updating geoip databases"); + logger.trace("Updating databases"); IngestGeoIpMetadata geoIpMeta = clusterState.metadata().custom(IngestGeoIpMetadata.TYPE, IngestGeoIpMetadata.EMPTY); // if there are entries in the cs that aren't in the persistent task state, @@ -173,15 +174,8 @@ void updateDatabases() throws IOException { DatabaseConfiguration database = entry.getValue().database(); if (existingDatabaseNames.contains(database.name() + ".mmdb") == false) { logger.debug("A new database appeared [{}]", database.name()); - - final String accountId = database.maxmind().accountId(); - try (HttpClient.PasswordAuthenticationHolder holder = credentialsBuilder.apply(accountId)) { - if (holder == null) { - logger.warn("No credentials found to download database [{}], skipping download...", id); - } else { - processDatabase(holder.get(), database); - addedSomething = true; - } + if (processDatabase(id, database)) { + addedSomething = true; } } } @@ -224,14 +218,8 @@ void updateDatabases() throws IOException { for (Map.Entry entry : geoIpMeta.getDatabases().entrySet()) { final String id = entry.getKey(); DatabaseConfiguration database = entry.getValue().database(); - - final String accountId = database.maxmind().accountId(); - try (HttpClient.PasswordAuthenticationHolder holder = credentialsBuilder.apply(accountId)) { - if (holder == null) { - logger.warn("No credentials found to download database [{}], skipping download...", id); - } else { - processDatabase(holder.get(), database); - } + try { + processDatabase(id, database); } catch (Exception e) { accumulator = ExceptionsHelper.useOrSuppress(accumulator, ExceptionsHelper.convertToRuntime(e)); } @@ -243,68 +231,69 @@ void updateDatabases() throws IOException { } /** - * This method fetches the sha256 file and tar.gz file for the given database from the Maxmind endpoint, then indexes that tar.gz - * file into the .geoip_databases Elasticsearch index, deleting any old versions of the database tar.gz from the index if they exist. - * If the computed sha256 does not match the expected sha256, an error will be logged and the database will not be put into the - * Elasticsearch index. + * This method fetches the checksum and database for the given database from the Maxmind endpoint, then indexes that database + * file into the .geoip_databases Elasticsearch index, deleting any old versions of the database from the index if they exist. + * If the computed checksum does not match the expected checksum, an error will be logged and the database will not be put into + * the Elasticsearch index. *

    - * As an implementation detail, this method retrieves the sha256 checksum of the database to download and then invokes - * {@link EnterpriseGeoIpDownloader#processDatabase(PasswordAuthentication, String, String, String)} with that checksum, deferring to - * that method to actually download and process the tar.gz itself. + * As an implementation detail, this method retrieves the checksum of the database to download and then invokes + * {@link EnterpriseGeoIpDownloader#processDatabase(String, Checksum, CheckedSupplier)} with that checksum, + * deferring to that method to actually download and process the database file itself. * - * @param auth The credentials to use to download from the Maxmind endpoint - * @param database The database to be downloaded from Maxmind and indexed into an Elasticsearch index - * @throws IOException If there is an error fetching the sha256 file + * @param id The identifier for this database (just for logging purposes) + * @param database The database to be downloaded and indexed into an Elasticsearch index + * @return true if the file was processed, false if the file wasn't processed (for example if credentials haven't been configured) + * @throws IOException If there is an error fetching the checksum or database file */ - void processDatabase(PasswordAuthentication auth, DatabaseConfiguration database) throws IOException { + boolean processDatabase(String id, DatabaseConfiguration database) throws IOException { final String name = database.name(); logger.debug("Processing database [{}] for configuration [{}]", name, database.id()); - final String sha256Url = downloadUrl(name, "tar.gz.sha256"); - final String tgzUrl = downloadUrl(name, "tar.gz"); - - String result = new String(httpClient.getBytes(auth, sha256Url), StandardCharsets.UTF_8).trim(); // this throws if the auth is bad - var matcher = CHECKSUM_PATTERN.matcher(result); - boolean match = matcher.matches(); - if (match == false) { - throw new RuntimeException("Unexpected sha256 response from [" + sha256Url + "]"); + try (ProviderDownload downloader = downloaderFor(database)) { + if (downloader != null && downloader.validCredentials()) { + // the name that comes from the enterprise downloader cluster state doesn't include the .mmdb extension, + // but the downloading and indexing of database code expects it to be there, so we add it on here before continuing + final String fileName = name + ".mmdb"; + processDatabase(fileName, downloader.checksum(), downloader.download()); + return true; + } else { + logger.warn("No credentials found to download database [{}], skipping download...", id); + return false; + } } - final String sha256 = matcher.group(1); - // the name that comes from the enterprise downloader cluster state doesn't include the .mmdb extension, - // but the downloading and indexing of database code expects it to be there, so we add it on here before further processing - processDatabase(auth, name + ".mmdb", sha256, tgzUrl); } /** - * This method fetches the tar.gz file for the given database from the Maxmind endpoint, then indexes that tar.gz - * file into the .geoip_databases Elasticsearch index, deleting any old versions of the database tar.gz from the index if they exist. + * This method fetches the database file for the given database from the passed-in source, then indexes that database + * file into the .geoip_databases Elasticsearch index, deleting any old versions of the database from the index if they exist. * - * @param auth The credentials to use to download from the Maxmind endpoint - * The name of the database to be downloaded from Maxmind and indexed into an Elasticsearch index - * @param sha256 The sha256 to compare to the computed sha256 of the downloaded tar.gz file - * @param url The URL for the Maxmind endpoint from which the database's tar.gz will be downloaded + * @param name The name of the database to be downloaded and indexed into an Elasticsearch index + * @param checksum The checksum to compare to the computed checksum of the downloaded file + * @param source The supplier of an InputStream that will actually download the file */ - private void processDatabase(PasswordAuthentication auth, String name, String sha256, String url) { + private void processDatabase(final String name, final Checksum checksum, final CheckedSupplier source) { Metadata metadata = state.getDatabases().getOrDefault(name, Metadata.EMPTY); - if (Objects.equals(metadata.sha256(), sha256)) { + if (checksum.matches(metadata)) { updateTimestamp(name, metadata); return; } - logger.debug("downloading geoip database [{}]", name); + logger.debug("downloading database [{}]", name); long start = System.currentTimeMillis(); - try (InputStream is = httpClient.get(auth, url)) { + try (InputStream is = source.get()) { int firstChunk = metadata.lastChunk() + 1; // if there is no metadata, then Metadata.EMPTY + 1 = 0 - Tuple tuple = indexChunks(name, is, firstChunk, MessageDigests.sha256(), sha256, start); + Tuple tuple = indexChunks(name, is, firstChunk, checksum, start); int lastChunk = tuple.v1(); - String md5 = tuple.v2(); + String md5 = tuple.v2(); // the md5 of the bytes as they passed through indexChunks if (lastChunk > firstChunk) { + // if there is a sha256 for this download, then record it (otherwise record null for it, which is also fine) + String sha256 = checksum.type == Checksum.Type.SHA256 ? checksum.checksum : null; state = state.put(name, new Metadata(start, firstChunk, lastChunk - 1, md5, start, sha256)); updateTaskState(); - logger.info("successfully downloaded geoip database [{}]", name); + logger.info("successfully downloaded database [{}]", name); deleteOldChunks(name, firstChunk); } } catch (Exception e) { - logger.error(() -> "error downloading geoip database [" + name + "]", e); + logger.error(() -> "error downloading database [" + name + "]", e); } } @@ -318,13 +307,13 @@ void deleteOldChunks(String name, int firstChunk) { client.execute( DeleteByQueryAction.INSTANCE, request, - ActionListener.wrap(r -> {}, e -> logger.warn("could not delete old chunks for geoip database [" + name + "]", e)) + ActionListener.wrap(r -> {}, e -> logger.warn("could not delete old chunks for database [" + name + "]", e)) ); } // visible for testing protected void updateTimestamp(String name, Metadata old) { - logger.debug("geoip database [{}] is up to date, updated timestamp", name); + logger.debug("database [{}] is up to date, updated timestamp", name); state = state.put( name, new Metadata(old.lastUpdate(), old.firstChunk(), old.lastChunk(), old.md5(), System.currentTimeMillis(), old.sha256()) @@ -339,15 +328,11 @@ void updateTaskState() { } // visible for testing - Tuple indexChunks( - String name, - InputStream is, - int chunk, - @Nullable MessageDigest digest, - String expectedChecksum, - long timestamp - ) throws IOException { + Tuple indexChunks(String name, InputStream is, int chunk, final Checksum checksum, long timestamp) throws IOException { + // we have to calculate and return md5 sums as a matter of course (see actualMd5 being return below), + // but we don't have to do it *twice* -- so if the passed-in checksum is also md5, then we'll get null here MessageDigest md5 = MessageDigests.md5(); + MessageDigest digest = checksum.digest(); // this returns null for md5 for (byte[] buf = getChunk(is); buf.length != 0; buf = getChunk(is)) { md5.update(buf); if (digest != null) { @@ -370,6 +355,7 @@ Tuple indexChunks( String actualMd5 = MessageDigests.toHexString(md5.digest()); String actualChecksum = digest == null ? actualMd5 : MessageDigests.toHexString(digest.digest()); + String expectedChecksum = checksum.checksum; if (Objects.equals(expectedChecksum, actualChecksum) == false) { throw new IOException("checksum mismatch, expected [" + expectedChecksum + "], actual [" + actualChecksum + "]"); } @@ -417,12 +403,12 @@ synchronized void runDownloader() { try { updateDatabases(); // n.b. this downloads bytes from the internet, it can take a while } catch (Exception e) { - logger.error("exception during geoip databases update", e); + logger.error("exception during databases update", e); } try { cleanDatabases(); } catch (Exception e) { - logger.error("exception during geoip databases cleanup", e); + logger.error("exception during databases cleanup", e); } } @@ -471,4 +457,238 @@ private void scheduleNextRun(TimeValue time) { } } + private ProviderDownload downloaderFor(DatabaseConfiguration database) { + if (database.provider() instanceof DatabaseConfiguration.Maxmind maxmind) { + return new MaxmindDownload(database.name(), maxmind); + } else if (database.provider() instanceof DatabaseConfiguration.Ipinfo ipinfo) { + return new IpinfoDownload(database.name(), ipinfo); + } else { + throw new IllegalArgumentException( + Strings.format("Unexpected provider [%s] for configuration [%s]", database.provider().getClass(), database.id()) + ); + } + } + + class MaxmindDownload implements ProviderDownload { + + final String name; + final DatabaseConfiguration.Maxmind maxmind; + HttpClient.PasswordAuthenticationHolder auth; + + MaxmindDownload(String name, DatabaseConfiguration.Maxmind maxmind) { + this.name = name; + this.maxmind = maxmind; + this.auth = buildCredentials(); + } + + @Override + public HttpClient.PasswordAuthenticationHolder buildCredentials() { + // if the username is missing, empty, or blank, return null as 'no auth' + final String username = maxmind.accountId(); + if (username == null || username.isEmpty() || username.isBlank()) { + return null; + } + + // likewise if the password chars array is missing or empty, return null as 'no auth' + final char[] passwordChars = tokenProvider.apply("maxmind"); + if (passwordChars == null || passwordChars.length == 0) { + return null; + } + + return new HttpClient.PasswordAuthenticationHolder(username, passwordChars); + } + + @Override + public boolean validCredentials() { + return auth != null && auth.get() != null; + } + + @Override + public String url(String suffix) { + String endpointPattern = DEFAULT_MAXMIND_ENDPOINT; + if (endpointPattern.contains("%")) { + throw new IllegalArgumentException("Invalid endpoint [" + endpointPattern + "]"); + } + if (endpointPattern.endsWith("/") == false) { + endpointPattern += "/"; + } + endpointPattern += "%s/download?suffix=%s"; + + // at this point the pattern looks like this (in the default case): + // https://download.maxmind.com/geoip/databases/%s/download?suffix=%s + + return Strings.format(endpointPattern, name, suffix); + } + + @Override + public Checksum checksum() throws IOException { + final String sha256Url = this.url("tar.gz.sha256"); + var result = new String(httpClient.getBytes(auth.get(), sha256Url), StandardCharsets.UTF_8).trim(); // throws if the auth is bad + var matcher = SHA256_CHECKSUM_PATTERN.matcher(result); + boolean match = matcher.matches(); + if (match == false) { + throw new RuntimeException("Unexpected sha256 response from [" + sha256Url + "]"); + } + final String sha256 = matcher.group(1); + return Checksum.sha256(sha256); + } + + @Override + public CheckedSupplier download() { + final String tgzUrl = this.url("tar.gz"); + return () -> httpClient.get(auth.get(), tgzUrl); + } + + @Override + public void close() throws IOException { + if (auth != null) auth.close(); + } + } + + class IpinfoDownload implements ProviderDownload { + + final String name; + final DatabaseConfiguration.Ipinfo ipinfo; + HttpClient.PasswordAuthenticationHolder auth; + + IpinfoDownload(String name, DatabaseConfiguration.Ipinfo ipinfo) { + this.name = name; + this.ipinfo = ipinfo; + this.auth = buildCredentials(); + } + + @Override + public HttpClient.PasswordAuthenticationHolder buildCredentials() { + final char[] tokenChars = tokenProvider.apply("ipinfo"); + + // if the token is missing or empty, return null as 'no auth' + if (tokenChars == null || tokenChars.length == 0) { + return null; + } + + // ipinfo uses the token as the username component of basic auth, see https://ipinfo.io/developers#authentication + return new HttpClient.PasswordAuthenticationHolder(new String(tokenChars), new char[] {}); + } + + @Override + public boolean validCredentials() { + return auth != null && auth.get() != null; + } + + private static final Set FREE_DATABASES = Set.of("asn", "country", "country_asn"); + + @Override + public String url(String suffix) { + // note: the 'free' databases are in the sub-path 'free/' in terms of the download endpoint + final String internalName; + if (FREE_DATABASES.contains(name)) { + internalName = "free/" + name; + } else { + internalName = name; + } + + // reminder, we're passing the ipinfo token as the username part of http basic auth, + // see https://ipinfo.io/developers#authentication + + String endpointPattern = DEFAULT_IPINFO_ENDPOINT; + if (endpointPattern.contains("%")) { + throw new IllegalArgumentException("Invalid endpoint [" + endpointPattern + "]"); + } + if (endpointPattern.endsWith("/") == false) { + endpointPattern += "/"; + } + endpointPattern += "%s.%s"; + + // at this point the pattern looks like this (in the default case): + // https://ipinfo.io/data/%s.%s + // also see https://ipinfo.io/developers/database-download, + // and https://ipinfo.io/developers/database-filename-reference for more + + return Strings.format(endpointPattern, internalName, suffix); + } + + @Override + public Checksum checksum() throws IOException { + final String checksumJsonUrl = this.url("mmdb/checksums"); // a minor abuse of the idea of a 'suffix', :shrug: + byte[] data = httpClient.getBytes(auth.get(), checksumJsonUrl); // this throws if the auth is bad + Map checksums; + try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, data)) { + checksums = parser.map(); + } + @SuppressWarnings("unchecked") + String md5 = ((Map) checksums.get("checksums")).get("md5"); + logger.trace("checksum was [{}]", md5); + + var matcher = MD5_CHECKSUM_PATTERN.matcher(md5); + boolean match = matcher.matches(); + if (match == false) { + throw new RuntimeException("Unexpected md5 response from [" + checksumJsonUrl + "]"); + } + return Checksum.md5(md5); + } + + @Override + public CheckedSupplier download() { + final String mmdbUrl = this.url("mmdb"); + return () -> httpClient.get(auth.get(), mmdbUrl); + } + + @Override + public void close() throws IOException { + if (auth != null) auth.close(); + } + } + + interface ProviderDownload extends Closeable { + // note: buildCredentials and url are inherently just implementation details of checksum() and download(), + // but it's useful to have unit tests for this logic and to keep it separate + HttpClient.PasswordAuthenticationHolder buildCredentials(); + + String url(String suffix); + + boolean validCredentials(); + + Checksum checksum() throws IOException; + + CheckedSupplier download(); + + @Override + void close() throws IOException; + } + + record Checksum(Type type, String checksum) { + + // use the static factory methods, though, rather than this + public Checksum { + Objects.requireNonNull(type); + Objects.requireNonNull(checksum); + } + + static Checksum md5(String checksum) { + return new Checksum(Type.MD5, checksum); + } + + static Checksum sha256(String checksum) { + return new Checksum(Type.SHA256, checksum); + } + + enum Type { + MD5, + SHA256 + } + + MessageDigest digest() { + return switch (type) { + case MD5 -> null; // a leaky implementation detail, we don't need to calculate two md5s + case SHA256 -> MessageDigests.sha256(); + }; + } + + boolean matches(Metadata metadata) { + return switch (type) { + case MD5 -> checksum.equals(metadata.md5()); + case SHA256 -> checksum.equals(metadata.sha256()); + }; + } + } } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTaskExecutor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTaskExecutor.java index 8fc46fe157548..ae9bb109a3bf8 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTaskExecutor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTaskExecutor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; @@ -53,11 +54,15 @@ public class EnterpriseGeoIpDownloaderTaskExecutor extends PersistentTasksExecut static final String MAXMIND_SETTINGS_PREFIX = "ingest.geoip.downloader.maxmind."; + static final String IPINFO_SETTINGS_PREFIX = "ingest.ip_location.downloader.ipinfo."; + public static final Setting MAXMIND_LICENSE_KEY_SETTING = SecureSetting.secureString( MAXMIND_SETTINGS_PREFIX + "license_key", null ); + public static final Setting IPINFO_TOKEN_SETTING = SecureSetting.secureString(IPINFO_SETTINGS_PREFIX + "token", null); + private final Client client; private final HttpClient httpClient; private final ClusterService clusterService; @@ -99,25 +104,18 @@ private void setPollInterval(TimeValue pollInterval) { } } - private HttpClient.PasswordAuthenticationHolder buildCredentials(final String username) { - final char[] passwordChars; - if (cachedSecureSettings.getSettingNames().contains(MAXMIND_LICENSE_KEY_SETTING.getKey())) { - passwordChars = cachedSecureSettings.getString(MAXMIND_LICENSE_KEY_SETTING.getKey()).getChars(); - } else { - passwordChars = null; - } - - // if the username is missing, empty, or blank, return null as 'no auth' - if (username == null || username.isEmpty() || username.isBlank()) { - return null; - } - - // likewise if the password chars array is missing or empty, return null as 'no auth' - if (passwordChars == null || passwordChars.length == 0) { - return null; + private char[] getSecureToken(final String type) { + char[] token = null; + if (type.equals("maxmind")) { + if (cachedSecureSettings.getSettingNames().contains(MAXMIND_LICENSE_KEY_SETTING.getKey())) { + token = cachedSecureSettings.getString(MAXMIND_LICENSE_KEY_SETTING.getKey()).getChars(); + } + } else if (type.equals("ipinfo")) { + if (cachedSecureSettings.getSettingNames().contains(IPINFO_TOKEN_SETTING.getKey())) { + token = cachedSecureSettings.getString(IPINFO_TOKEN_SETTING.getKey()).getChars(); + } } - - return new HttpClient.PasswordAuthenticationHolder(username, passwordChars); + return token; } @Override @@ -141,7 +139,7 @@ protected EnterpriseGeoIpDownloader createTask( parentTaskId, headers, () -> pollInterval, - this::buildCredentials + this::getSecureToken ); } @@ -176,7 +174,7 @@ public synchronized void reload(Settings settings) { // `SecureSettings` are available here! cache them as they will be needed // whenever dynamic cluster settings change and we have to rebuild the accounts try { - this.cachedSecureSettings = extractSecureSettings(settings, List.of(MAXMIND_LICENSE_KEY_SETTING)); + this.cachedSecureSettings = extractSecureSettings(settings, List.of(MAXMIND_LICENSE_KEY_SETTING, IPINFO_TOKEN_SETTING)); } catch (GeneralSecurityException e) { // rethrow as a runtime exception, there's logging higher up the call chain around ReloadablePlugin throw new ElasticsearchException("Exception while reloading enterprise geoip download task executor", e); diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpTaskState.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpTaskState.java index 57e944ef9b994..c128af69009be 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpTaskState.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpTaskState.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; @@ -122,7 +123,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ENTERPRISE_GEOIP_DOWNLOADER; + return TransportVersions.V_8_16_0; } @Override diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java index fc70e2fbb594a..d9c9c3aaf3266 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java @@ -1,21 +1,20 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; import com.maxmind.db.NodeCache; -import com.maxmind.geoip2.model.AbstractResponse; import org.elasticsearch.common.cache.Cache; import org.elasticsearch.common.cache.CacheBuilder; import org.elasticsearch.core.TimeValue; import org.elasticsearch.ingest.geoip.stats.CacheStats; -import java.net.InetAddress; import java.nio.file.Path; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; @@ -27,7 +26,7 @@ * cost of deserialization for each lookup (cached or not). This comes at slight expense of higher memory usage, but significant * reduction of CPU usage. */ -final class GeoIpCache { +public final class GeoIpCache { /** * Internal-only sentinel object for recording that a result from the geoip database was null (i.e. there was no result). By caching @@ -35,15 +34,15 @@ final class GeoIpCache { * something not being in the cache because the data doesn't exist in the database. */ // visible for testing - static final AbstractResponse NO_RESULT = new AbstractResponse() { + static final Object NO_RESULT = new Object() { @Override public String toString() { - return "AbstractResponse[NO_RESULT]"; + return "NO_RESULT"; } }; private final LongSupplier relativeNanoTimeProvider; - private final Cache cache; + private final Cache cache; private final AtomicLong hitsTimeInNanos = new AtomicLong(0); private final AtomicLong missesTimeInNanos = new AtomicLong(0); @@ -53,7 +52,7 @@ public String toString() { throw new IllegalArgumentException("geoip max cache size must be 0 or greater"); } this.relativeNanoTimeProvider = relativeNanoTimeProvider; - this.cache = CacheBuilder.builder().setMaximumWeight(maxSize).build(); + this.cache = CacheBuilder.builder().setMaximumWeight(maxSize).build(); } GeoIpCache(long maxSize) { @@ -61,16 +60,12 @@ public String toString() { } @SuppressWarnings("unchecked") - T putIfAbsent( - InetAddress ip, - String databasePath, - Function retrieveFunction - ) { + RESPONSE putIfAbsent(String ip, String databasePath, Function retrieveFunction) { // can't use cache.computeIfAbsent due to the elevated permissions for the jackson (run via the cache loader) CacheKey cacheKey = new CacheKey(ip, databasePath); long cacheStart = relativeNanoTimeProvider.getAsLong(); // intentionally non-locking for simplicity...it's OK if we re-put the same key/value in the cache during a race condition. - AbstractResponse response = cache.get(cacheKey); + Object response = cache.get(cacheKey); long cacheRequestTime = relativeNanoTimeProvider.getAsLong() - cacheStart; // populate the cache for this key, if necessary @@ -92,12 +87,12 @@ T putIfAbsent( if (response == NO_RESULT) { return null; // the no-result sentinel is an internal detail, don't expose it } else { - return (T) response; + return (RESPONSE) response; } } // only useful for testing - AbstractResponse get(InetAddress ip, String databasePath) { + Object get(String ip, String databasePath) { CacheKey cacheKey = new CacheKey(ip, databasePath); return cache.get(cacheKey); } @@ -140,5 +135,5 @@ public CacheStats getCacheStats() { * path is needed to be included in the cache key. For example, if we only used the IP address as the key the City and ASN the same * IP may be in both with different values and we need to cache both. */ - private record CacheKey(InetAddress ip, String databasePath) {} + private record CacheKey(String ip, String databasePath) {} } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDatabase.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDatabase.java deleted file mode 100644 index 674c500f069bc..0000000000000 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDatabase.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.ingest.geoip; - -import com.maxmind.geoip2.model.AnonymousIpResponse; -import com.maxmind.geoip2.model.AsnResponse; -import com.maxmind.geoip2.model.CityResponse; -import com.maxmind.geoip2.model.ConnectionTypeResponse; -import com.maxmind.geoip2.model.CountryResponse; -import com.maxmind.geoip2.model.DomainResponse; -import com.maxmind.geoip2.model.EnterpriseResponse; -import com.maxmind.geoip2.model.IspResponse; - -import org.elasticsearch.core.Nullable; - -import java.io.IOException; -import java.net.InetAddress; - -/** - * Provides a uniform interface for interacting with various GeoIP databases. - */ -public interface GeoIpDatabase { - - /** - * @return the database type as it is detailed in the database file metadata - * @throws IOException if the database file could not be read or the data could not be accessed - */ - String getDatabaseType() throws IOException; - - /** - * @param ipAddress the IP address to look up - * @return a response containing the city data for the given address if it exists, or null if it could not be found - * @throws UnsupportedOperationException may be thrown if the implementation does not support retrieving city data - */ - @Nullable - CityResponse getCity(InetAddress ipAddress); - - /** - * @param ipAddress the IP address to look up - * @return a response containing the country data for the given address if it exists, or null if it could not be found - * @throws UnsupportedOperationException may be thrown if the implementation does not support retrieving country data - */ - @Nullable - CountryResponse getCountry(InetAddress ipAddress); - - /** - * @param ipAddress the IP address to look up - * @return a response containing the Autonomous System Number for the given address if it exists, or null if it could not - * be found - * @throws UnsupportedOperationException may be thrown if the implementation does not support retrieving ASN data - */ - @Nullable - AsnResponse getAsn(InetAddress ipAddress); - - @Nullable - AnonymousIpResponse getAnonymousIp(InetAddress ipAddress); - - @Nullable - ConnectionTypeResponse getConnectionType(InetAddress ipAddress); - - @Nullable - DomainResponse getDomain(InetAddress ipAddress); - - @Nullable - EnterpriseResponse getEnterprise(InetAddress ipAddress); - - @Nullable - IspResponse getIsp(InetAddress ipAddress); - - /** - * Releases the current database object. Called after processing a single document. Databases should be closed or returned to a - * resource pool. No further interactions should be expected. - * @throws IOException if the implementation encounters any problem while cleaning up - */ - void release() throws IOException; -} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDatabaseProvider.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDatabaseProvider.java deleted file mode 100644 index 0212417aed26c..0000000000000 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDatabaseProvider.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.ingest.geoip; - -/** - * Provides construction and initialization logic for {@link GeoIpDatabase} instances. - */ -public interface GeoIpDatabaseProvider { - - /** - * Determines if the given database name corresponds to an expired database. Expired databases will not be loaded. - *

    - * Verifying database expiration is left to each provider implementation to determine. A return value of false does not - * preclude the possibility of a provider returning true in the future. - * - * @param name the name of the database to provide. - * @return false IFF the requested database file is expired, - * true for all other cases (including unknown file name, file missing, wrong database type, etc). - */ - Boolean isValid(String name); - - /** - * @param name the name of the database to provide. - * @return a ready-to-use database instance, or null if no database could be loaded. - */ - GeoIpDatabase getDatabase(String name); -} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java index ee6f2f16f051b..ae562d3c7359a 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java @@ -1,16 +1,16 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.flush.FlushRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; @@ -138,11 +138,18 @@ void updateDatabases() throws IOException { if (geoipIndex != null) { logger.trace("The {} index is not null", GeoIpDownloader.DATABASES_INDEX); if (clusterState.getRoutingTable().index(geoipIndex.getWriteIndex()).allPrimaryShardsActive() == false) { - throw new ElasticsearchException("not all primary shards of [" + DATABASES_INDEX + "] index are active"); + logger.debug( + "Not updating geoip database because not all primary shards of the [" + DATABASES_INDEX + "] index are active." + ); + return; } var blockException = clusterState.blocks().indexBlockedException(ClusterBlockLevel.WRITE, geoipIndex.getWriteIndex().getName()); if (blockException != null) { - throw blockException; + logger.debug( + "Not updating geoip database because there is a write block on the " + geoipIndex.getWriteIndex().getName() + " index", + blockException + ); + return; } } if (eagerDownloadSupplier.get() || atLeastOneGeoipProcessorSupplier.get()) { diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java index 3f89bb1dd5c50..f245b0cfd2405 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; @@ -42,17 +43,20 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.RemoteTransportException; +import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; import static org.elasticsearch.ingest.geoip.GeoIpDownloader.DATABASES_INDEX; import static org.elasticsearch.ingest.geoip.GeoIpDownloader.GEOIP_DOWNLOADER; import static org.elasticsearch.ingest.geoip.GeoIpProcessor.Factory.downloadDatabaseOnPipelineCreation; +import static org.elasticsearch.ingest.geoip.GeoIpProcessor.GEOIP_TYPE; +import static org.elasticsearch.ingest.geoip.GeoIpProcessor.IP_LOCATION_TYPE; /** * Persistent task executor that is responsible for starting {@link GeoIpDownloader} after task is allocated by master node. @@ -235,14 +239,11 @@ public void clusterChanged(ClusterChangedEvent event) { } static boolean hasAtLeastOneGeoipProcessor(ClusterState clusterState) { - if (pipelineConfigurationsWithGeoIpProcessor(clusterState, true).isEmpty() == false) { + if (pipelinesWithGeoIpProcessor(clusterState, true).isEmpty() == false) { return true; } - Set checkReferencedPipelines = pipelineConfigurationsWithGeoIpProcessor(clusterState, false).stream() - .map(PipelineConfiguration::getId) - .collect(Collectors.toSet()); - + final Set checkReferencedPipelines = pipelinesWithGeoIpProcessor(clusterState, false); if (checkReferencedPipelines.isEmpty()) { return false; } @@ -255,22 +256,24 @@ static boolean hasAtLeastOneGeoipProcessor(ClusterState clusterState) { } /** - * Retrieve list of pipelines that have at least one geoip processor. + * Retrieve the set of pipeline ids that have at least one geoip processor. * @param clusterState Cluster state. * @param downloadDatabaseOnPipelineCreation Filter the list to include only pipeline with the download_database_on_pipeline_creation * matching the param. - * @return A list of {@link PipelineConfiguration} matching criteria. + * @return A set of pipeline ids matching criteria. */ @SuppressWarnings("unchecked") - private static List pipelineConfigurationsWithGeoIpProcessor( - ClusterState clusterState, - boolean downloadDatabaseOnPipelineCreation - ) { - List pipelineDefinitions = IngestService.getPipelines(clusterState); - return pipelineDefinitions.stream().filter(pipelineConfig -> { - List> processors = (List>) pipelineConfig.getConfigAsMap().get(Pipeline.PROCESSORS_KEY); - return hasAtLeastOneGeoipProcessor(processors, downloadDatabaseOnPipelineCreation); - }).toList(); + private static Set pipelinesWithGeoIpProcessor(ClusterState clusterState, boolean downloadDatabaseOnPipelineCreation) { + List configurations = IngestService.getPipelines(clusterState); + Set ids = new HashSet<>(); + // note: this loop is unrolled rather than streaming-style because it's hot enough to show up in a flamegraph + for (PipelineConfiguration configuration : configurations) { + List> processors = (List>) configuration.getConfig().get(Pipeline.PROCESSORS_KEY); + if (hasAtLeastOneGeoipProcessor(processors, downloadDatabaseOnPipelineCreation)) { + ids.add(configuration.getId()); + } + } + return Collections.unmodifiableSet(ids); } /** @@ -280,7 +283,15 @@ private static List pipelineConfigurationsWithGeoIpProces * @return true if a geoip processor is found in the processor list. */ private static boolean hasAtLeastOneGeoipProcessor(List> processors, boolean downloadDatabaseOnPipelineCreation) { - return processors != null && processors.stream().anyMatch(p -> hasAtLeastOneGeoipProcessor(p, downloadDatabaseOnPipelineCreation)); + if (processors != null) { + // note: this loop is unrolled rather than streaming-style because it's hot enough to show up in a flamegraph + for (Map processor : processors) { + if (hasAtLeastOneGeoipProcessor(processor, downloadDatabaseOnPipelineCreation)) { + return true; + } + } + } + return false; } /** @@ -295,9 +306,18 @@ private static boolean hasAtLeastOneGeoipProcessor(Map processor return false; } - if (processor.containsKey(GeoIpProcessor.TYPE)) { - Map processorConfig = (Map) processor.get(GeoIpProcessor.TYPE); - return downloadDatabaseOnPipelineCreation(processorConfig) == downloadDatabaseOnPipelineCreation; + { + final Map processorConfig = (Map) processor.get(GEOIP_TYPE); + if (processorConfig != null) { + return downloadDatabaseOnPipelineCreation(processorConfig) == downloadDatabaseOnPipelineCreation; + } + } + + { + final Map processorConfig = (Map) processor.get(IP_LOCATION_TYPE); + if (processorConfig != null) { + return downloadDatabaseOnPipelineCreation(processorConfig) == downloadDatabaseOnPipelineCreation; + } } return isProcessorWithOnFailureGeoIpProcessor(processor, downloadDatabaseOnPipelineCreation) @@ -305,7 +325,7 @@ private static boolean hasAtLeastOneGeoipProcessor(Map processor } /** - * Check if a processor config is has an on_failure clause containing at least a geoip processor. + * Check if a processor config has an on_failure clause containing at least a geoip processor. * @param processor Processor config. * @param downloadDatabaseOnPipelineCreation Should the download_database_on_pipeline_creation of the geoip processor be true or false. * @return true if a geoip processor is found in the processor list. @@ -315,16 +335,17 @@ private static boolean isProcessorWithOnFailureGeoIpProcessor( Map processor, boolean downloadDatabaseOnPipelineCreation ) { - return processor != null - && processor.values() - .stream() - .anyMatch( - value -> value instanceof Map - && hasAtLeastOneGeoipProcessor( - ((Map>>) value).get("on_failure"), - downloadDatabaseOnPipelineCreation - ) - ); + // note: this loop is unrolled rather than streaming-style because it's hot enough to show up in a flamegraph + for (Object value : processor.values()) { + if (value instanceof Map + && hasAtLeastOneGeoipProcessor( + ((Map>>) value).get("on_failure"), + downloadDatabaseOnPipelineCreation + )) { + return true; + } + } + return false; } /** @@ -335,11 +356,9 @@ && hasAtLeastOneGeoipProcessor( */ @SuppressWarnings("unchecked") private static boolean isForeachProcessorWithGeoipProcessor(Map processor, boolean downloadDatabaseOnPipelineCreation) { - return processor.containsKey("foreach") - && hasAtLeastOneGeoipProcessor( - ((Map>) processor.get("foreach")).get("processor"), - downloadDatabaseOnPipelineCreation - ); + final Map processorConfig = (Map) processor.get("foreach"); + return processorConfig != null + && hasAtLeastOneGeoipProcessor((Map) processorConfig.get("processor"), downloadDatabaseOnPipelineCreation); } @UpdateForV9 // use MINUS_ONE once that means no timeout diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java index 82b9e930280b7..f99f8dbe2fdd0 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java @@ -1,46 +1,29 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; -import com.maxmind.db.Network; -import com.maxmind.geoip2.model.AnonymousIpResponse; -import com.maxmind.geoip2.model.AsnResponse; -import com.maxmind.geoip2.model.CityResponse; -import com.maxmind.geoip2.model.ConnectionTypeResponse; -import com.maxmind.geoip2.model.ConnectionTypeResponse.ConnectionType; -import com.maxmind.geoip2.model.CountryResponse; -import com.maxmind.geoip2.model.DomainResponse; -import com.maxmind.geoip2.model.EnterpriseResponse; -import com.maxmind.geoip2.model.IspResponse; -import com.maxmind.geoip2.record.City; -import com.maxmind.geoip2.record.Continent; -import com.maxmind.geoip2.record.Country; -import com.maxmind.geoip2.record.Location; -import com.maxmind.geoip2.record.Subdivision; - -import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.common.network.InetAddresses; -import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.core.Assertions; +import org.elasticsearch.core.Strings; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; import org.elasticsearch.ingest.geoip.Database.Property; +import org.elasticsearch.ingest.geoip.IpDataLookupFactories.IpDataLookupFactory; import java.io.IOException; -import java.net.InetAddress; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.function.Supplier; @@ -55,14 +38,18 @@ public final class GeoIpProcessor extends AbstractProcessor { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(GeoIpProcessor.class); static final String DEFAULT_DATABASES_DEPRECATION_MESSAGE = "the [fallback_to_default_databases] has been deprecated, because " + "Elasticsearch no longer includes the default Maxmind geoip databases. This setting will be removed in Elasticsearch 9.0"; + static final String UNSUPPORTED_DATABASE_DEPRECATION_MESSAGE = "the geoip processor will no longer support database type [{}] " + + "in a future version of Elasticsearch"; // TODO add a message about migration? - public static final String TYPE = "geoip"; + public static final String GEOIP_TYPE = "geoip"; + public static final String IP_LOCATION_TYPE = "ip_location"; + private final String type; private final String field; private final Supplier isValid; private final String targetField; - private final CheckedSupplier supplier; - private final Set properties; + private final CheckedSupplier supplier; + private final IpDataLookup ipDataLookup; private final boolean ignoreMissing; private final boolean firstOnly; private final String databaseFile; @@ -75,29 +62,31 @@ public final class GeoIpProcessor extends AbstractProcessor { * @param supplier a supplier of a geo-IP database reader; ideally this is lazily-loaded once on first use * @param isValid a supplier that determines if the available database files are up-to-date and license compliant * @param targetField the target field - * @param properties the properties; ideally this is lazily-loaded once on first use + * @param ipDataLookup a lookup capable of retrieving a result from an available geo-IP database reader * @param ignoreMissing true if documents with a missing value for the field should be ignored * @param firstOnly true if only first result should be returned in case of array * @param databaseFile the name of the database file being queried; used only for tagging documents if the database is unavailable */ GeoIpProcessor( + final String type, final String tag, final String description, final String field, - final CheckedSupplier supplier, + final CheckedSupplier supplier, final Supplier isValid, final String targetField, - final Set properties, + final IpDataLookup ipDataLookup, final boolean ignoreMissing, final boolean firstOnly, final String databaseFile ) { super(tag, description); + this.type = type; this.field = field; this.isValid = isValid; this.targetField = targetField; this.supplier = supplier; - this.properties = properties; + this.ipDataLookup = ipDataLookup; this.ignoreMissing = ignoreMissing; this.firstOnly = firstOnly; this.databaseFile = databaseFile; @@ -108,87 +97,64 @@ boolean isIgnoreMissing() { } @Override - public IngestDocument execute(IngestDocument ingestDocument) throws IOException { - Object ip = ingestDocument.getFieldValue(field, Object.class, ignoreMissing); + public IngestDocument execute(IngestDocument document) throws IOException { + Object ip = document.getFieldValue(field, Object.class, ignoreMissing); if (isValid.get() == false) { - ingestDocument.appendFieldValue("tags", "_geoip_expired_database", false); - return ingestDocument; + document.appendFieldValue("tags", "_" + type + "_expired_database", false); + return document; } else if (ip == null && ignoreMissing) { - return ingestDocument; + return document; } else if (ip == null) { throw new IllegalArgumentException("field [" + field + "] is null, cannot extract geoip information."); } - GeoIpDatabase geoIpDatabase = this.supplier.get(); - if (geoIpDatabase == null) { - if (ignoreMissing == false) { - tag(ingestDocument, databaseFile); + try (IpDatabase ipDatabase = this.supplier.get()) { + if (ipDatabase == null) { + if (ignoreMissing == false) { + tag(document, type, databaseFile); + } + return document; } - return ingestDocument; - } - try { if (ip instanceof String ipString) { - Map geoData = getGeoData(geoIpDatabase, ipString); - if (geoData.isEmpty() == false) { - ingestDocument.setFieldValue(targetField, geoData); + Map data = ipDataLookup.getData(ipDatabase, ipString); + if (data.isEmpty() == false) { + document.setFieldValue(targetField, data); } } else if (ip instanceof List ipList) { boolean match = false; - List> geoDataList = new ArrayList<>(ipList.size()); + List> dataList = new ArrayList<>(ipList.size()); for (Object ipAddr : ipList) { if (ipAddr instanceof String == false) { throw new IllegalArgumentException("array in field [" + field + "] should only contain strings"); } - Map geoData = getGeoData(geoIpDatabase, (String) ipAddr); - if (geoData.isEmpty()) { - geoDataList.add(null); + Map data = ipDataLookup.getData(ipDatabase, (String) ipAddr); + if (data.isEmpty()) { + dataList.add(null); continue; } if (firstOnly) { - ingestDocument.setFieldValue(targetField, geoData); - return ingestDocument; + document.setFieldValue(targetField, data); + return document; } match = true; - geoDataList.add(geoData); + dataList.add(data); } if (match) { - ingestDocument.setFieldValue(targetField, geoDataList); + document.setFieldValue(targetField, dataList); } } else { throw new IllegalArgumentException("field [" + field + "] should contain only string or array of strings"); } - } finally { - geoIpDatabase.release(); } - return ingestDocument; - } - private Map getGeoData(GeoIpDatabase geoIpDatabase, String ip) throws IOException { - final String databaseType = geoIpDatabase.getDatabaseType(); - final Database database; - try { - database = Database.getDatabase(databaseType, databaseFile); - } catch (IllegalArgumentException e) { - throw new ElasticsearchParseException(e.getMessage(), e); - } - final InetAddress ipAddress = InetAddresses.forString(ip); - return switch (database) { - case City -> retrieveCityGeoData(geoIpDatabase, ipAddress); - case Country -> retrieveCountryGeoData(geoIpDatabase, ipAddress); - case Asn -> retrieveAsnGeoData(geoIpDatabase, ipAddress); - case AnonymousIp -> retrieveAnonymousIpGeoData(geoIpDatabase, ipAddress); - case ConnectionType -> retrieveConnectionTypeGeoData(geoIpDatabase, ipAddress); - case Domain -> retrieveDomainGeoData(geoIpDatabase, ipAddress); - case Enterprise -> retrieveEnterpriseGeoData(geoIpDatabase, ipAddress); - case Isp -> retrieveIspGeoData(geoIpDatabase, ipAddress); - }; + return document; } @Override public String getType() { - return TYPE; + return type; } String getField() { @@ -204,510 +170,45 @@ String getDatabaseType() throws IOException { } Set getProperties() { - return properties; - } - - private Map retrieveCityGeoData(GeoIpDatabase geoIpDatabase, InetAddress ipAddress) { - CityResponse response = geoIpDatabase.getCity(ipAddress); - if (response == null) { - return Map.of(); - } - Country country = response.getCountry(); - City city = response.getCity(); - Location location = response.getLocation(); - Continent continent = response.getContinent(); - Subdivision subdivision = response.getMostSpecificSubdivision(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", NetworkAddress.format(ipAddress)); - case COUNTRY_ISO_CODE -> { - String countryIsoCode = country.getIsoCode(); - if (countryIsoCode != null) { - geoData.put("country_iso_code", countryIsoCode); - } - } - case COUNTRY_NAME -> { - String countryName = country.getName(); - if (countryName != null) { - geoData.put("country_name", countryName); - } - } - case CONTINENT_CODE -> { - String continentCode = continent.getCode(); - if (continentCode != null) { - geoData.put("continent_code", continentCode); - } - } - case CONTINENT_NAME -> { - String continentName = continent.getName(); - if (continentName != null) { - geoData.put("continent_name", continentName); - } - } - case REGION_ISO_CODE -> { - // ISO 3166-2 code for country subdivisions. - // See iso.org/iso-3166-country-codes.html - String countryIso = country.getIsoCode(); - String subdivisionIso = subdivision.getIsoCode(); - if (countryIso != null && subdivisionIso != null) { - String regionIsoCode = countryIso + "-" + subdivisionIso; - geoData.put("region_iso_code", regionIsoCode); - } - } - case REGION_NAME -> { - String subdivisionName = subdivision.getName(); - if (subdivisionName != null) { - geoData.put("region_name", subdivisionName); - } - } - case CITY_NAME -> { - String cityName = city.getName(); - if (cityName != null) { - geoData.put("city_name", cityName); - } - } - case TIMEZONE -> { - String locationTimeZone = location.getTimeZone(); - if (locationTimeZone != null) { - geoData.put("timezone", locationTimeZone); - } - } - case LOCATION -> { - Double latitude = location.getLatitude(); - Double longitude = location.getLongitude(); - if (latitude != null && longitude != null) { - Map locationObject = new HashMap<>(); - locationObject.put("lat", latitude); - locationObject.put("lon", longitude); - geoData.put("location", locationObject); - } - } - } - } - return geoData; - } - - private Map retrieveCountryGeoData(GeoIpDatabase geoIpDatabase, InetAddress ipAddress) { - CountryResponse response = geoIpDatabase.getCountry(ipAddress); - if (response == null) { - return Map.of(); - } - Country country = response.getCountry(); - Continent continent = response.getContinent(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", NetworkAddress.format(ipAddress)); - case COUNTRY_ISO_CODE -> { - String countryIsoCode = country.getIsoCode(); - if (countryIsoCode != null) { - geoData.put("country_iso_code", countryIsoCode); - } - } - case COUNTRY_NAME -> { - String countryName = country.getName(); - if (countryName != null) { - geoData.put("country_name", countryName); - } - } - case CONTINENT_CODE -> { - String continentCode = continent.getCode(); - if (continentCode != null) { - geoData.put("continent_code", continentCode); - } - } - case CONTINENT_NAME -> { - String continentName = continent.getName(); - if (continentName != null) { - geoData.put("continent_name", continentName); - } - } - } - } - return geoData; - } - - private Map retrieveAsnGeoData(GeoIpDatabase geoIpDatabase, InetAddress ipAddress) { - AsnResponse response = geoIpDatabase.getAsn(ipAddress); - if (response == null) { - return Map.of(); - } - Long asn = response.getAutonomousSystemNumber(); - String organizationName = response.getAutonomousSystemOrganization(); - Network network = response.getNetwork(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", NetworkAddress.format(ipAddress)); - case ASN -> { - if (asn != null) { - geoData.put("asn", asn); - } - } - case ORGANIZATION_NAME -> { - if (organizationName != null) { - geoData.put("organization_name", organizationName); - } - } - case NETWORK -> { - if (network != null) { - geoData.put("network", network.toString()); - } - } - } - } - return geoData; - } - - private Map retrieveAnonymousIpGeoData(GeoIpDatabase geoIpDatabase, InetAddress ipAddress) { - AnonymousIpResponse response = geoIpDatabase.getAnonymousIp(ipAddress); - if (response == null) { - return Map.of(); - } - - boolean isHostingProvider = response.isHostingProvider(); - boolean isTorExitNode = response.isTorExitNode(); - boolean isAnonymousVpn = response.isAnonymousVpn(); - boolean isAnonymous = response.isAnonymous(); - boolean isPublicProxy = response.isPublicProxy(); - boolean isResidentialProxy = response.isResidentialProxy(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", NetworkAddress.format(ipAddress)); - case HOSTING_PROVIDER -> { - geoData.put("hosting_provider", isHostingProvider); - } - case TOR_EXIT_NODE -> { - geoData.put("tor_exit_node", isTorExitNode); - } - case ANONYMOUS_VPN -> { - geoData.put("anonymous_vpn", isAnonymousVpn); - } - case ANONYMOUS -> { - geoData.put("anonymous", isAnonymous); - } - case PUBLIC_PROXY -> { - geoData.put("public_proxy", isPublicProxy); - } - case RESIDENTIAL_PROXY -> { - geoData.put("residential_proxy", isResidentialProxy); - } - } - } - return geoData; - } - - private Map retrieveConnectionTypeGeoData(GeoIpDatabase geoIpDatabase, InetAddress ipAddress) { - ConnectionTypeResponse response = geoIpDatabase.getConnectionType(ipAddress); - if (response == null) { - return Map.of(); - } - - ConnectionType connectionType = response.getConnectionType(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", NetworkAddress.format(ipAddress)); - case CONNECTION_TYPE -> { - if (connectionType != null) { - geoData.put("connection_type", connectionType.toString()); - } - } - } - } - return geoData; - } - - private Map retrieveDomainGeoData(GeoIpDatabase geoIpDatabase, InetAddress ipAddress) { - DomainResponse response = geoIpDatabase.getDomain(ipAddress); - if (response == null) { - return Map.of(); - } - - String domain = response.getDomain(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", NetworkAddress.format(ipAddress)); - case DOMAIN -> { - if (domain != null) { - geoData.put("domain", domain); - } - } - } - } - return geoData; - } - - private Map retrieveEnterpriseGeoData(GeoIpDatabase geoIpDatabase, InetAddress ipAddress) { - EnterpriseResponse response = geoIpDatabase.getEnterprise(ipAddress); - if (response == null) { - return Map.of(); - } - - Country country = response.getCountry(); - City city = response.getCity(); - Location location = response.getLocation(); - Continent continent = response.getContinent(); - Subdivision subdivision = response.getMostSpecificSubdivision(); - - Long asn = response.getTraits().getAutonomousSystemNumber(); - String organizationName = response.getTraits().getAutonomousSystemOrganization(); - Network network = response.getTraits().getNetwork(); - - String isp = response.getTraits().getIsp(); - String ispOrganization = response.getTraits().getOrganization(); - String mobileCountryCode = response.getTraits().getMobileCountryCode(); - String mobileNetworkCode = response.getTraits().getMobileNetworkCode(); - - boolean isHostingProvider = response.getTraits().isHostingProvider(); - boolean isTorExitNode = response.getTraits().isTorExitNode(); - boolean isAnonymousVpn = response.getTraits().isAnonymousVpn(); - boolean isAnonymous = response.getTraits().isAnonymous(); - boolean isPublicProxy = response.getTraits().isPublicProxy(); - boolean isResidentialProxy = response.getTraits().isResidentialProxy(); - - String userType = response.getTraits().getUserType(); - - String domain = response.getTraits().getDomain(); - - ConnectionType connectionType = response.getTraits().getConnectionType(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", NetworkAddress.format(ipAddress)); - case COUNTRY_ISO_CODE -> { - String countryIsoCode = country.getIsoCode(); - if (countryIsoCode != null) { - geoData.put("country_iso_code", countryIsoCode); - } - } - case COUNTRY_NAME -> { - String countryName = country.getName(); - if (countryName != null) { - geoData.put("country_name", countryName); - } - } - case CONTINENT_CODE -> { - String continentCode = continent.getCode(); - if (continentCode != null) { - geoData.put("continent_code", continentCode); - } - } - case CONTINENT_NAME -> { - String continentName = continent.getName(); - if (continentName != null) { - geoData.put("continent_name", continentName); - } - } - case REGION_ISO_CODE -> { - // ISO 3166-2 code for country subdivisions. - // See iso.org/iso-3166-country-codes.html - String countryIso = country.getIsoCode(); - String subdivisionIso = subdivision.getIsoCode(); - if (countryIso != null && subdivisionIso != null) { - String regionIsoCode = countryIso + "-" + subdivisionIso; - geoData.put("region_iso_code", regionIsoCode); - } - } - case REGION_NAME -> { - String subdivisionName = subdivision.getName(); - if (subdivisionName != null) { - geoData.put("region_name", subdivisionName); - } - } - case CITY_NAME -> { - String cityName = city.getName(); - if (cityName != null) { - geoData.put("city_name", cityName); - } - } - case TIMEZONE -> { - String locationTimeZone = location.getTimeZone(); - if (locationTimeZone != null) { - geoData.put("timezone", locationTimeZone); - } - } - case LOCATION -> { - Double latitude = location.getLatitude(); - Double longitude = location.getLongitude(); - if (latitude != null && longitude != null) { - Map locationObject = new HashMap<>(); - locationObject.put("lat", latitude); - locationObject.put("lon", longitude); - geoData.put("location", locationObject); - } - } - case ASN -> { - if (asn != null) { - geoData.put("asn", asn); - } - } - case ORGANIZATION_NAME -> { - if (organizationName != null) { - geoData.put("organization_name", organizationName); - } - } - case NETWORK -> { - if (network != null) { - geoData.put("network", network.toString()); - } - } - case HOSTING_PROVIDER -> { - geoData.put("hosting_provider", isHostingProvider); - } - case TOR_EXIT_NODE -> { - geoData.put("tor_exit_node", isTorExitNode); - } - case ANONYMOUS_VPN -> { - geoData.put("anonymous_vpn", isAnonymousVpn); - } - case ANONYMOUS -> { - geoData.put("anonymous", isAnonymous); - } - case PUBLIC_PROXY -> { - geoData.put("public_proxy", isPublicProxy); - } - case RESIDENTIAL_PROXY -> { - geoData.put("residential_proxy", isResidentialProxy); - } - case DOMAIN -> { - if (domain != null) { - geoData.put("domain", domain); - } - } - case ISP -> { - if (isp != null) { - geoData.put("isp", isp); - } - } - case ISP_ORGANIZATION_NAME -> { - if (ispOrganization != null) { - geoData.put("isp_organization_name", ispOrganization); - } - } - case MOBILE_COUNTRY_CODE -> { - if (mobileCountryCode != null) { - geoData.put("mobile_country_code", mobileCountryCode); - } - } - case MOBILE_NETWORK_CODE -> { - if (mobileNetworkCode != null) { - geoData.put("mobile_network_code", mobileNetworkCode); - } - } - case USER_TYPE -> { - if (userType != null) { - geoData.put("user_type", userType); - } - } - case CONNECTION_TYPE -> { - if (connectionType != null) { - geoData.put("connection_type", connectionType.toString()); - } - } - } - } - return geoData; - } - - private Map retrieveIspGeoData(GeoIpDatabase geoIpDatabase, InetAddress ipAddress) { - IspResponse response = geoIpDatabase.getIsp(ipAddress); - if (response == null) { - return Map.of(); - } - - String isp = response.getIsp(); - String ispOrganization = response.getOrganization(); - String mobileNetworkCode = response.getMobileNetworkCode(); - String mobileCountryCode = response.getMobileCountryCode(); - Long asn = response.getAutonomousSystemNumber(); - String organizationName = response.getAutonomousSystemOrganization(); - Network network = response.getNetwork(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", NetworkAddress.format(ipAddress)); - case ASN -> { - if (asn != null) { - geoData.put("asn", asn); - } - } - case ORGANIZATION_NAME -> { - if (organizationName != null) { - geoData.put("organization_name", organizationName); - } - } - case NETWORK -> { - if (network != null) { - geoData.put("network", network.toString()); - } - } - case ISP -> { - if (isp != null) { - geoData.put("isp", isp); - } - } - case ISP_ORGANIZATION_NAME -> { - if (ispOrganization != null) { - geoData.put("isp_organization_name", ispOrganization); - } - } - case MOBILE_COUNTRY_CODE -> { - if (mobileCountryCode != null) { - geoData.put("mobile_country_code", mobileCountryCode); - } - } - case MOBILE_NETWORK_CODE -> { - if (mobileNetworkCode != null) { - geoData.put("mobile_network_code", mobileNetworkCode); - } - } - } - } - return geoData; + return ipDataLookup.getProperties(); } /** - * Retrieves and verifies a {@link GeoIpDatabase} instance for each execution of the {@link GeoIpProcessor}. Guards against missing + * Retrieves and verifies a {@link IpDatabase} instance for each execution of the {@link GeoIpProcessor}. Guards against missing * custom databases, and ensures that database instances are of the proper type before use. */ - public static final class DatabaseVerifyingSupplier implements CheckedSupplier { - private final GeoIpDatabaseProvider geoIpDatabaseProvider; + public static final class DatabaseVerifyingSupplier implements CheckedSupplier { + private final IpDatabaseProvider ipDatabaseProvider; private final String databaseFile; private final String databaseType; - public DatabaseVerifyingSupplier(GeoIpDatabaseProvider geoIpDatabaseProvider, String databaseFile, String databaseType) { - this.geoIpDatabaseProvider = geoIpDatabaseProvider; + public DatabaseVerifyingSupplier(IpDatabaseProvider ipDatabaseProvider, String databaseFile, String databaseType) { + this.ipDatabaseProvider = ipDatabaseProvider; this.databaseFile = databaseFile; this.databaseType = databaseType; } @Override - public GeoIpDatabase get() throws IOException { - GeoIpDatabase loader = geoIpDatabaseProvider.getDatabase(databaseFile); + public IpDatabase get() throws IOException { + IpDatabase loader = ipDatabaseProvider.getDatabase(databaseFile); if (loader == null) { return null; } if (Assertions.ENABLED) { - // Only check whether the suffix has changed and not the entire database type. - // To sanity check whether a city db isn't overwriting with a country or asn db. - // For example overwriting a geoip lite city db with geoip city db is a valid change, but the db type is slightly different, - // by checking just the suffix this assertion doesn't fail. - String expectedSuffix = databaseType.substring(databaseType.lastIndexOf('-')); - assert loader.getDatabaseType().endsWith(expectedSuffix) - : "database type [" + loader.getDatabaseType() + "] doesn't match with expected suffix [" + expectedSuffix + "]"; + // Note that the expected suffix might be null for providers that aren't amenable to using dashes as separator for + // determining the database type. + int last = databaseType.lastIndexOf('-'); + final String expectedSuffix = last == -1 ? null : databaseType.substring(last); + + // If the entire database type matches, then that's a match. Otherwise, if there's a suffix to compare on, then + // check whether the suffix has changed (not the entire database type). + // This is to sanity check, for example, that a city db isn't overwritten with a country or asn db. + // But there are permissible overwrites that make sense, for example overwriting a geolite city db with a geoip city db + // is a valid change, but the db type is slightly different -- by checking just the suffix this assertion won't fail. + final String loaderType = loader.getDatabaseType(); + assert loaderType.equals(databaseType) || expectedSuffix == null || loaderType.endsWith(expectedSuffix) + : "database type [" + loaderType + "] doesn't match with expected suffix [" + expectedSuffix + "]"; } return loader; } @@ -715,10 +216,12 @@ public GeoIpDatabase get() throws IOException { public static final class Factory implements Processor.Factory { - private final GeoIpDatabaseProvider geoIpDatabaseProvider; + private final String type; // currently always just "geoip" + private final IpDatabaseProvider ipDatabaseProvider; - public Factory(GeoIpDatabaseProvider geoIpDatabaseProvider) { - this.geoIpDatabaseProvider = geoIpDatabaseProvider; + public Factory(String type, IpDatabaseProvider ipDatabaseProvider) { + this.type = type; + this.ipDatabaseProvider = ipDatabaseProvider; } @Override @@ -728,16 +231,15 @@ public Processor create( final String description, final Map config ) throws IOException { - String ipField = readStringProperty(TYPE, processorTag, config, "field"); - String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "geoip"); - String databaseFile = readStringProperty(TYPE, processorTag, config, "database_file", "GeoLite2-City.mmdb"); - List propertyNames = readOptionalList(TYPE, processorTag, config, "properties"); - boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); - boolean firstOnly = readBooleanProperty(TYPE, processorTag, config, "first_only", true); + String ipField = readStringProperty(type, processorTag, config, "field"); + String targetField = readStringProperty(type, processorTag, config, "target_field", type); + String databaseFile = readStringProperty(type, processorTag, config, "database_file", "GeoLite2-City.mmdb"); + List propertyNames = readOptionalList(type, processorTag, config, "properties"); + boolean ignoreMissing = readBooleanProperty(type, processorTag, config, "ignore_missing", false); + boolean firstOnly = readBooleanProperty(type, processorTag, config, "first_only", true); - // Validating the download_database_on_pipeline_creation even if the result - // is not used directly by the factory. - downloadDatabaseOnPipelineCreation(config, processorTag); + // validate (and consume) the download_database_on_pipeline_creation property even though the result is not used by the factory + readBooleanProperty(type, processorTag, config, "download_database_on_pipeline_creation", true); // noop, should be removed in 9.0 Object value = config.remove("fallback_to_default_databases"); @@ -745,77 +247,109 @@ public Processor create( deprecationLogger.warn(DeprecationCategory.OTHER, "default_databases_message", DEFAULT_DATABASES_DEPRECATION_MESSAGE); } - GeoIpDatabase geoIpDatabase = geoIpDatabaseProvider.getDatabase(databaseFile); - if (geoIpDatabase == null) { - // It's possible that the database could be downloaded via the GeoipDownloader process and could become available - // at a later moment, so a processor impl is returned that tags documents instead. If a database cannot be sourced then the - // processor will continue to tag documents with a warning until it is remediated by providing a database or changing the - // pipeline. - return new DatabaseUnavailableProcessor(processorTag, description, databaseFile); - } - final String databaseType; - try { - databaseType = geoIpDatabase.getDatabaseType(); - } finally { - geoIpDatabase.release(); + try (IpDatabase ipDatabase = ipDatabaseProvider.getDatabase(databaseFile)) { + if (ipDatabase == null) { + // It's possible that the database could be downloaded via the GeoipDownloader process and could become available + // at a later moment, so a processor impl is returned that tags documents instead. If a database cannot be sourced + // then the processor will continue to tag documents with a warning until it is remediated by providing a database + // or changing the pipeline. + return new DatabaseUnavailableProcessor(type, processorTag, description, databaseFile); + } + databaseType = ipDatabase.getDatabaseType(); } - final Database database; + final IpDataLookupFactory factory; try { - database = Database.getDatabase(databaseType, databaseFile); + factory = IpDataLookupFactories.get(databaseType, databaseFile); } catch (IllegalArgumentException e) { - throw newConfigurationException(TYPE, processorTag, "database_file", e.getMessage()); + throw newConfigurationException(type, processorTag, "database_file", e.getMessage()); + } + + // the "geoip" processor type does additional validation of the database_type + if (GEOIP_TYPE.equals(type)) { + // type sniffing is done with the lowercased type + final String lowerCaseDatabaseType = databaseType.toLowerCase(Locale.ROOT); + + // start with a strict positive rejection check -- as we support addition database providers, + // we should expand these checks when possible + if (lowerCaseDatabaseType.startsWith(IpinfoIpDataLookups.IPINFO_PREFIX)) { + throw newConfigurationException( + type, + processorTag, + "database_file", + Strings.format("Unsupported database type [%s] for file [%s]", databaseType, databaseFile) + ); + } + + // end with a lax negative rejection check -- if we aren't *certain* it's a maxmind database, then we'll warn -- + // it's possible for example that somebody cooked up a custom database of their own that happened to work with + // our preexisting code, they should migrate to the new processor, but we're not going to break them right now + if (lowerCaseDatabaseType.startsWith(MaxmindIpDataLookups.GEOIP2_PREFIX) == false + && lowerCaseDatabaseType.startsWith(MaxmindIpDataLookups.GEOLITE2_PREFIX) == false) { + deprecationLogger.warn( + DeprecationCategory.OTHER, + "unsupported_database_type", + UNSUPPORTED_DATABASE_DEPRECATION_MESSAGE, + databaseType + ); + } } - final Set properties; + final IpDataLookup ipDataLookup; try { - properties = database.parseProperties(propertyNames); + ipDataLookup = factory.create(propertyNames); } catch (IllegalArgumentException e) { - throw newConfigurationException(TYPE, processorTag, "properties", e.getMessage()); + throw newConfigurationException(type, processorTag, "properties", e.getMessage()); } + return new GeoIpProcessor( + type, processorTag, description, ipField, - new DatabaseVerifyingSupplier(geoIpDatabaseProvider, databaseFile, databaseType), - () -> geoIpDatabaseProvider.isValid(databaseFile), + new DatabaseVerifyingSupplier(ipDatabaseProvider, databaseFile, databaseType), + () -> ipDatabaseProvider.isValid(databaseFile), targetField, - properties, + ipDataLookup, ignoreMissing, firstOnly, databaseFile ); } + /** + * Get the value of the "download_database_on_pipeline_creation" property from a processor's config map. + *

    + * As with the actual property definition, the default value of the property is 'true'. Unlike the actual + * property definition, this method doesn't consume (that is, config.remove) the property from + * the config map. + */ public static boolean downloadDatabaseOnPipelineCreation(Map config) { - return downloadDatabaseOnPipelineCreation(config, null); + return (boolean) config.getOrDefault("download_database_on_pipeline_creation", true); } - - public static boolean downloadDatabaseOnPipelineCreation(Map config, String processorTag) { - return readBooleanProperty(GeoIpProcessor.TYPE, processorTag, config, "download_database_on_pipeline_creation", true); - } - } static class DatabaseUnavailableProcessor extends AbstractProcessor { + private final String type; private final String databaseName; - DatabaseUnavailableProcessor(String tag, String description, String databaseName) { + DatabaseUnavailableProcessor(String type, String tag, String description, String databaseName) { super(tag, description); + this.type = type; this.databaseName = databaseName; } @Override public IngestDocument execute(IngestDocument ingestDocument) throws Exception { - tag(ingestDocument, databaseName); + tag(ingestDocument, this.type, databaseName); return ingestDocument; } @Override public String getType() { - return TYPE; + return type; } public String getDatabaseName() { @@ -823,7 +357,7 @@ public String getDatabaseName() { } } - private static void tag(IngestDocument ingestDocument, String databaseName) { - ingestDocument.appendFieldValue("tags", "_geoip_database_unavailable_" + databaseName, true); + private static void tag(IngestDocument ingestDocument, String type, String databaseName) { + ingestDocument.appendFieldValue("tags", "_" + type + "_database_unavailable_" + databaseName, true); } } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskParams.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskParams.java index 276f67f6ee736..128f41c1eecca 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskParams.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskParams.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java index 56f96786d9b7f..96525d427d3e8 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; @@ -40,11 +41,10 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; -class GeoIpTaskState implements PersistentTaskState, VersionedNamedWriteable { +public class GeoIpTaskState implements PersistentTaskState, VersionedNamedWriteable { private static boolean includeSha256(TransportVersion version) { - return version.isPatchFrom(TransportVersions.ENTERPRISE_GEOIP_DOWNLOADER_BACKPORT_8_15) - || version.onOrAfter(TransportVersions.ENTERPRISE_GEOIP_DOWNLOADER); + return version.onOrAfter(TransportVersions.V_8_15_0); } private static final ParseField DATABASES = new ParseField("databases"); @@ -149,7 +149,7 @@ public void writeTo(StreamOutput out) throws IOException { }); } - record Metadata(long lastUpdate, int firstChunk, int lastChunk, String md5, long lastCheck, @Nullable String sha256) + public record Metadata(long lastUpdate, int firstChunk, int lastChunk, String md5, long lastCheck, @Nullable String sha256) implements ToXContentObject { @@ -197,7 +197,7 @@ public static Metadata fromXContent(XContentParser parser) { } } - Metadata { + public Metadata { Objects.requireNonNull(md5); } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/HttpClient.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/HttpClient.java index 2f6bd6ef20fd0..0123c5c3ba582 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/HttpClient.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/HttpClient.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java index f5ac755b6b980..a50fe7dee9008 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; @@ -14,10 +15,9 @@ import org.elasticsearch.cluster.DiffableUtils; import org.elasticsearch.cluster.NamedDiff; import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.ingest.geoip.direct.DatabaseConfigurationMetadata; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ParseField; @@ -69,7 +69,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ENTERPRISE_GEOIP_DOWNLOADER; + return TransportVersions.V_8_16_0; } public Map getDatabases() { @@ -90,8 +90,8 @@ public static IngestGeoIpMetadata fromXContent(XContentParser parser) throws IOE } @Override - public Iterator toXContentChunked(ToXContent.Params ignored) { - return Iterators.concat(ChunkedToXContentHelper.xContentValuesMap(DATABASES_FIELD.getPreferredName(), databases)); + public Iterator toXContentChunked(ToXContent.Params params) { + return ChunkedToXContent.builder(params).xContentObjectFields(DATABASES_FIELD.getPreferredName(), databases); } @Override @@ -138,7 +138,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ENTERPRISE_GEOIP_DOWNLOADER; + return TransportVersions.V_8_16_0; } } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java index e606688ad60a0..3107f0bed55e8 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; @@ -30,6 +31,7 @@ import org.elasticsearch.ingest.EnterpriseGeoIpTask.EnterpriseGeoIpTaskParams; import org.elasticsearch.ingest.IngestService; import org.elasticsearch.ingest.Processor; +import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration; import org.elasticsearch.ingest.geoip.direct.DeleteDatabaseConfigurationAction; import org.elasticsearch.ingest.geoip.direct.GetDatabaseConfigurationAction; import org.elasticsearch.ingest.geoip.direct.PutDatabaseConfigurationAction; @@ -69,6 +71,7 @@ import java.util.function.Predicate; import java.util.function.Supplier; +import static java.util.Map.entry; import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; import static org.elasticsearch.ingest.EnterpriseGeoIpTask.ENTERPRISE_GEOIP_DOWNLOADER; import static org.elasticsearch.ingest.IngestService.INGEST_ORIGIN; @@ -109,7 +112,8 @@ public List> getSettings() { GeoIpDownloaderTaskExecutor.ENABLED_SETTING, GeoIpDownloader.ENDPOINT_SETTING, GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING, - EnterpriseGeoIpDownloaderTaskExecutor.MAXMIND_LICENSE_KEY_SETTING + EnterpriseGeoIpDownloaderTaskExecutor.MAXMIND_LICENSE_KEY_SETTING, + EnterpriseGeoIpDownloaderTaskExecutor.IPINFO_TOKEN_SETTING ); } @@ -127,7 +131,10 @@ public Map getProcessors(Processor.Parameters paramet parameters.ingestService.getClusterService() ); databaseRegistry.set(registry); - return Map.of(GeoIpProcessor.TYPE, new GeoIpProcessor.Factory(registry)); + return Map.ofEntries( + entry(GeoIpProcessor.GEOIP_TYPE, new GeoIpProcessor.Factory(GeoIpProcessor.GEOIP_TYPE, registry)), + entry(GeoIpProcessor.IP_LOCATION_TYPE, new GeoIpProcessor.Factory(GeoIpProcessor.IP_LOCATION_TYPE, registry)) + ); } @Override @@ -160,7 +167,7 @@ public Collection createComponents(PluginServices services) { @Override public void close() throws IOException { - databaseRegistry.get().close(); + databaseRegistry.get().shutdown(); } @Override @@ -231,7 +238,27 @@ public List getNamedWriteables() { new NamedWriteableRegistry.Entry(PersistentTaskParams.class, GEOIP_DOWNLOADER, GeoIpTaskParams::new), new NamedWriteableRegistry.Entry(PersistentTaskState.class, ENTERPRISE_GEOIP_DOWNLOADER, EnterpriseGeoIpTaskState::new), new NamedWriteableRegistry.Entry(PersistentTaskParams.class, ENTERPRISE_GEOIP_DOWNLOADER, EnterpriseGeoIpTaskParams::new), - new NamedWriteableRegistry.Entry(Task.Status.class, GEOIP_DOWNLOADER, GeoIpDownloaderStats::new) + new NamedWriteableRegistry.Entry(Task.Status.class, GEOIP_DOWNLOADER, GeoIpDownloaderStats::new), + new NamedWriteableRegistry.Entry( + DatabaseConfiguration.Provider.class, + DatabaseConfiguration.Maxmind.NAME, + DatabaseConfiguration.Maxmind::new + ), + new NamedWriteableRegistry.Entry( + DatabaseConfiguration.Provider.class, + DatabaseConfiguration.Ipinfo.NAME, + DatabaseConfiguration.Ipinfo::new + ), + new NamedWriteableRegistry.Entry( + DatabaseConfiguration.Provider.class, + DatabaseConfiguration.Local.NAME, + DatabaseConfiguration.Local::new + ), + new NamedWriteableRegistry.Entry( + DatabaseConfiguration.Provider.class, + DatabaseConfiguration.Web.NAME, + DatabaseConfiguration.Web::new + ) ); } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookup.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookup.java new file mode 100644 index 0000000000000..7442c8e930886 --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookup.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import java.io.IOException; +import java.util.Map; +import java.util.Set; + +interface IpDataLookup { + /** + * Gets data from the provided {@code ipDatabase} for the provided {@code ip} + * + * @param ipDatabase the database from which to lookup a result + * @param ip the ip address + * @return a map of data corresponding to the configured properties + * @throws IOException if the implementation encounters any problem while retrieving the response + */ + Map getData(IpDatabase ipDatabase, String ip) throws IOException; + + /** + * @return the set of properties this lookup will provide + */ + Set getProperties(); +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java new file mode 100644 index 0000000000000..e879f0e0e3514 --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.core.Nullable; + +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.function.Function; + +import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.IPINFO_PREFIX; +import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.getIpinfoDatabase; +import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.getIpinfoLookup; +import static org.elasticsearch.ingest.geoip.MaxmindIpDataLookups.getMaxmindDatabase; +import static org.elasticsearch.ingest.geoip.MaxmindIpDataLookups.getMaxmindLookup; + +final class IpDataLookupFactories { + + private IpDataLookupFactories() { + // utility class + } + + interface IpDataLookupFactory { + IpDataLookup create(List properties); + } + + /** + * Parses the passed-in databaseType and return the Database instance that is + * associated with that databaseType. + * + * @param databaseType the database type String from the metadata of the database file + * @return the Database instance that is associated with the databaseType (or null) + */ + @Nullable + static Database getDatabase(final String databaseType) { + Database database = null; + + if (Strings.hasText(databaseType)) { + final String databaseTypeLowerCase = databaseType.toLowerCase(Locale.ROOT); + if (databaseTypeLowerCase.startsWith(IPINFO_PREFIX)) { + database = getIpinfoDatabase(databaseTypeLowerCase); // all lower case! + } else { + // for historical reasons, fall back to assuming maxmind-like type parsing + database = getMaxmindDatabase(databaseType); + } + } + + return database; + } + + static IpDataLookupFactory get(final String databaseType, final String databaseFile) { + final Database database = getDatabase(databaseType); + if (database == null) { + throw new IllegalArgumentException("Unsupported database type [" + databaseType + "] for file [" + databaseFile + "]"); + } + + final Function, IpDataLookup> factoryMethod; + final String databaseTypeLowerCase = databaseType.toLowerCase(Locale.ROOT); + if (databaseTypeLowerCase.startsWith(IPINFO_PREFIX)) { + factoryMethod = getIpinfoLookup(database); + } else { + // for historical reasons, fall back to assuming maxmind-like types + factoryMethod = getMaxmindLookup(database); + } + + if (factoryMethod == null) { + throw new IllegalArgumentException("Unsupported database type [" + databaseType + "] for file [" + databaseFile + "]"); + } + + return (properties) -> factoryMethod.apply(database.parseProperties(properties)); + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDatabase.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDatabase.java new file mode 100644 index 0000000000000..db1ffc1c682b8 --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDatabase.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import com.maxmind.db.Reader; + +import org.elasticsearch.common.CheckedBiFunction; +import org.elasticsearch.core.Nullable; + +import java.io.IOException; + +/** + * Provides a uniform interface for interacting with various ip databases. + */ +public interface IpDatabase extends AutoCloseable { + + /** + * @return the database type as it is detailed in the database file metadata + * @throws IOException if the database file could not be read or the data could not be accessed + */ + String getDatabaseType() throws IOException; + + /** + * Returns a response from this database's reader for the given IP address. + * + * @param ipAddress the address to lookup + * @param responseProvider a method for extracting a response from a {@link Reader}, usually this will be a method reference + * @return a possibly-null response + * @param the type of response that will be returned + */ + @Nullable + RESPONSE getResponse(String ipAddress, CheckedBiFunction responseProvider); + + /** + * Releases the current database object. Called after processing a single document. Databases should be closed or returned to a + * resource pool. No further interactions should be expected. + * + * @throws IOException if the implementation encounters any problem while cleaning up + */ + @Override + void close() throws IOException; +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDatabaseProvider.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDatabaseProvider.java new file mode 100644 index 0000000000000..9438bf74f8c12 --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDatabaseProvider.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +/** + * Provides construction and initialization logic for {@link IpDatabase} instances. + */ +public interface IpDatabaseProvider { + + /** + * Determines if the given database name corresponds to an expired database. Expired databases will not be loaded. + *

    + * Verifying database expiration is left to each provider implementation to determine. A return value of false does not + * preclude the possibility of a provider returning true in the future. + * + * @param name the name of the database to provide. + * @return false IFF the requested database file is expired, + * true for all other cases (including unknown file name, file missing, wrong database type, etc). + */ + Boolean isValid(String name); + + /** + * @param name the name of the database to provide. + * @return a ready-to-use database instance, or null if no database could be loaded. + */ + IpDatabase getDatabase(String name); +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java new file mode 100644 index 0000000000000..8ce2424844d9d --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java @@ -0,0 +1,512 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import com.maxmind.db.DatabaseRecord; +import com.maxmind.db.MaxMindDbConstructor; +import com.maxmind.db.MaxMindDbParameter; +import com.maxmind.db.Reader; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.core.Nullable; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * A collection of {@link IpDataLookup} implementations for IPinfo databases + */ +final class IpinfoIpDataLookups { + + private IpinfoIpDataLookups() { + // utility class + } + + private static final Logger logger = LogManager.getLogger(IpinfoIpDataLookups.class); + + // the actual prefix from the metadata is cased like the literal string, and + // prefix dispatch and checks case-insensitive, so that works out nicely + static final String IPINFO_PREFIX = "ipinfo"; + + private static final Set IPINFO_TYPE_STOP_WORDS = Set.of( + "ipinfo", + "extended", + "free", + "generic", + "ip", + "sample", + "standard", + "mmdb" + ); + + /** + * Cleans up the database_type String from an ipinfo database by splitting on punctuation, removing stop words, and then joining + * with an underscore. + *

    + * e.g. "ipinfo free_foo_sample.mmdb" -> "foo" + * + * @param type the database_type from an ipinfo database + * @return a cleaned up database_type string + */ + // n.b. this is just based on observation of the types from a survey of such databases -- it's like browser user agent sniffing, + // there aren't necessarily any amazing guarantees about this behavior + static String ipinfoTypeCleanup(String type) { + List parts = Arrays.asList(type.split("[ _.]")); + return parts.stream().filter((s) -> IPINFO_TYPE_STOP_WORDS.contains(s) == false).collect(Collectors.joining("_")); + } + + @Nullable + static Database getIpinfoDatabase(final String databaseType) { + // for ipinfo the database selection is more along the lines of user-agent sniffing than + // string-based dispatch. the specific database_type strings could change in the future, + // hence the somewhat loose nature of this checking. + + final String cleanedType = ipinfoTypeCleanup(databaseType); + + // early detection on any of the 'extended' types + if (databaseType.contains("extended")) { + // which are not currently supported + logger.trace("returning null for unsupported database_type [{}]", databaseType); + return null; + } + + // early detection on 'country_asn' so the 'country' and 'asn' checks don't get faked out + if (cleanedType.contains("country_asn")) { + // but it's not currently supported + logger.trace("returning null for unsupported database_type [{}]", databaseType); + return null; + } + + if (cleanedType.contains("asn")) { + return Database.AsnV2; + } else if (cleanedType.contains("country")) { + return Database.CountryV2; + } else if (cleanedType.contains("location")) { // note: catches 'location' and 'geolocation' ;) + return Database.CityV2; + } else if (cleanedType.contains("privacy")) { + return Database.PrivacyDetection; + } else { + // no match was found + logger.trace("returning null for unsupported database_type [{}]", databaseType); + return null; + } + } + + @Nullable + static Function, IpDataLookup> getIpinfoLookup(final Database database) { + return switch (database) { + case AsnV2 -> IpinfoIpDataLookups.Asn::new; + case CountryV2 -> IpinfoIpDataLookups.Country::new; + case CityV2 -> IpinfoIpDataLookups.Geolocation::new; + case PrivacyDetection -> IpinfoIpDataLookups.PrivacyDetection::new; + default -> null; + }; + } + + /** + * Lax-ly parses a string that (ideally) looks like 'AS123' into a Long like 123L (or null, if such parsing isn't possible). + * @param asn a potentially empty (or null) ASN string that is expected to contain 'AS' and then a parsable long + * @return the parsed asn + */ + static Long parseAsn(final String asn) { + if (asn == null || Strings.hasText(asn) == false) { + return null; + } else { + String stripped = asn.toUpperCase(Locale.ROOT).replaceAll("AS", "").trim(); + try { + return Long.parseLong(stripped); + } catch (NumberFormatException e) { + logger.trace("Unable to parse non-compliant ASN string [{}]", asn); + return null; + } + } + } + + /** + * Lax-ly parses a string that contains a boolean into a Boolean (or null, if such parsing isn't possible). + * @param bool a potentially empty (or null) string that is expected to contain a parsable boolean + * @return the parsed boolean + */ + static Boolean parseBoolean(final String bool) { + if (bool == null) { + return null; + } else { + String trimmed = bool.toLowerCase(Locale.ROOT).trim(); + if ("true".equals(trimmed)) { + return true; + } else if ("false".equals(trimmed)) { + // "false" can represent false -- this an expected future enhancement in how the database represents booleans + return false; + } else if (trimmed.isEmpty()) { + // empty string can represent false -- this is how the database currently represents 'false' values + return false; + } else { + logger.trace("Unable to parse non-compliant boolean string [{}]", bool); + return null; + } + } + } + + /** + * Lax-ly parses a string that contains a double into a Double (or null, if such parsing isn't possible). + * @param latlon a potentially empty (or null) string that is expected to contain a parsable double + * @return the parsed double + */ + static Double parseLocationDouble(final String latlon) { + if (latlon == null || Strings.hasText(latlon) == false) { + return null; + } else { + String stripped = latlon.trim(); + try { + return Double.parseDouble(stripped); + } catch (NumberFormatException e) { + logger.trace("Unable to parse non-compliant location string [{}]", latlon); + return null; + } + } + } + + public record AsnResult( + Long asn, + @Nullable String country, // not present in the free asn database + String domain, + String name, + @Nullable String type // not present in the free asn database + ) { + @SuppressWarnings("checkstyle:RedundantModifier") + @MaxMindDbConstructor + public AsnResult( + @MaxMindDbParameter(name = "asn") String asn, + @Nullable @MaxMindDbParameter(name = "country") String country, + @MaxMindDbParameter(name = "domain") String domain, + @MaxMindDbParameter(name = "name") String name, + @Nullable @MaxMindDbParameter(name = "type") String type + ) { + this(parseAsn(asn), country, domain, name, type); + } + } + + public record CountryResult( + @MaxMindDbParameter(name = "continent") String continent, + @MaxMindDbParameter(name = "continent_name") String continentName, + @MaxMindDbParameter(name = "country") String country, + @MaxMindDbParameter(name = "country_name") String countryName + ) { + @MaxMindDbConstructor + public CountryResult {} + } + + public record GeolocationResult( + String city, + String country, + Double lat, + Double lng, + String postalCode, + String region, + String timezone + ) { + @SuppressWarnings("checkstyle:RedundantModifier") + @MaxMindDbConstructor + public GeolocationResult( + @MaxMindDbParameter(name = "city") String city, + @MaxMindDbParameter(name = "country") String country, + // @MaxMindDbParameter(name = "geoname_id") String geonameId, // for now we're not exposing this + @MaxMindDbParameter(name = "lat") String lat, + @MaxMindDbParameter(name = "lng") String lng, + @MaxMindDbParameter(name = "postal_code") String postalCode, + @MaxMindDbParameter(name = "region") String region, + // @MaxMindDbParameter(name = "region_code") String regionCode, // for now we're not exposing this + @MaxMindDbParameter(name = "timezone") String timezone + ) { + this(city, country, parseLocationDouble(lat), parseLocationDouble(lng), postalCode, region, timezone); + } + } + + public record PrivacyDetectionResult(Boolean hosting, Boolean proxy, Boolean relay, String service, Boolean tor, Boolean vpn) { + @SuppressWarnings("checkstyle:RedundantModifier") + @MaxMindDbConstructor + public PrivacyDetectionResult( + @MaxMindDbParameter(name = "hosting") String hosting, + // @MaxMindDbParameter(name = "network") String network, // for now we're not exposing this + @MaxMindDbParameter(name = "proxy") String proxy, + @MaxMindDbParameter(name = "relay") String relay, + @MaxMindDbParameter(name = "service") String service, // n.b. this remains a string, the rest are parsed as booleans + @MaxMindDbParameter(name = "tor") String tor, + @MaxMindDbParameter(name = "vpn") String vpn + ) { + this(parseBoolean(hosting), parseBoolean(proxy), parseBoolean(relay), service, parseBoolean(tor), parseBoolean(vpn)); + } + } + + static class Asn extends AbstractBase { + Asn(Set properties) { + super(properties, AsnResult.class); + } + + @Override + protected Map transform(final Result result) { + AsnResult response = result.result; + Long asn = response.asn; + String organizationName = response.name; + String network = result.network; + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", result.ip); + case ASN -> { + if (asn != null) { + data.put("asn", asn); + } + } + case ORGANIZATION_NAME -> { + if (organizationName != null) { + data.put("organization_name", organizationName); + } + } + case NETWORK -> { + if (network != null) { + data.put("network", network); + } + } + case COUNTRY_ISO_CODE -> { + if (response.country != null) { + data.put("country_iso_code", response.country); + } + } + case DOMAIN -> { + if (response.domain != null) { + data.put("domain", response.domain); + } + } + case TYPE -> { + if (response.type != null) { + data.put("type", response.type); + } + } + } + } + return data; + } + } + + static class Country extends AbstractBase { + Country(Set properties) { + super(properties, CountryResult.class); + } + + @Override + protected Map transform(final Result result) { + CountryResult response = result.result; + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", result.ip); + case COUNTRY_ISO_CODE -> { + String countryIsoCode = response.country; + if (countryIsoCode != null) { + data.put("country_iso_code", countryIsoCode); + } + } + case COUNTRY_NAME -> { + String countryName = response.countryName; + if (countryName != null) { + data.put("country_name", countryName); + } + } + case CONTINENT_CODE -> { + String continentCode = response.continent; + if (continentCode != null) { + data.put("continent_code", continentCode); + } + } + case CONTINENT_NAME -> { + String continentName = response.continentName; + if (continentName != null) { + data.put("continent_name", continentName); + } + } + } + } + return data; + } + } + + static class Geolocation extends AbstractBase { + Geolocation(final Set properties) { + super(properties, GeolocationResult.class); + } + + @Override + protected Map transform(final Result result) { + GeolocationResult response = result.result; + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", result.ip); + case COUNTRY_ISO_CODE -> { + String countryIsoCode = response.country; + if (countryIsoCode != null) { + data.put("country_iso_code", countryIsoCode); + } + } + case REGION_NAME -> { + String subdivisionName = response.region; + if (subdivisionName != null) { + data.put("region_name", subdivisionName); + } + } + case CITY_NAME -> { + String cityName = response.city; + if (cityName != null) { + data.put("city_name", cityName); + } + } + case TIMEZONE -> { + String locationTimeZone = response.timezone; + if (locationTimeZone != null) { + data.put("timezone", locationTimeZone); + } + } + case POSTAL_CODE -> { + String postalCode = response.postalCode; + if (postalCode != null) { + data.put("postal_code", postalCode); + } + } + case LOCATION -> { + Double latitude = response.lat; + Double longitude = response.lng; + if (latitude != null && longitude != null) { + Map locationObject = new HashMap<>(); + locationObject.put("lat", latitude); + locationObject.put("lon", longitude); + data.put("location", locationObject); + } + } + } + } + return data; + } + } + + static class PrivacyDetection extends AbstractBase { + PrivacyDetection(Set properties) { + super(properties, PrivacyDetectionResult.class); + } + + @Override + protected Map transform(final Result result) { + PrivacyDetectionResult response = result.result; + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", result.ip); + case HOSTING -> { + if (response.hosting != null) { + data.put("hosting", response.hosting); + } + } + case TOR -> { + if (response.tor != null) { + data.put("tor", response.tor); + } + } + case PROXY -> { + if (response.proxy != null) { + data.put("proxy", response.proxy); + } + } + case RELAY -> { + if (response.relay != null) { + data.put("relay", response.relay); + } + } + case VPN -> { + if (response.vpn != null) { + data.put("vpn", response.vpn); + } + } + case SERVICE -> { + if (Strings.hasText(response.service)) { + data.put("service", response.service); + } + } + } + } + return data; + } + } + + /** + * Just a little record holder -- there's the data that we receive via the binding to our record objects from the Reader via the + * getRecord call, but then we also need to capture the passed-in ip address that came from the caller as well as the network for + * the returned DatabaseRecord from the Reader. + */ + public record Result(T result, String ip, String network) {} + + /** + * The {@link IpinfoIpDataLookups.AbstractBase} is an abstract base implementation of {@link IpDataLookup} that + * provides common functionality for getting a {@link IpinfoIpDataLookups.Result} that wraps a record from a {@link IpDatabase}. + * + * @param the record type that will be wrapped and returned + */ + private abstract static class AbstractBase implements IpDataLookup { + + protected final Set properties; + protected final Class clazz; + + AbstractBase(final Set properties, final Class clazz) { + this.properties = Set.copyOf(properties); + this.clazz = clazz; + } + + @Override + public Set getProperties() { + return this.properties; + } + + @Override + public final Map getData(final IpDatabase ipDatabase, final String ipAddress) { + final Result response = ipDatabase.getResponse(ipAddress, this::lookup); + return (response == null || response.result == null) ? Map.of() : transform(response); + } + + @Nullable + private Result lookup(final Reader reader, final String ipAddress) throws IOException { + final InetAddress ip = InetAddresses.forString(ipAddress); + final DatabaseRecord record = reader.getRecord(ip, clazz); + final RESPONSE data = record.getData(); + return (data == null) ? null : new Result<>(data, NetworkAddress.format(ip), record.getNetwork().toString()); + } + + /** + * Extract the configured properties from the retrieved response + * @param response the non-null response that was retrieved + * @return a mapping of properties for the ip from the response + */ + protected abstract Map transform(Result response); + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MMDBUtil.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MMDBUtil.java new file mode 100644 index 0000000000000..3c21296ff294a --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MMDBUtil.java @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.zip.GZIPInputStream; +import java.util.zip.ZipException; + +public final class MMDBUtil { + + private MMDBUtil() { + // utility class + } + + private static final byte[] DATABASE_TYPE_MARKER = "database_type".getBytes(StandardCharsets.UTF_8); + + // note: technically the metadata can be up to 128k long, but we only handle it correctly as long as it's less than + // or equal to this buffer size. in practice, that seems to be plenty for ordinary files. + private static final int BUFFER_SIZE = 2048; + + /** + * Read the database type from the database. We do this manually instead of relying on the built-in mechanism to avoid reading the + * entire database into memory merely to read the type. This is especially important to maintain on master nodes where pipelines are + * validated. If we read the entire database into memory, we could potentially run into low-memory constraints on such nodes where + * loading this data would otherwise be wasteful if they are not also ingest nodes. + * + * @return the database type + * @throws IOException if an I/O exception occurs reading the database type + */ + public static String getDatabaseType(final Path database) throws IOException { + final long fileSize = Files.size(database); + try (InputStream in = Files.newInputStream(database)) { + // read the last BUFFER_SIZE bytes (or the fileSize, whichever is smaller) + final long skip = fileSize > BUFFER_SIZE ? fileSize - BUFFER_SIZE : 0; + final long skipped = in.skip(skip); + if (skipped != skip) { + throw new IOException("failed to skip [" + skip + "] bytes while reading [" + database + "]"); + } + final byte[] tail = new byte[BUFFER_SIZE]; + int read = 0; + int actualBytesRead; + do { + actualBytesRead = in.read(tail, read, BUFFER_SIZE - read); + read += actualBytesRead; + } while (actualBytesRead > 0); + + // find the database_type header + int metadataOffset = -1; + int markerOffset = 0; + for (int i = 0; i < tail.length; i++) { + byte b = tail[i]; + + if (b == DATABASE_TYPE_MARKER[markerOffset]) { + markerOffset++; + } else { + markerOffset = 0; + } + if (markerOffset == DATABASE_TYPE_MARKER.length) { + metadataOffset = i + 1; + break; + } + } + + if (metadataOffset == -1) { + throw new IOException("database type marker not found"); + } + + // read the database type + final int offsetByte = fromBytes(tail[metadataOffset]); + final int type = offsetByte >>> 5; + if (type != 2) { // 2 is the type indicator in the mmdb format for a UTF-8 string + throw new IOException("type must be UTF-8 string"); + } + int size = offsetByte & 0x1f; + if (size == 29) { + // then we need to read in yet another byte and add it onto this size + // this can actually occur in practice, a 29+ character type description isn't that hard to imagine + size = 29 + fromBytes(tail[metadataOffset + 1]); + metadataOffset += 1; + } else if (size >= 30) { + // we'd need to read two or three more bytes to get the size, but this means the type length is >=285 + throw new IOException("database_type too long [size indicator == " + size + "]"); + } + + return new String(tail, metadataOffset + 1, size, StandardCharsets.UTF_8); + } + } + + private static int fromBytes(byte b1) { + return b1 & 0xFF; + } + + public static boolean isGzip(Path path) throws IOException { + try (InputStream is = Files.newInputStream(path); InputStream gzis = new GZIPInputStream(is)) { + gzis.read(); // nooping, the point is just whether it's a gzip or not + return true; + } catch (ZipException e) { + return false; + } + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java new file mode 100644 index 0000000000000..8bc74c0e4aac4 --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java @@ -0,0 +1,779 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import com.maxmind.db.DatabaseRecord; +import com.maxmind.db.Network; +import com.maxmind.db.Reader; +import com.maxmind.geoip2.model.AbstractResponse; +import com.maxmind.geoip2.model.AnonymousIpResponse; +import com.maxmind.geoip2.model.AsnResponse; +import com.maxmind.geoip2.model.CityResponse; +import com.maxmind.geoip2.model.ConnectionTypeResponse; +import com.maxmind.geoip2.model.CountryResponse; +import com.maxmind.geoip2.model.DomainResponse; +import com.maxmind.geoip2.model.EnterpriseResponse; +import com.maxmind.geoip2.model.IspResponse; +import com.maxmind.geoip2.record.Continent; +import com.maxmind.geoip2.record.Location; +import com.maxmind.geoip2.record.Postal; +import com.maxmind.geoip2.record.Subdivision; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.core.Nullable; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +/** + * A collection of {@link IpDataLookup} implementations for MaxMind databases + */ +final class MaxmindIpDataLookups { + + private MaxmindIpDataLookups() { + // utility class + } + + private static final Logger logger = LogManager.getLogger(MaxmindIpDataLookups.class); + + // the actual prefixes from the metadata are cased like the literal strings, but + // prefix dispatch and checks case-insensitive, so the actual constants are lowercase + static final String GEOIP2_PREFIX = "GeoIP2".toLowerCase(Locale.ROOT); + static final String GEOLITE2_PREFIX = "GeoLite2".toLowerCase(Locale.ROOT); + + // note: the secondary dispatch on suffix happens to be case sensitive + private static final String CITY_DB_SUFFIX = "-City"; + private static final String COUNTRY_DB_SUFFIX = "-Country"; + private static final String ASN_DB_SUFFIX = "-ASN"; + private static final String ANONYMOUS_IP_DB_SUFFIX = "-Anonymous-IP"; + private static final String CONNECTION_TYPE_DB_SUFFIX = "-Connection-Type"; + private static final String DOMAIN_DB_SUFFIX = "-Domain"; + private static final String ENTERPRISE_DB_SUFFIX = "-Enterprise"; + private static final String ISP_DB_SUFFIX = "-ISP"; + + @Nullable + static Database getMaxmindDatabase(final String databaseType) { + if (databaseType.endsWith(CITY_DB_SUFFIX)) { + return Database.City; + } else if (databaseType.endsWith(COUNTRY_DB_SUFFIX)) { + return Database.Country; + } else if (databaseType.endsWith(ASN_DB_SUFFIX)) { + return Database.Asn; + } else if (databaseType.endsWith(ANONYMOUS_IP_DB_SUFFIX)) { + return Database.AnonymousIp; + } else if (databaseType.endsWith(CONNECTION_TYPE_DB_SUFFIX)) { + return Database.ConnectionType; + } else if (databaseType.endsWith(DOMAIN_DB_SUFFIX)) { + return Database.Domain; + } else if (databaseType.endsWith(ENTERPRISE_DB_SUFFIX)) { + return Database.Enterprise; + } else if (databaseType.endsWith(ISP_DB_SUFFIX)) { + return Database.Isp; + } else { + // no match was found + logger.trace("returning null for unsupported database_type [{}]", databaseType); + return null; + } + } + + @Nullable + static Function, IpDataLookup> getMaxmindLookup(final Database database) { + return switch (database) { + case City -> MaxmindIpDataLookups.City::new; + case Country -> MaxmindIpDataLookups.Country::new; + case Asn -> MaxmindIpDataLookups.Asn::new; + case AnonymousIp -> MaxmindIpDataLookups.AnonymousIp::new; + case ConnectionType -> MaxmindIpDataLookups.ConnectionType::new; + case Domain -> MaxmindIpDataLookups.Domain::new; + case Enterprise -> MaxmindIpDataLookups.Enterprise::new; + case Isp -> MaxmindIpDataLookups.Isp::new; + default -> null; + }; + } + + static class AnonymousIp extends AbstractBase { + AnonymousIp(final Set properties) { + super( + properties, + AnonymousIpResponse.class, + (response, ipAddress, network, locales) -> new AnonymousIpResponse(response, ipAddress, network) + ); + } + + @Override + protected Map transform(final AnonymousIpResponse response) { + boolean isHostingProvider = response.isHostingProvider(); + boolean isTorExitNode = response.isTorExitNode(); + boolean isAnonymousVpn = response.isAnonymousVpn(); + boolean isAnonymous = response.isAnonymous(); + boolean isPublicProxy = response.isPublicProxy(); + boolean isResidentialProxy = response.isResidentialProxy(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getIpAddress()); + case HOSTING_PROVIDER -> { + data.put("hosting_provider", isHostingProvider); + } + case TOR_EXIT_NODE -> { + data.put("tor_exit_node", isTorExitNode); + } + case ANONYMOUS_VPN -> { + data.put("anonymous_vpn", isAnonymousVpn); + } + case ANONYMOUS -> { + data.put("anonymous", isAnonymous); + } + case PUBLIC_PROXY -> { + data.put("public_proxy", isPublicProxy); + } + case RESIDENTIAL_PROXY -> { + data.put("residential_proxy", isResidentialProxy); + } + } + } + return data; + } + } + + static class Asn extends AbstractBase { + Asn(Set properties) { + super(properties, AsnResponse.class, (response, ipAddress, network, locales) -> new AsnResponse(response, ipAddress, network)); + } + + @Override + protected Map transform(final AsnResponse response) { + Long asn = response.getAutonomousSystemNumber(); + String organizationName = response.getAutonomousSystemOrganization(); + Network network = response.getNetwork(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getIpAddress()); + case ASN -> { + if (asn != null) { + data.put("asn", asn); + } + } + case ORGANIZATION_NAME -> { + if (organizationName != null) { + data.put("organization_name", organizationName); + } + } + case NETWORK -> { + if (network != null) { + data.put("network", network.toString()); + } + } + } + } + return data; + } + } + + static class City extends AbstractBase { + City(final Set properties) { + super(properties, CityResponse.class, CityResponse::new); + } + + @Override + protected Map transform(final CityResponse response) { + com.maxmind.geoip2.record.Country country = response.getCountry(); + com.maxmind.geoip2.record.Country registeredCountry = response.getRegisteredCountry(); + com.maxmind.geoip2.record.City city = response.getCity(); + Location location = response.getLocation(); + Continent continent = response.getContinent(); + Subdivision subdivision = response.getMostSpecificSubdivision(); + Postal postal = response.getPostal(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getTraits().getIpAddress()); + case COUNTRY_IN_EUROPEAN_UNION -> { + if (country.getIsoCode() != null) { + // isInEuropeanUnion is a boolean so it can't be null. But it really only makes sense if we have a country + data.put("country_in_european_union", country.isInEuropeanUnion()); + } + } + case COUNTRY_ISO_CODE -> { + String countryIsoCode = country.getIsoCode(); + if (countryIsoCode != null) { + data.put("country_iso_code", countryIsoCode); + } + } + case COUNTRY_NAME -> { + String countryName = country.getName(); + if (countryName != null) { + data.put("country_name", countryName); + } + } + case CONTINENT_CODE -> { + String continentCode = continent.getCode(); + if (continentCode != null) { + data.put("continent_code", continentCode); + } + } + case CONTINENT_NAME -> { + String continentName = continent.getName(); + if (continentName != null) { + data.put("continent_name", continentName); + } + } + case REGION_ISO_CODE -> { + // ISO 3166-2 code for country subdivisions. + // See iso.org/iso-3166-country-codes.html + String countryIso = country.getIsoCode(); + String subdivisionIso = subdivision.getIsoCode(); + if (countryIso != null && subdivisionIso != null) { + String regionIsoCode = countryIso + "-" + subdivisionIso; + data.put("region_iso_code", regionIsoCode); + } + } + case REGION_NAME -> { + String subdivisionName = subdivision.getName(); + if (subdivisionName != null) { + data.put("region_name", subdivisionName); + } + } + case CITY_NAME -> { + String cityName = city.getName(); + if (cityName != null) { + data.put("city_name", cityName); + } + } + case TIMEZONE -> { + String locationTimeZone = location.getTimeZone(); + if (locationTimeZone != null) { + data.put("timezone", locationTimeZone); + } + } + case LOCATION -> { + Double latitude = location.getLatitude(); + Double longitude = location.getLongitude(); + if (latitude != null && longitude != null) { + Map locationObject = new HashMap<>(); + locationObject.put("lat", latitude); + locationObject.put("lon", longitude); + data.put("location", locationObject); + } + } + case ACCURACY_RADIUS -> { + Integer accuracyRadius = location.getAccuracyRadius(); + if (accuracyRadius != null) { + data.put("accuracy_radius", accuracyRadius); + } + } + case POSTAL_CODE -> { + if (postal != null && postal.getCode() != null) { + data.put("postal_code", postal.getCode()); + } + } + case REGISTERED_COUNTRY_IN_EUROPEAN_UNION -> { + if (registeredCountry.getIsoCode() != null) { + // isInEuropeanUnion is a boolean so it can't be null. But it really only makes sense if we have a country + data.put("registered_country_in_european_union", registeredCountry.isInEuropeanUnion()); + } + } + case REGISTERED_COUNTRY_ISO_CODE -> { + if (registeredCountry.getIsoCode() != null) { + data.put("registered_country_iso_code", registeredCountry.getIsoCode()); + } + } + case REGISTERED_COUNTRY_NAME -> { + if (registeredCountry.getName() != null) { + data.put("registered_country_name", registeredCountry.getName()); + } + } + } + } + return data; + } + } + + static class ConnectionType extends AbstractBase { + ConnectionType(final Set properties) { + super( + properties, + ConnectionTypeResponse.class, + (response, ipAddress, network, locales) -> new ConnectionTypeResponse(response, ipAddress, network) + ); + } + + @Override + protected Map transform(final ConnectionTypeResponse response) { + ConnectionTypeResponse.ConnectionType connectionType = response.getConnectionType(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getIpAddress()); + case CONNECTION_TYPE -> { + if (connectionType != null) { + data.put("connection_type", connectionType.toString()); + } + } + } + } + return data; + } + } + + static class Country extends AbstractBase { + Country(final Set properties) { + super(properties, CountryResponse.class, CountryResponse::new); + } + + @Override + protected Map transform(final CountryResponse response) { + com.maxmind.geoip2.record.Country country = response.getCountry(); + com.maxmind.geoip2.record.Country registeredCountry = response.getRegisteredCountry(); + Continent continent = response.getContinent(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getTraits().getIpAddress()); + case COUNTRY_IN_EUROPEAN_UNION -> { + if (country.getIsoCode() != null) { + // isInEuropeanUnion is a boolean so it can't be null. But it really only makes sense if we have a country + data.put("country_in_european_union", country.isInEuropeanUnion()); + } + } + case COUNTRY_ISO_CODE -> { + String countryIsoCode = country.getIsoCode(); + if (countryIsoCode != null) { + data.put("country_iso_code", countryIsoCode); + } + } + case COUNTRY_NAME -> { + String countryName = country.getName(); + if (countryName != null) { + data.put("country_name", countryName); + } + } + case CONTINENT_CODE -> { + String continentCode = continent.getCode(); + if (continentCode != null) { + data.put("continent_code", continentCode); + } + } + case CONTINENT_NAME -> { + String continentName = continent.getName(); + if (continentName != null) { + data.put("continent_name", continentName); + } + } + case REGISTERED_COUNTRY_IN_EUROPEAN_UNION -> { + if (registeredCountry.getIsoCode() != null) { + // isInEuropeanUnion is a boolean so it can't be null. But it really only makes sense if we have a country + data.put("registered_country_in_european_union", registeredCountry.isInEuropeanUnion()); + } + } + case REGISTERED_COUNTRY_ISO_CODE -> { + if (registeredCountry.getIsoCode() != null) { + data.put("registered_country_iso_code", registeredCountry.getIsoCode()); + } + } + case REGISTERED_COUNTRY_NAME -> { + if (registeredCountry.getName() != null) { + data.put("registered_country_name", registeredCountry.getName()); + } + } + } + } + return data; + } + } + + static class Domain extends AbstractBase { + Domain(final Set properties) { + super( + properties, + DomainResponse.class, + (response, ipAddress, network, locales) -> new DomainResponse(response, ipAddress, network) + ); + } + + @Override + protected Map transform(final DomainResponse response) { + String domain = response.getDomain(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getIpAddress()); + case DOMAIN -> { + if (domain != null) { + data.put("domain", domain); + } + } + } + } + return data; + } + } + + static class Enterprise extends AbstractBase { + Enterprise(final Set properties) { + super(properties, EnterpriseResponse.class, EnterpriseResponse::new); + } + + @Override + protected Map transform(final EnterpriseResponse response) { + com.maxmind.geoip2.record.Country country = response.getCountry(); + com.maxmind.geoip2.record.Country registeredCountry = response.getRegisteredCountry(); + com.maxmind.geoip2.record.City city = response.getCity(); + Location location = response.getLocation(); + Continent continent = response.getContinent(); + Subdivision subdivision = response.getMostSpecificSubdivision(); + Postal postal = response.getPostal(); + + Long asn = response.getTraits().getAutonomousSystemNumber(); + String organizationName = response.getTraits().getAutonomousSystemOrganization(); + Network network = response.getTraits().getNetwork(); + + String isp = response.getTraits().getIsp(); + String ispOrganization = response.getTraits().getOrganization(); + String mobileCountryCode = response.getTraits().getMobileCountryCode(); + String mobileNetworkCode = response.getTraits().getMobileNetworkCode(); + + boolean isHostingProvider = response.getTraits().isHostingProvider(); + boolean isTorExitNode = response.getTraits().isTorExitNode(); + boolean isAnonymousVpn = response.getTraits().isAnonymousVpn(); + boolean isAnonymous = response.getTraits().isAnonymous(); + boolean isPublicProxy = response.getTraits().isPublicProxy(); + boolean isResidentialProxy = response.getTraits().isResidentialProxy(); + + String userType = response.getTraits().getUserType(); + + String domain = response.getTraits().getDomain(); + + ConnectionTypeResponse.ConnectionType connectionType = response.getTraits().getConnectionType(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getTraits().getIpAddress()); + case COUNTRY_CONFIDENCE -> { + Integer countryConfidence = country.getConfidence(); + if (countryConfidence != null) { + data.put("country_confidence", countryConfidence); + } + } + case COUNTRY_IN_EUROPEAN_UNION -> { + if (country.getIsoCode() != null) { + // isInEuropeanUnion is a boolean so it can't be null. But it really only makes sense if we have a country + data.put("country_in_european_union", country.isInEuropeanUnion()); + } + } + case COUNTRY_ISO_CODE -> { + String countryIsoCode = country.getIsoCode(); + if (countryIsoCode != null) { + data.put("country_iso_code", countryIsoCode); + } + } + case COUNTRY_NAME -> { + String countryName = country.getName(); + if (countryName != null) { + data.put("country_name", countryName); + } + } + case CONTINENT_CODE -> { + String continentCode = continent.getCode(); + if (continentCode != null) { + data.put("continent_code", continentCode); + } + } + case CONTINENT_NAME -> { + String continentName = continent.getName(); + if (continentName != null) { + data.put("continent_name", continentName); + } + } + case REGION_ISO_CODE -> { + // ISO 3166-2 code for country subdivisions. + // See iso.org/iso-3166-country-codes.html + String countryIso = country.getIsoCode(); + String subdivisionIso = subdivision.getIsoCode(); + if (countryIso != null && subdivisionIso != null) { + String regionIsoCode = countryIso + "-" + subdivisionIso; + data.put("region_iso_code", regionIsoCode); + } + } + case REGION_NAME -> { + String subdivisionName = subdivision.getName(); + if (subdivisionName != null) { + data.put("region_name", subdivisionName); + } + } + case CITY_CONFIDENCE -> { + Integer cityConfidence = city.getConfidence(); + if (cityConfidence != null) { + data.put("city_confidence", cityConfidence); + } + } + case CITY_NAME -> { + String cityName = city.getName(); + if (cityName != null) { + data.put("city_name", cityName); + } + } + case TIMEZONE -> { + String locationTimeZone = location.getTimeZone(); + if (locationTimeZone != null) { + data.put("timezone", locationTimeZone); + } + } + case LOCATION -> { + Double latitude = location.getLatitude(); + Double longitude = location.getLongitude(); + if (latitude != null && longitude != null) { + Map locationObject = new HashMap<>(); + locationObject.put("lat", latitude); + locationObject.put("lon", longitude); + data.put("location", locationObject); + } + } + case ACCURACY_RADIUS -> { + Integer accuracyRadius = location.getAccuracyRadius(); + if (accuracyRadius != null) { + data.put("accuracy_radius", accuracyRadius); + } + } + case POSTAL_CODE -> { + if (postal != null && postal.getCode() != null) { + data.put("postal_code", postal.getCode()); + } + } + case POSTAL_CONFIDENCE -> { + Integer postalConfidence = postal.getConfidence(); + if (postalConfidence != null) { + data.put("postal_confidence", postalConfidence); + } + } + case ASN -> { + if (asn != null) { + data.put("asn", asn); + } + } + case ORGANIZATION_NAME -> { + if (organizationName != null) { + data.put("organization_name", organizationName); + } + } + case NETWORK -> { + if (network != null) { + data.put("network", network.toString()); + } + } + case HOSTING_PROVIDER -> { + data.put("hosting_provider", isHostingProvider); + } + case TOR_EXIT_NODE -> { + data.put("tor_exit_node", isTorExitNode); + } + case ANONYMOUS_VPN -> { + data.put("anonymous_vpn", isAnonymousVpn); + } + case ANONYMOUS -> { + data.put("anonymous", isAnonymous); + } + case PUBLIC_PROXY -> { + data.put("public_proxy", isPublicProxy); + } + case RESIDENTIAL_PROXY -> { + data.put("residential_proxy", isResidentialProxy); + } + case DOMAIN -> { + if (domain != null) { + data.put("domain", domain); + } + } + case ISP -> { + if (isp != null) { + data.put("isp", isp); + } + } + case ISP_ORGANIZATION_NAME -> { + if (ispOrganization != null) { + data.put("isp_organization_name", ispOrganization); + } + } + case MOBILE_COUNTRY_CODE -> { + if (mobileCountryCode != null) { + data.put("mobile_country_code", mobileCountryCode); + } + } + case MOBILE_NETWORK_CODE -> { + if (mobileNetworkCode != null) { + data.put("mobile_network_code", mobileNetworkCode); + } + } + case USER_TYPE -> { + if (userType != null) { + data.put("user_type", userType); + } + } + case CONNECTION_TYPE -> { + if (connectionType != null) { + data.put("connection_type", connectionType.toString()); + } + } + case REGISTERED_COUNTRY_IN_EUROPEAN_UNION -> { + if (registeredCountry.getIsoCode() != null) { + // isInEuropeanUnion is a boolean so it can't be null. But it really only makes sense if we have a country + data.put("registered_country_in_european_union", registeredCountry.isInEuropeanUnion()); + } + } + case REGISTERED_COUNTRY_ISO_CODE -> { + if (registeredCountry.getIsoCode() != null) { + data.put("registered_country_iso_code", registeredCountry.getIsoCode()); + } + } + case REGISTERED_COUNTRY_NAME -> { + if (registeredCountry.getName() != null) { + data.put("registered_country_name", registeredCountry.getName()); + } + } + } + } + return data; + } + } + + static class Isp extends AbstractBase { + Isp(final Set properties) { + super(properties, IspResponse.class, (response, ipAddress, network, locales) -> new IspResponse(response, ipAddress, network)); + } + + @Override + protected Map transform(final IspResponse response) { + String isp = response.getIsp(); + String ispOrganization = response.getOrganization(); + String mobileNetworkCode = response.getMobileNetworkCode(); + String mobileCountryCode = response.getMobileCountryCode(); + Long asn = response.getAutonomousSystemNumber(); + String organizationName = response.getAutonomousSystemOrganization(); + Network network = response.getNetwork(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getIpAddress()); + case ASN -> { + if (asn != null) { + data.put("asn", asn); + } + } + case ORGANIZATION_NAME -> { + if (organizationName != null) { + data.put("organization_name", organizationName); + } + } + case NETWORK -> { + if (network != null) { + data.put("network", network.toString()); + } + } + case ISP -> { + if (isp != null) { + data.put("isp", isp); + } + } + case ISP_ORGANIZATION_NAME -> { + if (ispOrganization != null) { + data.put("isp_organization_name", ispOrganization); + } + } + case MOBILE_COUNTRY_CODE -> { + if (mobileCountryCode != null) { + data.put("mobile_country_code", mobileCountryCode); + } + } + case MOBILE_NETWORK_CODE -> { + if (mobileNetworkCode != null) { + data.put("mobile_network_code", mobileNetworkCode); + } + } + } + } + return data; + } + } + + /** + * As an internal detail, the {@code com.maxmind.geoip2.model } classes that are populated by + * {@link Reader#getRecord(InetAddress, Class)} are kinda half-populated and need to go through a second round of construction + * with context from the querying caller. This method gives us a place do that additional binding. Cleverly, the signature + * here matches the constructor for many of these model classes exactly, so an appropriate implementation can 'just' be a method + * reference in some cases (in other cases it needs to be a lambda). + */ + @FunctionalInterface + private interface ResponseBuilder { + RESPONSE build(RESPONSE resp, String address, Network network, List locales); + } + + /** + * The {@link MaxmindIpDataLookups.AbstractBase} is an abstract base implementation of {@link IpDataLookup} that + * provides common functionality for getting a specific kind of {@link AbstractResponse} from a {@link IpDatabase}. + * + * @param the intermediate type of {@link AbstractResponse} + */ + private abstract static class AbstractBase implements IpDataLookup { + + protected final Set properties; + protected final Class clazz; + protected final ResponseBuilder builder; + + AbstractBase(final Set properties, final Class clazz, final ResponseBuilder builder) { + this.properties = Set.copyOf(properties); + this.clazz = clazz; + this.builder = builder; + } + + @Override + public Set getProperties() { + return this.properties; + } + + @Override + public final Map getData(final IpDatabase ipDatabase, final String ipAddress) { + final RESPONSE response = ipDatabase.getResponse(ipAddress, this::lookup); + return (response == null) ? Map.of() : transform(response); + } + + @Nullable + private RESPONSE lookup(final Reader reader, final String ipAddress) throws IOException { + final InetAddress ip = InetAddresses.forString(ipAddress); + final DatabaseRecord record = reader.getRecord(ip, clazz); + final RESPONSE data = record.getData(); + return (data == null) ? null : builder.build(data, NetworkAddress.format(ip), record.getNetwork(), List.of("en")); + } + + /** + * Extract the configured properties from the retrieved response + * @param response the non-null response that was retrieved + * @return a mapping of properties for the ip from the response + */ + protected abstract Map transform(RESPONSE response); + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/TarInputStream.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/TarInputStream.java index 22d8e6c01a3ef..8d7f91fa4e1a5 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/TarInputStream.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/TarInputStream.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java index 0a43d7a2d830b..aa48c73cf1d73 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java @@ -1,20 +1,24 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip.direct; +import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -22,6 +26,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.Objects; import java.util.Set; import java.util.regex.Pattern; @@ -33,19 +38,19 @@ * That is, it has an id e.g. "my_db_config_1" and it says "download the file named XXXX from SomeCompany, and here's the * magic token to use to do that." */ -public record DatabaseConfiguration(String id, String name, Maxmind maxmind) implements Writeable, ToXContentObject { +public record DatabaseConfiguration(String id, String name, Provider provider) implements Writeable, ToXContentObject { // id is a user selected signifier like 'my_domain_db' // name is the name of a file that can be downloaded (like 'GeoIP2-Domain') - // a configuration will have a 'type' like "maxmind", and that might have some more details, + // a configuration will have a 'provider' like "maxmind", and that might have some more details, // for now, though the important thing is that the json has to have it even though we don't model it meaningfully in this class public DatabaseConfiguration { // these are invariants, not actual validation Objects.requireNonNull(id); Objects.requireNonNull(name); - Objects.requireNonNull(maxmind); + Objects.requireNonNull(provider); } /** @@ -74,26 +79,71 @@ public record DatabaseConfiguration(String id, String name, Maxmind maxmind) imp // "GeoLite2-Country" ); + public static final Set IPINFO_NAMES = Set.of( + // these file names are from https://ipinfo.io/developers/database-filename-reference + "asn", // "Free IP to ASN" + "country", // "Free IP to Country" + // "country_asn" // "Free IP to Country + IP to ASN", not supported at present + "standard_asn", // commercial "ASN" + "standard_location", // commercial "IP Geolocation" + "standard_privacy" // commercial "Privacy Detection" (sometimes "Anonymous IP") + ); + private static final ParseField NAME = new ParseField("name"); - private static final ParseField MAXMIND = new ParseField("maxmind"); + private static final ParseField MAXMIND = new ParseField(Maxmind.NAME); + private static final ParseField IPINFO = new ParseField(Ipinfo.NAME); + private static final ParseField WEB = new ParseField(Web.NAME); + private static final ParseField LOCAL = new ParseField(Local.NAME); private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "database", false, (a, id) -> { String name = (String) a[0]; - Maxmind maxmind = (Maxmind) a[1]; - return new DatabaseConfiguration(id, name, maxmind); + Provider provider; + + // one and only one provider object must be present + final long numNonNulls = Arrays.stream(a, 1, a.length).filter(Objects::nonNull).count(); + if (numNonNulls != 1) { + throw new IllegalArgumentException("Exactly one provider object must be specified, but [" + numNonNulls + "] were found"); + } + + if (a[1] != null) { + provider = (Maxmind) a[1]; + } else if (a[2] != null) { + provider = (Ipinfo) a[2]; + } else if (a[3] != null) { + provider = (Web) a[3]; + } else { + provider = (Local) a[4]; + } + return new DatabaseConfiguration(id, name, provider); } ); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME); - PARSER.declareObject(ConstructingObjectParser.constructorArg(), (parser, id) -> Maxmind.PARSER.apply(parser, null), MAXMIND); + PARSER.declareObject( + ConstructingObjectParser.optionalConstructorArg(), + (parser, id) -> Maxmind.PARSER.apply(parser, null), + MAXMIND + ); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (parser, id) -> Ipinfo.PARSER.apply(parser, null), IPINFO); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (parser, id) -> Web.PARSER.apply(parser, null), WEB); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (parser, id) -> Local.PARSER.apply(parser, null), LOCAL); } public DatabaseConfiguration(StreamInput in) throws IOException { - this(in.readString(), in.readString(), new Maxmind(in)); + this(in.readString(), in.readString(), readProvider(in)); + } + + private static Provider readProvider(StreamInput in) throws IOException { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { + return in.readNamedWriteable(Provider.class); + } else { + // prior to the above version, everything was always a maxmind, so this half of the if is logical + return new Maxmind(in.readString()); + } } public static DatabaseConfiguration parse(XContentParser parser, String id) { @@ -104,14 +154,27 @@ public static DatabaseConfiguration parse(XContentParser parser, String id) { public void writeTo(StreamOutput out) throws IOException { out.writeString(id); out.writeString(name); - maxmind.writeTo(out); + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { + out.writeNamedWriteable(provider); + } else { + if (provider instanceof Maxmind maxmind) { + out.writeString(maxmind.accountId); + } else { + /* + * The existence of a non-Maxmind providers is gated on the feature get_database_configuration_action.multi_node, and + * get_database_configuration_action.multi_node is only available on or after + * TransportVersions.INGEST_GEO_DATABASE_PROVIDERS. + */ + assert false : "non-maxmind DatabaseConfiguration.Provider [" + provider.getWriteableName() + "]"; + } + } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field("name", name); - builder.field("maxmind", maxmind); + builder.field(provider.getWriteableName(), provider); builder.endObject(); return builder; } @@ -153,8 +216,16 @@ public ActionRequestValidationException validate() { err.addValidationError("invalid name [" + name + "]: cannot be empty"); } - if (MAXMIND_NAMES.contains(name) == false) { - err.addValidationError("invalid name [" + name + "]: must be a supported name ([" + MAXMIND_NAMES + "])"); + // provider-specific name validation + if (provider instanceof Maxmind) { + if (MAXMIND_NAMES.contains(name) == false) { + err.addValidationError("invalid name [" + name + "]: must be a supported name ([" + MAXMIND_NAMES + "])"); + } + } + if (provider instanceof Ipinfo) { + if (IPINFO_NAMES.contains(name) == false) { + err.addValidationError("invalid name [" + name + "]: must be a supported name ([" + IPINFO_NAMES + "])"); + } } // important: the name must be unique across all configurations of this same type, @@ -167,7 +238,24 @@ public ActionRequestValidationException validate() { return err.validationErrors().isEmpty() ? null : err; } - public record Maxmind(String accountId) implements Writeable, ToXContentObject { + public boolean isReadOnly() { + return provider.isReadOnly(); + } + + /** + * A marker interface that all providers need to implement. + */ + public interface Provider extends NamedWriteable, ToXContentObject { + boolean isReadOnly(); + } + + public record Maxmind(String accountId) implements Provider { + public static final String NAME = "maxmind"; + + @Override + public String getWriteableName() { + return NAME; + } public Maxmind { // this is an invariant, not actual validation @@ -176,7 +264,7 @@ public record Maxmind(String accountId) implements Writeable, ToXContentObject { private static final ParseField ACCOUNT_ID = new ParseField("account_id"); - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("database", false, (a, id) -> { + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("maxmind", false, (a, id) -> { String accountId = (String) a[0]; return new Maxmind(accountId); }); @@ -189,10 +277,6 @@ public Maxmind(StreamInput in) throws IOException { this(in.readString()); } - public static Maxmind parse(XContentParser parser) { - return PARSER.apply(parser, null); - } - @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(accountId); @@ -205,5 +289,113 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); return builder; } + + @Override + public boolean isReadOnly() { + return false; + } + } + + public record Ipinfo() implements Provider { + public static final String NAME = "ipinfo"; + + // this'll become a ConstructingObjectParser once we accept the token (securely) in the json definition + private static final ObjectParser PARSER = new ObjectParser<>("ipinfo", Ipinfo::new); + + public Ipinfo(StreamInput in) throws IOException { + this(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException {} + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.endObject(); + return builder; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public boolean isReadOnly() { + return false; + } + } + + public record Local(String type) implements Provider { + public static final String NAME = "local"; + + private static final ParseField TYPE = new ParseField("type"); + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("database", false, (a, id) -> { + String type = (String) a[0]; + return new Local(type); + }); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), TYPE); + } + + public Local(StreamInput in) throws IOException { + this(in.readString()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(type); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("type", type); + builder.endObject(); + return builder; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public boolean isReadOnly() { + return true; + } + } + + public record Web() implements Provider { + public static final String NAME = "web"; + + private static final ObjectParser PARSER = new ObjectParser<>("database", Web::new); + + public Web(StreamInput in) throws IOException { + this(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException {} + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.endObject(); + return builder; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public boolean isReadOnly() { + return true; + } } } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationMetadata.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationMetadata.java index 574f97e4c5e64..fcfd8e51aabb5 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationMetadata.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationMetadata.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip.direct; @@ -65,7 +66,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws // (we'll be a in a json map where the id is the key) builder.startObject(); builder.field(VERSION.getPreferredName(), version); - builder.timeField(MODIFIED_DATE_MILLIS.getPreferredName(), MODIFIED_DATE.getPreferredName(), modifiedDate); + builder.timestampFieldsFromUnixEpochMillis(MODIFIED_DATE_MILLIS.getPreferredName(), MODIFIED_DATE.getPreferredName(), modifiedDate); builder.field(DATABASE.getPreferredName(), database); builder.endObject(); return builder; diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DeleteDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DeleteDatabaseConfigurationAction.java index 843cc986c47e7..a9b8beb89048a 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DeleteDatabaseConfigurationAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DeleteDatabaseConfigurationAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip.direct; diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationAction.java index 546c0c2df821d..af4ac8c93b550 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationAction.java @@ -1,34 +1,40 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip.direct; -import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.support.master.AcknowledgedRequest; -import org.elasticsearch.common.Strings; +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.support.nodes.BaseNodeResponse; +import org.elasticsearch.action.support.nodes.BaseNodesRequest; +import org.elasticsearch.action.support.nodes.BaseNodesResponse; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.core.TimeValue; +import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Objects; import static org.elasticsearch.ingest.geoip.direct.DatabaseConfigurationMetadata.DATABASE; import static org.elasticsearch.ingest.geoip.direct.DatabaseConfigurationMetadata.MODIFIED_DATE; import static org.elasticsearch.ingest.geoip.direct.DatabaseConfigurationMetadata.MODIFIED_DATE_MILLIS; import static org.elasticsearch.ingest.geoip.direct.DatabaseConfigurationMetadata.VERSION; +import static org.elasticsearch.ingest.geoip.direct.GetDatabaseConfigurationAction.Response; -public class GetDatabaseConfigurationAction extends ActionType { +public class GetDatabaseConfigurationAction extends ActionType { public static final GetDatabaseConfigurationAction INSTANCE = new GetDatabaseConfigurationAction(); public static final String NAME = "cluster:admin/ingest/geoip/database/get"; @@ -36,28 +42,16 @@ protected GetDatabaseConfigurationAction() { super(NAME); } - public static class Request extends AcknowledgedRequest { - + public static class Request extends BaseNodesRequest { private final String[] databaseIds; - public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String... databaseIds) { - super(masterNodeTimeout, ackTimeout); - this.databaseIds = Objects.requireNonNull(databaseIds, "ids may not be null"); - } - - public Request(StreamInput in) throws IOException { - super(in); - databaseIds = in.readStringArray(); + public Request(String... databaseIds) { + super((String[]) null); + this.databaseIds = databaseIds; } public String[] getDatabaseIds() { - return this.databaseIds; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeStringArray(databaseIds); + return databaseIds; } @Override @@ -76,27 +70,41 @@ public boolean equals(Object obj) { Request other = (Request) obj; return Arrays.equals(databaseIds, other.databaseIds); } + } - public static class Response extends ActionResponse implements ToXContentObject { + public static class Response extends BaseNodesResponse implements ToXContentObject { private final List databases; - public Response(List databases) { + public Response( + List databases, + ClusterName clusterName, + List nodes, + List failures + ) { + super(clusterName, nodes, failures); this.databases = List.copyOf(databases); // defensive copy } - public Response(StreamInput in) throws IOException { - this(in.readCollectionAsList(DatabaseConfigurationMetadata::new)); + protected Response(StreamInput in) throws IOException { + super(in); + this.databases = in.readCollectionAsList(DatabaseConfigurationMetadata::new); } - public List getDatabases() { - return this.databases; + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeCollection(databases); + } + + @Override + protected List readNodesFrom(StreamInput in) throws IOException { + return in.readCollectionAsList(NodeResponse::new); } @Override - public String toString() { - return Strings.toString(this); + protected void writeNodesTo(StreamOutput out, List nodes) throws IOException { + out.writeCollection(nodes); } @Override @@ -108,7 +116,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); builder.field("id", database.id()); // serialize including the id -- this is get response serialization builder.field(VERSION.getPreferredName(), item.version()); - builder.timeField(MODIFIED_DATE_MILLIS.getPreferredName(), MODIFIED_DATE.getPreferredName(), item.modifiedDate()); + builder.timestampFieldsFromUnixEpochMillis( + MODIFIED_DATE_MILLIS.getPreferredName(), + MODIFIED_DATE.getPreferredName(), + item.modifiedDate() + ); builder.field(DATABASE.getPreferredName(), database); builder.endObject(); } @@ -117,8 +129,127 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + /* + * This implementation of equals exists solely for testing the serialization of this object. + */ + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Response response = (Response) o; + return Objects.equals(databases, response.databases) + && Objects.equals(getClusterName(), response.getClusterName()) + && Objects.equals(equalsHashCodeFailures(), response.equalsHashCodeFailures()) + && Objects.equals(getNodes(), response.getNodes()) + && Objects.equals(equalsHashCodeNodesMap(), response.equalsHashCodeNodesMap()); + } + + /* + * This implementation of hashCode exists solely for testing the serialization of this object. + */ + @Override + public int hashCode() { + return Objects.hash(databases, getClusterName(), equalsHashCodeFailures(), getNodes(), equalsHashCodeNodesMap()); + } + + /* + * FailedNodeException does not implement equals or hashCode, making it difficult to test the serialization of this class. This + * helper method wraps the failures() list with a class that does implement equals and hashCode. + */ + private List equalsHashCodeFailures() { + return failures().stream().map(EqualsHashCodeFailedNodeException::new).toList(); + } + + private record EqualsHashCodeFailedNodeException(FailedNodeException failedNodeException) { + @Override + public boolean equals(Object o) { + if (o == this) return true; + if (o == null || getClass() != o.getClass()) return false; + EqualsHashCodeFailedNodeException other = (EqualsHashCodeFailedNodeException) o; + return Objects.equals(failedNodeException.nodeId(), other.failedNodeException.nodeId()) + && Objects.equals(failedNodeException.getMessage(), other.failedNodeException.getMessage()); + } + + @Override + public int hashCode() { + return Objects.hash(failedNodeException.nodeId(), failedNodeException.getMessage()); + } + } + + /* + * The getNodesMap method changes the value of the nodesMap, causing failures when testing the concurrent serialization and + * deserialization of this class. Since this is a response object, we do not actually care about concurrency since it will not + * happen in practice. So this helper method synchronizes access to getNodesMap, which can be used from equals and hashCode for + * tests. + */ + private synchronized Map equalsHashCodeNodesMap() { + return getNodesMap(); + } + } + + public static class NodeRequest extends TransportRequest { + + private final String[] databaseIds; + + public NodeRequest(String... databaseIds) { + super(); + this.databaseIds = Objects.requireNonNull(databaseIds, "ids may not be null"); + } + + public NodeRequest(StreamInput in) throws IOException { + super(in); + databaseIds = in.readStringArray(); + } + + public String[] getDatabaseIds() { + return this.databaseIds; + } + @Override public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeStringArray(databaseIds); + } + + @Override + public int hashCode() { + return Arrays.hashCode(databaseIds); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj.getClass() != getClass()) { + return false; + } + NodeRequest other = (NodeRequest) obj; + return Arrays.equals(databaseIds, other.databaseIds); + } + } + + public static class NodeResponse extends BaseNodeResponse { + + private final List databases; + + public NodeResponse(DiscoveryNode node, List databases) { + super(node); + this.databases = List.copyOf(databases); // defensive copy + } + + public NodeResponse(StreamInput in) throws IOException { + super(in); + this.databases = in.readCollectionAsList(DatabaseConfigurationMetadata::new); + } + + public List getDatabases() { + return this.databases; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); out.writeCollection(databases); } @@ -135,7 +266,7 @@ public boolean equals(Object obj) { if (obj.getClass() != getClass()) { return false; } - Response other = (Response) obj; + NodeResponse other = (NodeResponse) obj; return databases.equals(other.databases); } } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/PutDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/PutDatabaseConfigurationAction.java index 7bd5e1fa5cc68..b5343f17e47b6 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/PutDatabaseConfigurationAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/PutDatabaseConfigurationAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip.direct; @@ -48,7 +49,12 @@ public DatabaseConfiguration getDatabase() { } public static Request parseRequest(TimeValue masterNodeTimeout, TimeValue ackTimeout, String id, XContentParser parser) { - return new Request(masterNodeTimeout, ackTimeout, DatabaseConfiguration.parse(parser, id)); + DatabaseConfiguration database = DatabaseConfiguration.parse(parser, id); + if (database.isReadOnly()) { + throw new IllegalArgumentException("Database " + id + " is read only"); + } else { + return new Request(masterNodeTimeout, ackTimeout, database); + } } @Override diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestDeleteDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestDeleteDatabaseConfigurationAction.java index 4dc263224ad0a..78ea73250d632 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestDeleteDatabaseConfigurationAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestDeleteDatabaseConfigurationAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip.direct; @@ -26,7 +27,7 @@ public class RestDeleteDatabaseConfigurationAction extends BaseRestHandler { @Override public List routes() { - return List.of(new Route(DELETE, "/_ingest/geoip/database/{id}")); + return List.of(new Route(DELETE, "/_ingest/ip_location/database/{id}"), new Route(DELETE, "/_ingest/geoip/database/{id}")); } @Override diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestGetDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestGetDatabaseConfigurationAction.java index b237ceb638918..af446ee8d2bd9 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestGetDatabaseConfigurationAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestGetDatabaseConfigurationAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip.direct; @@ -19,15 +20,18 @@ import java.util.List; import static org.elasticsearch.rest.RestRequest.Method.GET; -import static org.elasticsearch.rest.RestUtils.getAckTimeout; -import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout; @ServerlessScope(Scope.INTERNAL) public class RestGetDatabaseConfigurationAction extends BaseRestHandler { @Override public List routes() { - return List.of(new Route(GET, "/_ingest/geoip/database"), new Route(GET, "/_ingest/geoip/database/{id}")); + return List.of( + new Route(GET, "/_ingest/ip_location/database"), + new Route(GET, "/_ingest/ip_location/database/{id}"), + new Route(GET, "/_ingest/geoip/database"), + new Route(GET, "/_ingest/geoip/database/{id}") + ); } @Override @@ -37,11 +41,7 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) { - final var req = new GetDatabaseConfigurationAction.Request( - getMasterNodeTimeout(request), - getAckTimeout(request), - Strings.splitStringByCommaToArray(request.param("id")) - ); + final var req = new GetDatabaseConfigurationAction.Request(Strings.splitStringByCommaToArray(request.param("id"))); return channel -> client.execute(GetDatabaseConfigurationAction.INSTANCE, req, new RestToXContentListener<>(channel)); } } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestPutDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestPutDatabaseConfigurationAction.java index 62b01b930d5cd..95b40df12fd1f 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestPutDatabaseConfigurationAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestPutDatabaseConfigurationAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip.direct; @@ -28,7 +29,7 @@ public class RestPutDatabaseConfigurationAction extends BaseRestHandler { @Override public List routes() { - return List.of(new Route(PUT, "/_ingest/geoip/database/{id}")); + return List.of(new Route(PUT, "/_ingest/ip_location/database/{id}"), new Route(PUT, "/_ingest/geoip/database/{id}")); } @Override diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportDeleteDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportDeleteDatabaseConfigurationAction.java index 4f9b9062332e4..b73b2fd4beb08 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportDeleteDatabaseConfigurationAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportDeleteDatabaseConfigurationAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip.direct; @@ -90,6 +91,8 @@ protected void masterOperation(Task task, Request request, ClusterState state, A final IngestGeoIpMetadata geoIpMeta = state.metadata().custom(IngestGeoIpMetadata.TYPE, IngestGeoIpMetadata.EMPTY); if (geoIpMeta.getDatabases().containsKey(id) == false) { throw new ResourceNotFoundException("Database configuration not found: {}", id); + } else if (geoIpMeta.getDatabases().get(id).database().isReadOnly()) { + throw new IllegalArgumentException("Database " + id + " is read only"); } deleteDatabaseConfigurationTaskQueue.submitTask( Strings.format("delete-geoip-database-configuration-[%s]", id), diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationAction.java index ae090dc4c64f6..c83c40e56b749 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationAction.java @@ -1,40 +1,57 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip.direct; -import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.master.TransportMasterNodeAction; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.block.ClusterBlockException; -import org.elasticsearch.cluster.block.ClusterBlockLevel; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.action.support.nodes.TransportNodesAction; +import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.regex.Regex; -import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.features.FeatureService; +import org.elasticsearch.ingest.geoip.DatabaseNodeService; +import org.elasticsearch.ingest.geoip.GeoIpTaskState; import org.elasticsearch.ingest.geoip.IngestGeoIpMetadata; import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.Base64; +import java.util.Collection; +import java.util.Comparator; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; -public class TransportGetDatabaseConfigurationAction extends TransportMasterNodeAction< +import static org.elasticsearch.ingest.IngestGeoIpFeatures.GET_DATABASE_CONFIGURATION_ACTION_MULTI_NODE; + +public class TransportGetDatabaseConfigurationAction extends TransportNodesAction< GetDatabaseConfigurationAction.Request, - GetDatabaseConfigurationAction.Response> { + GetDatabaseConfigurationAction.Response, + GetDatabaseConfigurationAction.NodeRequest, + GetDatabaseConfigurationAction.NodeResponse, + List> { + + private final FeatureService featureService; + private final DatabaseNodeService databaseNodeService; @Inject public TransportGetDatabaseConfigurationAction( @@ -42,28 +59,51 @@ public TransportGetDatabaseConfigurationAction( ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver + FeatureService featureService, + DatabaseNodeService databaseNodeService ) { super( GetDatabaseConfigurationAction.NAME, - transportService, clusterService, - threadPool, + transportService, actionFilters, - GetDatabaseConfigurationAction.Request::new, - indexNameExpressionResolver, - GetDatabaseConfigurationAction.Response::new, - EsExecutors.DIRECT_EXECUTOR_SERVICE + GetDatabaseConfigurationAction.NodeRequest::new, + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); + this.featureService = featureService; + this.databaseNodeService = databaseNodeService; } @Override - protected void masterOperation( - final Task task, - final GetDatabaseConfigurationAction.Request request, - final ClusterState state, - final ActionListener listener + protected void doExecute( + Task task, + GetDatabaseConfigurationAction.Request request, + ActionListener listener ) { + if (featureService.clusterHasFeature(clusterService.state(), GET_DATABASE_CONFIGURATION_ACTION_MULTI_NODE) == false) { + /* + * TransportGetDatabaseConfigurationAction used to be a TransportMasterNodeAction, and not all nodes in the cluster have been + * updated. So we don't want to send node requests to the other nodes because they will blow up. Instead, we just return + * the information that we used to return from the master node (it doesn't make any difference that this might not be the master + * node, because we're only reading the cluster state). Because older nodes only know about the Maxmind provider type, we filter + * out all others here to avoid causing problems on those nodes. + */ + newResponseAsync( + task, + request, + createActionContext(task, request).stream() + .filter(database -> database.database().provider() instanceof DatabaseConfiguration.Maxmind) + .toList(), + List.of(), + List.of(), + listener + ); + } else { + super.doExecute(task, request, listener); + } + } + + protected List createActionContext(Task task, GetDatabaseConfigurationAction.Request request) { final Set ids; if (request.getDatabaseIds().length == 0) { // if we did not ask for a specific name, then return all databases @@ -78,32 +118,193 @@ protected void masterOperation( ); } - final IngestGeoIpMetadata geoIpMeta = state.metadata().custom(IngestGeoIpMetadata.TYPE, IngestGeoIpMetadata.EMPTY); List results = new ArrayList<>(); - + PersistentTasksCustomMetadata tasksMetadata = PersistentTasksCustomMetadata.getPersistentTasksCustomMetadata( + clusterService.state() + ); for (String id : ids) { - if (Regex.isSimpleMatchPattern(id)) { - for (Map.Entry entry : geoIpMeta.getDatabases().entrySet()) { - if (Regex.simpleMatch(id, entry.getKey())) { - results.add(entry.getValue()); + results.addAll(getWebDatabases(tasksMetadata, id)); + results.addAll(getMaxmindDatabases(clusterService, id)); + } + return results; + } + + /* + * This returns read-only database information about the databases managed by the standard downloader + */ + private static Collection getWebDatabases(PersistentTasksCustomMetadata tasksMetadata, String id) { + List webDatabases = new ArrayList<>(); + if (tasksMetadata != null) { + PersistentTasksCustomMetadata.PersistentTask maybeGeoIpTask = tasksMetadata.getTask("geoip-downloader"); + if (maybeGeoIpTask != null) { + GeoIpTaskState geoIpTaskState = (GeoIpTaskState) maybeGeoIpTask.getState(); + if (geoIpTaskState != null) { + Map databases = geoIpTaskState.getDatabases(); + for (String databaseFileName : databases.keySet()) { + String databaseName = getDatabaseNameForFileName(databaseFileName); + String databaseId = getDatabaseIdForFileName(DatabaseConfiguration.Web.NAME, databaseFileName); + if ((Regex.isSimpleMatchPattern(id) && Regex.simpleMatch(id, databaseId)) || id.equals(databaseId)) { + webDatabases.add( + new DatabaseConfigurationMetadata( + new DatabaseConfiguration(databaseId, databaseName, new DatabaseConfiguration.Web()), + -1, + databases.get(databaseFileName).lastUpdate() + ) + ); + } } } - } else { - DatabaseConfigurationMetadata meta = geoIpMeta.getDatabases().get(id); - if (meta == null) { - listener.onFailure(new ResourceNotFoundException("database configuration not found: {}", id)); - return; - } else { - results.add(meta); + } + } + return webDatabases; + } + + private static String getDatabaseIdForFileName(String providerType, String databaseFileName) { + return "_" + providerType + "_" + Base64.getEncoder().encodeToString(databaseFileName.getBytes(StandardCharsets.UTF_8)); + } + + private static String getDatabaseNameForFileName(String databaseFileName) { + return databaseFileName.endsWith(".mmdb") + ? databaseFileName.substring(0, databaseFileName.length() + 1 - ".mmmdb".length()) + : databaseFileName; + } + + /* + * This returns information about databases that are downloaded from maxmind. + */ + private static Collection getMaxmindDatabases(ClusterService clusterService, String id) { + List maxmindDatabases = new ArrayList<>(); + final IngestGeoIpMetadata geoIpMeta = clusterService.state().metadata().custom(IngestGeoIpMetadata.TYPE, IngestGeoIpMetadata.EMPTY); + if (Regex.isSimpleMatchPattern(id)) { + for (Map.Entry entry : geoIpMeta.getDatabases().entrySet()) { + if (Regex.simpleMatch(id, entry.getKey())) { + maxmindDatabases.add(entry.getValue()); } } + } else { + DatabaseConfigurationMetadata meta = geoIpMeta.getDatabases().get(id); + if (meta != null) { + maxmindDatabases.add(meta); + } } + return maxmindDatabases; + } + + @Override + protected void newResponseAsync( + Task task, + GetDatabaseConfigurationAction.Request request, + List results, + List responses, + List failures, + ActionListener listener + ) { + ActionListener.run(listener, l -> { + List combinedResults = new ArrayList<>(results); + combinedResults.addAll( + deduplicateNodeResponses(responses, results.stream().map(result -> result.database().name()).collect(Collectors.toSet())) + ); + ActionListener.respondAndRelease( + l, + new GetDatabaseConfigurationAction.Response(combinedResults, clusterService.getClusterName(), responses, failures) + ); + }); + } - listener.onResponse(new GetDatabaseConfigurationAction.Response(results)); + /* + * This deduplicates the nodeResponses by name, favoring the most recent. This is because each node is reporting the local databases + * that it has, and we don't want to report duplicates to the user. It also filters out any that already exist in the set of + * preExistingNames. This is because the non-local databases take precedence, so any local database with the same name as a non-local + * one will not be used. + * Non-private for unit testing + */ + static Collection deduplicateNodeResponses( + List nodeResponses, + Set preExistingNames + ) { + /* + * Each node reports the list of databases that are in its config/ingest-geoip directory. For the sake of this API we assume all + * local databases with the same name are the same database, and deduplicate by name and just return the newest. + */ + return nodeResponses.stream() + .flatMap(response -> response.getDatabases().stream()) + .collect( + Collectors.groupingBy( + database -> database.database().name(), + Collectors.maxBy(Comparator.comparing(DatabaseConfigurationMetadata::modifiedDate)) + ) + ) + .values() + .stream() + .filter(Optional::isPresent) + .map(Optional::get) + .filter(database -> preExistingNames.contains(database.database().name()) == false) + .toList(); + } + + @Override + protected GetDatabaseConfigurationAction.Response newResponse( + GetDatabaseConfigurationAction.Request request, + List nodeResponses, + List failures + ) { + throw new UnsupportedOperationException("Use newResponseAsync instead"); + } + + @Override + protected GetDatabaseConfigurationAction.NodeRequest newNodeRequest(GetDatabaseConfigurationAction.Request request) { + return new GetDatabaseConfigurationAction.NodeRequest(request.getDatabaseIds()); } @Override - protected ClusterBlockException checkBlock(GetDatabaseConfigurationAction.Request request, ClusterState state) { - return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + protected GetDatabaseConfigurationAction.NodeResponse newNodeResponse(StreamInput in, DiscoveryNode node) throws IOException { + return new GetDatabaseConfigurationAction.NodeResponse(in); + } + + @Override + protected GetDatabaseConfigurationAction.NodeResponse nodeOperation(GetDatabaseConfigurationAction.NodeRequest request, Task task) { + final Set ids; + if (request.getDatabaseIds().length == 0) { + // if we did not ask for a specific name, then return all databases + ids = Set.of("*"); + } else { + ids = new LinkedHashSet<>(Arrays.asList(request.getDatabaseIds())); + } + if (ids.size() > 1 && ids.stream().anyMatch(Regex::isSimpleMatchPattern)) { + throw new IllegalArgumentException( + "wildcard only supports a single value, please use comma-separated values or a single wildcard value" + ); + } + + List results = new ArrayList<>(); + for (String id : ids) { + results.addAll(getLocalDatabases(databaseNodeService, id)); + } + return new GetDatabaseConfigurationAction.NodeResponse(transportService.getLocalNode(), results); + } + + /* + * This returns information about the databases that users have put in the config/ingest-geoip directory on the node. + */ + private static List getLocalDatabases(DatabaseNodeService databaseNodeService, String id) { + List localDatabases = new ArrayList<>(); + Map configDatabases = databaseNodeService.getConfigDatabasesDetail(); + for (DatabaseNodeService.ConfigDatabaseDetail configDatabase : configDatabases.values()) { + String databaseId = getDatabaseIdForFileName(DatabaseConfiguration.Local.NAME, configDatabase.name()); + if ((Regex.isSimpleMatchPattern(id) && Regex.simpleMatch(id, databaseId)) || id.equals(databaseId)) { + localDatabases.add( + new DatabaseConfigurationMetadata( + new DatabaseConfiguration( + databaseId, + getDatabaseNameForFileName(configDatabase.name()), + new DatabaseConfiguration.Local(configDatabase.type()) + ), + -1, + configDatabase.buildDateInMillis() == null ? -1 : configDatabase.buildDateInMillis() + ) + ); + } + } + return localDatabases; } } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java index 490a9edbec89a..dfb8fa78089d2 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip.direct; @@ -28,6 +29,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Strings; import org.elasticsearch.core.Tuple; +import org.elasticsearch.features.FeatureService; import org.elasticsearch.ingest.geoip.IngestGeoIpMetadata; import org.elasticsearch.ingest.geoip.direct.PutDatabaseConfigurationAction.Request; import org.elasticsearch.injection.guice.Inject; @@ -40,6 +42,8 @@ import java.util.Map; import java.util.Optional; +import static org.elasticsearch.ingest.IngestGeoIpFeatures.PUT_DATABASE_CONFIGURATION_ACTION_IPINFO; + public class TransportPutDatabaseConfigurationAction extends TransportMasterNodeAction { private static final Logger logger = LogManager.getLogger(TransportPutDatabaseConfigurationAction.class); @@ -57,6 +61,7 @@ public void taskSucceeded(UpdateDatabaseConfigurationTask task, Void unused) { } }; + private final FeatureService featureService; private final MasterServiceTaskQueue updateDatabaseConfigurationTaskQueue; @Inject @@ -65,7 +70,8 @@ public TransportPutDatabaseConfigurationAction( ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver + IndexNameExpressionResolver indexNameExpressionResolver, + FeatureService featureService ) { super( PutDatabaseConfigurationAction.NAME, @@ -78,6 +84,7 @@ public TransportPutDatabaseConfigurationAction( AcknowledgedResponse::readFrom, EsExecutors.DIRECT_EXECUTOR_SERVICE ); + this.featureService = featureService; this.updateDatabaseConfigurationTaskQueue = clusterService.createTaskQueue( "update-geoip-database-configuration-state-update", Priority.NORMAL, @@ -88,6 +95,19 @@ public TransportPutDatabaseConfigurationAction( @Override protected void masterOperation(Task task, Request request, ClusterState state, ActionListener listener) { final String id = request.getDatabase().id(); + + // if this is an ipinfo configuration, then make sure the whole cluster supports that feature + if (request.getDatabase().provider() instanceof DatabaseConfiguration.Ipinfo + && featureService.clusterHasFeature(clusterService.state(), PUT_DATABASE_CONFIGURATION_ACTION_IPINFO) == false) { + listener.onFailure( + new IllegalArgumentException( + "Unable to use ipinfo database configurations in mixed-clusters with nodes that do not support feature " + + PUT_DATABASE_CONFIGURATION_ACTION_IPINFO.id() + ) + ); + return; + } + updateDatabaseConfigurationTaskQueue.submitTask( Strings.format("update-geoip-database-configuration-[%s]", id), new UpdateDatabaseConfigurationTask(listener, request.getDatabase()), diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/CacheStats.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/CacheStats.java index 16a234e37e99e..f3be04c998b13 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/CacheStats.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/CacheStats.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip.stats; diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStats.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStats.java index 64b704a484058..9e84e5500358c 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStats.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStats.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip.stats; diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsAction.java index 81836cda29568..79865faa3b5b1 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip.stats; diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsTransportAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsTransportAction.java index 7a4b1fec900e9..9ebf97ca4e9ee 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsTransportAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsTransportAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip.stats; @@ -29,7 +30,7 @@ import java.io.IOException; import java.util.List; -public class GeoIpStatsTransportAction extends TransportNodesAction { +public class GeoIpStatsTransportAction extends TransportNodesAction { private final DatabaseNodeService registry; private final GeoIpDownloaderTaskExecutor geoIpDownloaderTaskExecutor; diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/RestGeoIpStatsAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/RestGeoIpStatsAction.java index ac6022205d04e..7813400058a49 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/RestGeoIpStatsAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/RestGeoIpStatsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip.stats; diff --git a/modules/ingest-geoip/src/main/plugin-metadata/plugin-security.policy b/modules/ingest-geoip/src/main/plugin-metadata/plugin-security.policy index 7002fba5c0c4f..bfd77bc296124 100644 --- a/modules/ingest-geoip/src/main/plugin-metadata/plugin-security.policy +++ b/modules/ingest-geoip/src/main/plugin-metadata/plugin-security.policy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ grant { diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ConfigDatabasesTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ConfigDatabasesTests.java index 7b962fed0ca83..7f38a37b43edf 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ConfigDatabasesTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ConfigDatabasesTests.java @@ -1,16 +1,16 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; import com.maxmind.geoip2.model.CityResponse; -import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.ESTestCase; @@ -126,7 +126,7 @@ public void testDatabasesUpdateExistingConfDatabase() throws Exception { DatabaseReaderLazyLoader loader = configDatabases.getDatabase("GeoLite2-City.mmdb"); assertThat(loader.getDatabaseType(), equalTo("GeoLite2-City")); - CityResponse cityResponse = loader.getCity(InetAddresses.forString("89.160.20.128")); + CityResponse cityResponse = loader.getResponse("89.160.20.128", GeoIpTestUtils::getCity); assertThat(cityResponse.getCity().getName(), equalTo("Tumba")); assertThat(cache.count(), equalTo(1)); } @@ -138,7 +138,7 @@ public void testDatabasesUpdateExistingConfDatabase() throws Exception { DatabaseReaderLazyLoader loader = configDatabases.getDatabase("GeoLite2-City.mmdb"); assertThat(loader.getDatabaseType(), equalTo("GeoLite2-City")); - CityResponse cityResponse = loader.getCity(InetAddresses.forString("89.160.20.128")); + CityResponse cityResponse = loader.getResponse("89.160.20.128", GeoIpTestUtils::getCity); assertThat(cityResponse.getCity().getName(), equalTo("Linköping")); assertThat(cache.count(), equalTo(1)); }); diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceTests.java index 1579c7020c58a..0ef4686dc033e 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; @@ -63,6 +64,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -310,12 +312,24 @@ public void testUpdateDatabase() throws Exception { private String mockSearches(String databaseName, int firstChunk, int lastChunk) throws IOException { String dummyContent = "test: " + databaseName; - List data = gzip(databaseName, dummyContent, lastChunk - firstChunk + 1); - assertThat(gunzip(data), equalTo(dummyContent)); + List data; + // We want to make sure we handle gzip files or plain mmdb files equally well: + if (randomBoolean()) { + data = gzip(databaseName, dummyContent, lastChunk - firstChunk + 1); + assertThat(gunzip(data), equalTo(dummyContent)); + } else { + data = chunkBytes(dummyContent, lastChunk - firstChunk + 1); + assertThat(unchunkBytes(data), equalTo(dummyContent)); + } Map> requestMap = new HashMap<>(); for (int i = firstChunk; i <= lastChunk; i++) { - byte[] chunk = data.get(i - firstChunk); + byte[] chunk; + if (i - firstChunk < data.size()) { + chunk = data.get(i - firstChunk); + } else { + chunk = new byte[0]; // We had so little data that the chunk(s) at the end will be empty + } SearchHit hit = SearchHit.unpooled(i); try (XContentBuilder builder = XContentBuilder.builder(XContentType.SMILE.xContent())) { builder.map(Map.of("data", chunk)); @@ -389,6 +403,39 @@ static ClusterState createClusterState(PersistentTasksCustomMetadata tasksCustom .build(); } + private static List chunkBytes(String content, int chunks) throws IOException { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + OutputStream outputStream = byteArrayOutputStream; + byte[] contentBytes = content.getBytes(StandardCharsets.UTF_8); + outputStream.write(contentBytes); + outputStream.close(); + + byte[] all = byteArrayOutputStream.toByteArray(); + int chunkSize = Math.max(1, all.length / chunks); + List data = new ArrayList<>(); + + for (int from = 0; from < all.length;) { + int to = from + chunkSize; + if (to > all.length) { + to = all.length; + } + data.add(Arrays.copyOfRange(all, from, to)); + from = to; + } + + while (data.size() > chunks) { + byte[] last = data.remove(data.size() - 1); + byte[] secondLast = data.remove(data.size() - 1); + byte[] merged = new byte[secondLast.length + last.length]; + System.arraycopy(secondLast, 0, merged, 0, secondLast.length); + System.arraycopy(last, 0, merged, secondLast.length, last.length); + data.add(merged); + } + + assert data.size() == Math.min(chunks, content.length()); + return data; + } + private static List gzip(String name, String content, int chunks) throws IOException { ByteArrayOutputStream bytes = new ByteArrayOutputStream(); GZIPOutputStream gzipOutputStream = new GZIPOutputStream(bytes); @@ -431,13 +478,23 @@ private static List gzip(String name, String content, int chunks) throws return data; } - private static String gunzip(List chunks) throws IOException { - byte[] gzippedContent = new byte[chunks.stream().mapToInt(value -> value.length).sum()]; + private static byte[] unchunkBytesToByteArray(List chunks) throws IOException { + byte[] allBytes = new byte[chunks.stream().mapToInt(value -> value.length).sum()]; int written = 0; for (byte[] chunk : chunks) { - System.arraycopy(chunk, 0, gzippedContent, written, chunk.length); + System.arraycopy(chunk, 0, allBytes, written, chunk.length); written += chunk.length; } + return allBytes; + } + + private static String unchunkBytes(List chunks) throws IOException { + byte[] allBytes = unchunkBytesToByteArray(chunks); + return new String(allBytes, StandardCharsets.UTF_8); + } + + private static String gunzip(List chunks) throws IOException { + byte[] gzippedContent = unchunkBytesToByteArray(chunks); TarInputStream gzipInputStream = new TarInputStream(new GZIPInputStream(new ByteArrayInputStream(gzippedContent))); gzipInputStream.getNextEntry(); return Streams.readFully(gzipInputStream).utf8ToString(); diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseTests.java new file mode 100644 index 0000000000000..39ecf4e70383b --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseTests.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.test.ESTestCase; + +import java.util.Set; + +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +public class DatabaseTests extends ESTestCase { + + public void testDatabasePropertyInvariants() { + // the city database is like a specialization of the country database + assertThat(Sets.difference(Database.Country.properties(), Database.City.properties()), is(empty())); + assertThat(Sets.difference(Database.Country.defaultProperties(), Database.City.defaultProperties()), is(empty())); + + // the isp database is like a specialization of the asn database + assertThat(Sets.difference(Database.Asn.properties(), Database.Isp.properties()), is(empty())); + assertThat(Sets.difference(Database.Asn.defaultProperties(), Database.Isp.defaultProperties()), is(empty())); + + // the enterprise database is like these other databases joined together + for (Database type : Set.of( + Database.City, + Database.Country, + Database.Asn, + Database.AnonymousIp, + Database.ConnectionType, + Database.Domain, + Database.Isp + )) { + assertThat(Sets.difference(type.properties(), Database.Enterprise.properties()), is(empty())); + } + // but in terms of the default fields, it's like a drop-in replacement for the city database + // n.b. this is just a choice we decided to make here at Elastic + assertThat(Database.Enterprise.defaultProperties(), equalTo(Database.City.defaultProperties())); + } + + public void testDatabaseVariantPropertyInvariants() { + // the second ASN variant database is like a specialization of the ASN database + assertThat(Sets.difference(Database.Asn.properties(), Database.AsnV2.properties()), is(empty())); + assertThat(Database.Asn.defaultProperties(), equalTo(Database.AsnV2.defaultProperties())); + + // the second City variant database is like a version of the ordinary City database but lacking many fields + assertThat(Sets.difference(Database.CityV2.properties(), Database.City.properties()), is(empty())); + assertThat(Sets.difference(Database.CityV2.defaultProperties(), Database.City.defaultProperties()), is(empty())); + + // the second Country variant database is like a version of the ordinary Country database but lacking come fields + assertThat(Sets.difference(Database.CountryV2.properties(), Database.CountryV2.properties()), is(empty())); + assertThat(Database.CountryV2.defaultProperties(), equalTo(Database.Country.defaultProperties())); + } +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTests.java index 1676ce14698a9..e1cd127be9c87 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; @@ -28,11 +29,11 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.ReferenceDocs; -import org.elasticsearch.common.hash.MessageDigests; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Tuple; import org.elasticsearch.ingest.EnterpriseGeoIpTask; +import org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloader.Checksum; import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration; import org.elasticsearch.node.Node; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; @@ -50,11 +51,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -import java.net.PasswordAuthentication; import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.time.Instant; -import java.time.temporal.ChronoUnit; import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -110,7 +107,7 @@ public void setup() throws IOException { EMPTY_TASK_ID, Map.of(), () -> GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING.getDefault(Settings.EMPTY), - (input) -> new HttpClient.PasswordAuthenticationHolder("name", "password".toCharArray()) + (type) -> "password".toCharArray() ) { { EnterpriseGeoIpTask.EnterpriseGeoIpTaskParams geoIpTaskParams = mock(EnterpriseGeoIpTask.EnterpriseGeoIpTaskParams.class); @@ -205,8 +202,7 @@ public void testIndexChunksNoData() throws IOException { "test", empty, 0, - MessageDigests.sha256(), - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + Checksum.sha256("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"), 0 ) ); @@ -227,7 +223,7 @@ public void testIndexChunksMd5Mismatch() { IOException exception = expectThrows( IOException.class, - () -> geoIpDownloader.indexChunks("test", new ByteArrayInputStream(new byte[0]), 0, MessageDigests.sha256(), "123123", 0) + () -> geoIpDownloader.indexChunks("test", new ByteArrayInputStream(new byte[0]), 0, Checksum.sha256("123123"), 0) ); assertEquals( "checksum mismatch, expected [123123], actual [e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855]", @@ -278,8 +274,7 @@ public void testIndexChunks() throws IOException { "test", big, 15, - MessageDigests.sha256(), - "f2304545f224ff9ffcc585cb0a993723f911e03beb552cc03937dd443e931eab", + Checksum.sha256("f2304545f224ff9ffcc585cb0a993723f911e03beb552cc03937dd443e931eab"), 0 ) ); @@ -303,7 +298,7 @@ public void testProcessDatabaseNew() throws IOException { EMPTY_TASK_ID, Map.of(), () -> GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING.getDefault(Settings.EMPTY), - (input) -> new HttpClient.PasswordAuthenticationHolder("name", "password".toCharArray()) + (type) -> "password".toCharArray() ) { @Override protected void updateTimestamp(String name, GeoIpTaskState.Metadata metadata) { @@ -311,18 +306,11 @@ protected void updateTimestamp(String name, GeoIpTaskState.Metadata metadata) { } @Override - Tuple indexChunks( - String name, - InputStream is, - int chunk, - MessageDigest digest, - String expectedMd5, - long start - ) { + Tuple indexChunks(String name, InputStream is, int chunk, Checksum checksum, long start) { assertSame(bais, is); assertEquals(0, chunk); indexedChunks.set(true); - return Tuple.tuple(11, expectedMd5); + return Tuple.tuple(11, checksum.checksum()); } @Override @@ -339,10 +327,9 @@ void deleteOldChunks(String name, int firstChunk) { }; geoIpDownloader.setState(EnterpriseGeoIpTaskState.EMPTY); - PasswordAuthentication auth = new PasswordAuthentication("name", "password".toCharArray()); String id = randomIdentifier(); DatabaseConfiguration databaseConfiguration = new DatabaseConfiguration(id, "test", new DatabaseConfiguration.Maxmind("name")); - geoIpDownloader.processDatabase(auth, databaseConfiguration); + geoIpDownloader.processDatabase(id, databaseConfiguration); assertThat(indexedChunks.get(), equalTo(true)); } @@ -362,7 +349,7 @@ public void testProcessDatabaseUpdate() throws IOException { EMPTY_TASK_ID, Map.of(), () -> GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING.getDefault(Settings.EMPTY), - (input) -> new HttpClient.PasswordAuthenticationHolder("name", "password".toCharArray()) + (type) -> "password".toCharArray() ) { @Override protected void updateTimestamp(String name, GeoIpTaskState.Metadata metadata) { @@ -370,18 +357,11 @@ protected void updateTimestamp(String name, GeoIpTaskState.Metadata metadata) { } @Override - Tuple indexChunks( - String name, - InputStream is, - int chunk, - MessageDigest digest, - String expectedMd5, - long start - ) { + Tuple indexChunks(String name, InputStream is, int chunk, Checksum checksum, long start) { assertSame(bais, is); assertEquals(9, chunk); indexedChunks.set(true); - return Tuple.tuple(1, expectedMd5); + return Tuple.tuple(1, checksum.checksum()); } @Override @@ -398,10 +378,9 @@ void deleteOldChunks(String name, int firstChunk) { }; geoIpDownloader.setState(EnterpriseGeoIpTaskState.EMPTY.put("test.mmdb", new GeoIpTaskState.Metadata(0, 5, 8, "0", 0))); - PasswordAuthentication auth = new PasswordAuthentication("name", "password".toCharArray()); String id = randomIdentifier(); DatabaseConfiguration databaseConfiguration = new DatabaseConfiguration(id, "test", new DatabaseConfiguration.Maxmind("name")); - geoIpDownloader.processDatabase(auth, databaseConfiguration); + geoIpDownloader.processDatabase(id, databaseConfiguration); assertThat(indexedChunks.get(), equalTo(true)); } @@ -430,7 +409,7 @@ public void testProcessDatabaseSame() throws IOException { EMPTY_TASK_ID, Map.of(), () -> GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING.getDefault(Settings.EMPTY), - (input) -> new HttpClient.PasswordAuthenticationHolder("name", "password".toCharArray()) + (type) -> "password".toCharArray() ) { @Override protected void updateTimestamp(String name, GeoIpTaskState.Metadata newMetadata) { @@ -439,16 +418,9 @@ protected void updateTimestamp(String name, GeoIpTaskState.Metadata newMetadata) } @Override - Tuple indexChunks( - String name, - InputStream is, - int chunk, - MessageDigest digest, - String expectedChecksum, - long start - ) { + Tuple indexChunks(String name, InputStream is, int chunk, Checksum checksum, long start) { fail(); - return Tuple.tuple(0, expectedChecksum); + return Tuple.tuple(0, checksum.checksum()); } @Override @@ -462,10 +434,9 @@ void deleteOldChunks(String name, int firstChunk) { } }; geoIpDownloader.setState(taskState); - PasswordAuthentication auth = new PasswordAuthentication("name", "password".toCharArray()); String id = randomIdentifier(); DatabaseConfiguration databaseConfiguration = new DatabaseConfiguration(id, "test", new DatabaseConfiguration.Maxmind("name")); - geoIpDownloader.processDatabase(auth, databaseConfiguration); + geoIpDownloader.processDatabase(id, databaseConfiguration); } public void testUpdateDatabasesWriteBlock() { @@ -501,14 +472,50 @@ public void testUpdateDatabasesIndexNotReady() throws IOException { verifyNoInteractions(httpClient); } - private GeoIpTaskState.Metadata newGeoIpTaskStateMetadata(boolean expired) { - Instant lastChecked; - if (expired) { - lastChecked = Instant.now().minus(randomIntBetween(31, 100), ChronoUnit.DAYS); - } else { - lastChecked = Instant.now().minus(randomIntBetween(0, 29), ChronoUnit.DAYS); + public void testMaxmindUrls() { + // non-static classes have fun syntax, but it's nice to be able to test this behavior by itself + final EnterpriseGeoIpDownloader.MaxmindDownload download = geoIpDownloader.new MaxmindDownload( + "GeoLite2-City", new DatabaseConfiguration.Maxmind("account_id") + ); + + { + String url = "https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz"; + assertThat(download.url("tar.gz"), equalTo(url)); + } + { + String url = "https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz.sha256"; + assertThat(download.url("tar.gz.sha256"), equalTo(url)); + } + } + + public void testIpinfoUrls() { + // a 'free' database like 'asn' has 'free/' in the url (automatically) + final EnterpriseGeoIpDownloader.IpinfoDownload download = geoIpDownloader.new IpinfoDownload( + "asn", new DatabaseConfiguration.Ipinfo() + ); + + { + String url = "https://ipinfo.io/data/free/asn.mmdb"; + assertThat(download.url("mmdb"), equalTo(url)); + } + { + String url = "https://ipinfo.io/data/free/asn.mmdb/checksums"; + assertThat(download.url("mmdb/checksums"), equalTo(url)); + } + + // but a non-'free' database like 'standard_asn' does not + final EnterpriseGeoIpDownloader.IpinfoDownload download2 = geoIpDownloader.new IpinfoDownload( + "standard_asn", new DatabaseConfiguration.Ipinfo() + ); + + { + String url = "https://ipinfo.io/data/standard_asn.mmdb"; + assertThat(download2.url("mmdb"), equalTo(url)); + } + { + String url = "https://ipinfo.io/data/standard_asn.mmdb/checksums"; + assertThat(download2.url("mmdb/checksums"), equalTo(url)); } - return new GeoIpTaskState.Metadata(0, 0, 0, randomAlphaOfLength(20), lastChecked.toEpochMilli()); } private static class MockClient extends NoOpClient { diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpTaskStateSerializationTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpTaskStateSerializationTests.java index a136f90780989..5b6b7b971e1bb 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpTaskStateSerializationTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpTaskStateSerializationTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpCacheTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpCacheTests.java index 03ab51d510176..0c92aca882913 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpCacheTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpCacheTests.java @@ -1,21 +1,20 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; import com.maxmind.geoip2.model.AbstractResponse; -import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.core.TimeValue; import org.elasticsearch.ingest.geoip.stats.CacheStats; import org.elasticsearch.test.ESTestCase; -import java.net.InetAddress; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; @@ -31,34 +30,34 @@ public void testCachesAndEvictsResults() { AbstractResponse response2 = mock(AbstractResponse.class); // add a key - AbstractResponse cachedResponse = cache.putIfAbsent(InetAddresses.forString("127.0.0.1"), "path/to/db", ip -> response1); + AbstractResponse cachedResponse = cache.putIfAbsent("127.0.0.1", "path/to/db", ip -> response1); assertSame(cachedResponse, response1); - assertSame(cachedResponse, cache.putIfAbsent(InetAddresses.forString("127.0.0.1"), "path/to/db", ip -> response1)); - assertSame(cachedResponse, cache.get(InetAddresses.forString("127.0.0.1"), "path/to/db")); + assertSame(cachedResponse, cache.putIfAbsent("127.0.0.1", "path/to/db", ip -> response1)); + assertSame(cachedResponse, cache.get("127.0.0.1", "path/to/db")); // evict old key by adding another value - cachedResponse = cache.putIfAbsent(InetAddresses.forString("127.0.0.2"), "path/to/db", ip -> response2); + cachedResponse = cache.putIfAbsent("127.0.0.2", "path/to/db", ip -> response2); assertSame(cachedResponse, response2); - assertSame(cachedResponse, cache.putIfAbsent(InetAddresses.forString("127.0.0.2"), "path/to/db", ip -> response2)); - assertSame(cachedResponse, cache.get(InetAddresses.forString("127.0.0.2"), "path/to/db")); - assertNotSame(response1, cache.get(InetAddresses.forString("127.0.0.1"), "path/to/db")); + assertSame(cachedResponse, cache.putIfAbsent("127.0.0.2", "path/to/db", ip -> response2)); + assertSame(cachedResponse, cache.get("127.0.0.2", "path/to/db")); + assertNotSame(response1, cache.get("127.0.0.1", "path/to/db")); } public void testCachesNoResult() { GeoIpCache cache = new GeoIpCache(1); final AtomicInteger count = new AtomicInteger(0); - Function countAndReturnNull = (ip) -> { + Function countAndReturnNull = (ip) -> { count.incrementAndGet(); return null; }; - AbstractResponse response = cache.putIfAbsent(InetAddresses.forString("127.0.0.1"), "path/to/db", countAndReturnNull); + AbstractResponse response = cache.putIfAbsent("127.0.0.1", "path/to/db", countAndReturnNull); assertNull(response); - assertNull(cache.putIfAbsent(InetAddresses.forString("127.0.0.1"), "path/to/db", countAndReturnNull)); + assertNull(cache.putIfAbsent("127.0.0.1", "path/to/db", countAndReturnNull)); assertEquals(1, count.get()); // the cached value is not actually *null*, it's the NO_RESULT sentinel - assertSame(GeoIpCache.NO_RESULT, cache.get(InetAddresses.forString("127.0.0.1"), "path/to/db")); + assertSame(GeoIpCache.NO_RESULT, cache.get("127.0.0.1", "path/to/db")); } public void testCacheKey() { @@ -66,17 +65,17 @@ public void testCacheKey() { AbstractResponse response1 = mock(AbstractResponse.class); AbstractResponse response2 = mock(AbstractResponse.class); - assertSame(response1, cache.putIfAbsent(InetAddresses.forString("127.0.0.1"), "path/to/db1", ip -> response1)); - assertSame(response2, cache.putIfAbsent(InetAddresses.forString("127.0.0.1"), "path/to/db2", ip -> response2)); - assertSame(response1, cache.get(InetAddresses.forString("127.0.0.1"), "path/to/db1")); - assertSame(response2, cache.get(InetAddresses.forString("127.0.0.1"), "path/to/db2")); + assertSame(response1, cache.putIfAbsent("127.0.0.1", "path/to/db1", ip -> response1)); + assertSame(response2, cache.putIfAbsent("127.0.0.1", "path/to/db2", ip -> response2)); + assertSame(response1, cache.get("127.0.0.1", "path/to/db1")); + assertSame(response2, cache.get("127.0.0.1", "path/to/db2")); } public void testThrowsFunctionsException() { GeoIpCache cache = new GeoIpCache(1); IllegalArgumentException ex = expectThrows( IllegalArgumentException.class, - () -> cache.putIfAbsent(InetAddresses.forString("127.0.0.1"), "path/to/db", ip -> { + () -> cache.putIfAbsent("127.0.0.1", "path/to/db", ip -> { throw new IllegalArgumentException("bad"); }) ); @@ -95,9 +94,9 @@ public void testGetCacheStats() { GeoIpCache cache = new GeoIpCache(maxCacheSize, () -> testNanoTime.addAndGet(TimeValue.timeValueMillis(1).getNanos())); AbstractResponse response = mock(AbstractResponse.class); String databasePath = "path/to/db1"; - InetAddress key1 = InetAddresses.forString("127.0.0.1"); - InetAddress key2 = InetAddresses.forString("127.0.0.2"); - InetAddress key3 = InetAddresses.forString("127.0.0.3"); + String key1 = "127.0.0.1"; + String key2 = "127.0.0.2"; + String key3 = "127.0.0.3"; cache.putIfAbsent(key1, databasePath, ip -> response); // cache miss cache.putIfAbsent(key2, databasePath, ip -> response); // cache miss diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutorTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutorTests.java index b33b66696cfca..0929e050ba999 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutorTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java index f213868fb65a1..5698328792787 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java @@ -1,14 +1,14 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; @@ -24,11 +24,9 @@ import org.elasticsearch.action.index.TransportIndexAction; import org.elasticsearch.action.support.broadcast.BroadcastResponse; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.reindex.BulkByScrollResponse; @@ -582,37 +580,28 @@ void processDatabase(Map databaseInfo) { assertFalse(it.hasNext()); } - public void testUpdateDatabasesWriteBlock() { + public void testUpdateDatabasesWriteBlock() throws IOException { + /* + * Here we make sure that we bail out before making an httpClient request if there is write block on the .geoip_databases index + */ ClusterState state = createClusterState(new PersistentTasksCustomMetadata(1L, Map.of())); var geoIpIndex = state.getMetadata().getIndicesLookup().get(GeoIpDownloader.DATABASES_INDEX).getWriteIndex().getName(); state = ClusterState.builder(state) .blocks(new ClusterBlocks.Builder().addIndexBlock(geoIpIndex, IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK)) .build(); when(clusterService.state()).thenReturn(state); - var e = expectThrows(ClusterBlockException.class, () -> geoIpDownloader.updateDatabases()); - assertThat( - e.getMessage(), - equalTo( - "index [" - + geoIpIndex - + "] blocked by: [TOO_MANY_REQUESTS/12/disk usage exceeded flood-stage watermark, " - + "index has read-only-allow-delete block; for more information, see " - + ReferenceDocs.FLOOD_STAGE_WATERMARK - + "];" - ) - ); + geoIpDownloader.updateDatabases(); verifyNoInteractions(httpClient); } - public void testUpdateDatabasesIndexNotReady() { + public void testUpdateDatabasesIndexNotReady() throws IOException { + /* + * Here we make sure that we bail out before making an httpClient request if there are unallocated shards on the .geoip_databases + * index + */ ClusterState state = createClusterState(new PersistentTasksCustomMetadata(1L, Map.of()), true); - var geoIpIndex = state.getMetadata().getIndicesLookup().get(GeoIpDownloader.DATABASES_INDEX).getWriteIndex().getName(); - state = ClusterState.builder(state) - .blocks(new ClusterBlocks.Builder().addIndexBlock(geoIpIndex, IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK)) - .build(); when(clusterService.state()).thenReturn(state); - var e = expectThrows(ElasticsearchException.class, () -> geoIpDownloader.updateDatabases()); - assertThat(e.getMessage(), equalTo("not all primary shards of [.geoip_databases] index are active")); + geoIpDownloader.updateDatabases(); verifyNoInteractions(httpClient); } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java index a0541df0d4d8a..5ac0c76054d33 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; @@ -42,6 +43,7 @@ import java.util.Map; import java.util.Set; +import static org.elasticsearch.ingest.geoip.GeoIpProcessor.GEOIP_TYPE; import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.DEFAULT_DATABASES; import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase; import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases; @@ -82,12 +84,12 @@ public void loadDatabaseReaders() throws IOException { @After public void closeDatabaseReaders() throws IOException { - databaseNodeService.close(); + databaseNodeService.shutdown(); databaseNodeService = null; } public void testBuildDefaults() throws Exception { - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(GEOIP_TYPE, databaseNodeService); Map config = new HashMap<>(); config.put("field", "_field"); @@ -103,7 +105,7 @@ public void testBuildDefaults() throws Exception { } public void testSetIgnoreMissing() throws Exception { - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(GEOIP_TYPE, databaseNodeService); Map config = new HashMap<>(); config.put("field", "_field"); @@ -120,7 +122,7 @@ public void testSetIgnoreMissing() throws Exception { } public void testCountryBuildDefaults() throws Exception { - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(GEOIP_TYPE, databaseNodeService); Map config = new HashMap<>(); config.put("field", "_field"); @@ -138,7 +140,7 @@ public void testCountryBuildDefaults() throws Exception { } public void testAsnBuildDefaults() throws Exception { - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(GEOIP_TYPE, databaseNodeService); Map config = new HashMap<>(); config.put("field", "_field"); @@ -156,7 +158,7 @@ public void testAsnBuildDefaults() throws Exception { } public void testBuildTargetField() throws Exception { - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(GEOIP_TYPE, databaseNodeService); Map config = new HashMap<>(); config.put("field", "_field"); config.put("target_field", "_field"); @@ -167,7 +169,7 @@ public void testBuildTargetField() throws Exception { } public void testBuildDbFile() throws Exception { - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(GEOIP_TYPE, databaseNodeService); Map config = new HashMap<>(); config.put("field", "_field"); config.put("database_file", "GeoLite2-Country.mmdb"); @@ -180,7 +182,7 @@ public void testBuildDbFile() throws Exception { } public void testBuildWithCountryDbAndAsnFields() { - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(GEOIP_TYPE, databaseNodeService); Map config = new HashMap<>(); config.put("field", "_field"); config.put("database_file", "GeoLite2-Country.mmdb"); @@ -194,13 +196,14 @@ public void testBuildWithCountryDbAndAsnFields() { equalTo( "[properties] illegal property value [" + asnProperty - + "]. valid values are [IP, COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_CODE, CONTINENT_NAME]" + + "]. valid values are [IP, COUNTRY_IN_EUROPEAN_UNION, COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_CODE, " + + "CONTINENT_NAME, REGISTERED_COUNTRY_IN_EUROPEAN_UNION, REGISTERED_COUNTRY_ISO_CODE, REGISTERED_COUNTRY_NAME]" ) ); } public void testBuildWithAsnDbAndCityFields() { - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(GEOIP_TYPE, databaseNodeService); Map config = new HashMap<>(); config.put("field", "_field"); config.put("database_file", "GeoLite2-ASN.mmdb"); @@ -218,7 +221,7 @@ public void testBuildWithAsnDbAndCityFields() { public void testBuildNonExistingDbFile() throws Exception { copyDatabase("GeoLite2-City-Test.mmdb", geoipTmpDir.resolve("GeoLite2-City.mmdb")); databaseNodeService.updateDatabase("GeoLite2-City.mmdb", "md5", geoipTmpDir.resolve("GeoLite2-City.mmdb")); - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(GEOIP_TYPE, databaseNodeService); Map config = new HashMap<>(); config.put("field", "_field"); @@ -228,7 +231,7 @@ public void testBuildNonExistingDbFile() throws Exception { } public void testBuildBuiltinDatabaseMissing() throws Exception { - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(GEOIP_TYPE, databaseNodeService); cleanDatabases(geoIpConfigDir, configDatabases); Map config = new HashMap<>(); @@ -239,7 +242,7 @@ public void testBuildBuiltinDatabaseMissing() throws Exception { } public void testBuildFields() throws Exception { - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(GEOIP_TYPE, databaseNodeService); Set properties = new HashSet<>(); List fieldNames = new ArrayList<>(); @@ -263,7 +266,7 @@ public void testBuildFields() throws Exception { } public void testBuildIllegalFieldOption() { - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(GEOIP_TYPE, databaseNodeService); Map config1 = new HashMap<>(); config1.put("field", "_field"); @@ -272,8 +275,10 @@ public void testBuildIllegalFieldOption() { assertThat( e.getMessage(), equalTo( - "[properties] illegal property value [invalid]. valid values are [IP, COUNTRY_ISO_CODE, " - + "COUNTRY_NAME, CONTINENT_CODE, CONTINENT_NAME, REGION_ISO_CODE, REGION_NAME, CITY_NAME, TIMEZONE, LOCATION]" + "[properties] illegal property value [invalid]. valid values are [IP, COUNTRY_IN_EUROPEAN_UNION, COUNTRY_ISO_CODE, " + + "COUNTRY_NAME, CONTINENT_CODE, CONTINENT_NAME, REGION_ISO_CODE, REGION_NAME, CITY_NAME, TIMEZONE, " + + "LOCATION, POSTAL_CODE, ACCURACY_RADIUS, REGISTERED_COUNTRY_IN_EUROPEAN_UNION, REGISTERED_COUNTRY_ISO_CODE, " + + "REGISTERED_COUNTRY_NAME]" ) ); @@ -286,12 +291,12 @@ public void testBuildIllegalFieldOption() { public void testBuildUnsupportedDatabase() throws Exception { // mock up some unsupported database (it has a databaseType that we don't recognize) - GeoIpDatabase database = mock(GeoIpDatabase.class); + IpDatabase database = mock(IpDatabase.class); when(database.getDatabaseType()).thenReturn("some-unsupported-database"); - GeoIpDatabaseProvider provider = mock(GeoIpDatabaseProvider.class); + IpDatabaseProvider provider = mock(IpDatabaseProvider.class); when(provider.getDatabase(anyString())).thenReturn(database); - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(provider); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(GEOIP_TYPE, provider); Map config1 = new HashMap<>(); config1.put("field", "_field"); @@ -305,12 +310,12 @@ public void testBuildUnsupportedDatabase() throws Exception { public void testBuildNullDatabase() throws Exception { // mock up a provider that returns a null databaseType - GeoIpDatabase database = mock(GeoIpDatabase.class); + IpDatabase database = mock(IpDatabase.class); when(database.getDatabaseType()).thenReturn(null); - GeoIpDatabaseProvider provider = mock(GeoIpDatabaseProvider.class); + IpDatabaseProvider provider = mock(IpDatabaseProvider.class); when(provider.getDatabase(anyString())).thenReturn(database); - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(provider); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(GEOIP_TYPE, provider); Map config1 = new HashMap<>(); config1.put("field", "_field"); @@ -319,6 +324,44 @@ public void testBuildNullDatabase() throws Exception { assertThat(e.getMessage(), equalTo("[database_file] Unsupported database type [null] for file [GeoLite2-City.mmdb]")); } + public void testStrictMaxmindSupport() throws Exception { + IpDatabase database = mock(IpDatabase.class); + when(database.getDatabaseType()).thenReturn("ipinfo some_ipinfo_database.mmdb-City"); + IpDatabaseProvider provider = mock(IpDatabaseProvider.class); + when(provider.getDatabase(anyString())).thenReturn(database); + + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(GEOIP_TYPE, provider); + + Map config1 = new HashMap<>(); + config1.put("database_file", "some-ipinfo-database.mmdb"); + config1.put("field", "_field"); + config1.put("properties", List.of("ip")); + Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config1)); + assertThat( + e.getMessage(), + equalTo( + "[database_file] Unsupported database type [ipinfo some_ipinfo_database.mmdb-City] " + + "for file [some-ipinfo-database.mmdb]" + ) + ); + } + + public void testLaxMaxmindSupport() throws Exception { + IpDatabase database = mock(IpDatabase.class); + when(database.getDatabaseType()).thenReturn("some_custom_database.mmdb-City"); + IpDatabaseProvider provider = mock(IpDatabaseProvider.class); + when(provider.getDatabase(anyString())).thenReturn(database); + + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(GEOIP_TYPE, provider); + + Map config1 = new HashMap<>(); + config1.put("database_file", "some-custom-database.mmdb"); + config1.put("field", "_field"); + config1.put("properties", List.of("ip")); + factory.create(null, null, null, config1); + assertWarnings(GeoIpProcessor.UNSUPPORTED_DATABASE_DEPRECATION_MESSAGE.replaceAll("\\{}", "some_custom_database.mmdb-City")); + } + public void testLazyLoading() throws Exception { final Path configDir = createTempDir(); final Path geoIpConfigDir = configDir.resolve("ingest-geoip"); @@ -339,7 +382,7 @@ public void testLazyLoading() throws Exception { Runnable::run, clusterService ); - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(GEOIP_TYPE, databaseNodeService); for (DatabaseReaderLazyLoader lazyLoader : configDatabases.getConfigDatabases().values()) { assertNull(lazyLoader.databaseReader.get()); } @@ -408,7 +451,7 @@ public void testLoadingCustomDatabase() throws IOException { clusterService ); databaseNodeService.initialize("nodeId", resourceWatcherService, mock(IngestService.class)); - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(GEOIP_TYPE, databaseNodeService); for (DatabaseReaderLazyLoader lazyLoader : configDatabases.getConfigDatabases().values()) { assertNull(lazyLoader.databaseReader.get()); } @@ -431,7 +474,7 @@ public void testLoadingCustomDatabase() throws IOException { } public void testFallbackUsingDefaultDatabases() throws Exception { - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(GEOIP_TYPE, databaseNodeService); Map config = new HashMap<>(); config.put("field", "source_field"); config.put("fallback_to_default_databases", randomBoolean()); @@ -440,7 +483,7 @@ public void testFallbackUsingDefaultDatabases() throws Exception { } public void testDownloadDatabaseOnPipelineCreation() throws IOException { - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(GEOIP_TYPE, databaseNodeService); Map config = new HashMap<>(); config.put("field", randomIdentifier()); config.put("download_database_on_pipeline_creation", randomBoolean()); @@ -458,7 +501,7 @@ public void testDefaultDatabaseWithTaskPresent() throws Exception { .metadata(Metadata.builder().putCustom(PersistentTasksCustomMetadata.TYPE, tasks)) .build(); when(clusterService.state()).thenReturn(clusterState); - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(GEOIP_TYPE, databaseNodeService); Map config = new HashMap<>(); config.put("field", "_field"); @@ -470,7 +513,7 @@ public void testDefaultDatabaseWithTaskPresent() throws Exception { } public void testUpdateDatabaseWhileIngesting() throws Exception { - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(GEOIP_TYPE, databaseNodeService); Map config = new HashMap<>(); config.put("field", "source_field"); GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config); @@ -509,7 +552,7 @@ public void testUpdateDatabaseWhileIngesting() throws Exception { } public void testDatabaseNotReadyYet() throws Exception { - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(GEOIP_TYPE, databaseNodeService); cleanDatabases(geoIpConfigDir, configDatabases); { diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java index 762818a7c65db..4548e92239ce1 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java @@ -1,107 +1,126 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; -import com.maxmind.geoip2.DatabaseReader; - import org.elasticsearch.common.CheckedSupplier; -import org.elasticsearch.common.util.set.Sets; -import org.elasticsearch.core.PathUtils; +import org.elasticsearch.core.IOUtils; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.RandomDocumentPicks; -import org.elasticsearch.ingest.geoip.Database.Property; import org.elasticsearch.test.ESTestCase; +import org.junit.After; +import org.junit.Before; import java.io.IOException; -import java.io.InputStream; +import java.nio.file.Path; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Supplier; import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument; +import static org.elasticsearch.ingest.geoip.GeoIpProcessor.GEOIP_TYPE; +import static org.elasticsearch.ingest.geoip.GeoIpProcessor.IP_LOCATION_TYPE; +import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; public class GeoIpProcessorTests extends ESTestCase { - private static final Set ALL_PROPERTIES = Set.of(Property.values()); + // a temporary directory that mmdb files can be copied to and read from + private Path tmpDir; + + @Before + public void setup() { + tmpDir = createTempDir(); + } + + @After + public void cleanup() throws IOException { + IOUtils.rm(tmpDir); + } - public void testDatabasePropertyInvariants() { - // the city database is like a specialization of the country database - assertThat(Sets.difference(Database.Country.properties(), Database.City.properties()), is(empty())); - assertThat(Sets.difference(Database.Country.defaultProperties(), Database.City.defaultProperties()), is(empty())); + public void testMaxmindCity() throws Exception { + String ip = "2602:306:33d3:8000::3257:9652"; + GeoIpProcessor processor = new GeoIpProcessor( + GEOIP_TYPE, // n.b. this is a "geoip" processor + randomAlphaOfLength(10), + null, + "source_field", + loader("GeoLite2-City.mmdb"), + () -> true, + "target_field", + getMaxmindCityLookup(), + false, + false, + "filename" + ); - // the isp database is like a specialization of the asn database - assertThat(Sets.difference(Database.Asn.properties(), Database.Isp.properties()), is(empty())); - assertThat(Sets.difference(Database.Asn.defaultProperties(), Database.Isp.defaultProperties()), is(empty())); + Map document = new HashMap<>(); + document.put("source_field", ip); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + processor.execute(ingestDocument); - // the enterprise database is like everything joined together - for (Database type : Database.values()) { - assertThat(Sets.difference(type.properties(), Database.Enterprise.properties()), is(empty())); - } - // but in terms of the default fields, it's like a drop-in replacement for the city database - // n.b. this is just a choice we decided to make here at Elastic - assertThat(Database.Enterprise.defaultProperties(), equalTo(Database.City.defaultProperties())); + assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); + @SuppressWarnings("unchecked") + Map data = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); + assertThat(data, notNullValue()); + assertThat(data.get("ip"), equalTo(ip)); + assertThat(data.get("city_name"), equalTo("Homestead")); + // see MaxmindIpDataLookupsTests for more tests of the data lookup behavior } - public void testCity() throws Exception { + public void testIpinfoGeolocation() throws Exception { + String ip = "72.20.12.220"; GeoIpProcessor processor = new GeoIpProcessor( + IP_LOCATION_TYPE, // n.b. this is an "ip_location" processor randomAlphaOfLength(10), null, "source_field", - loader("/GeoLite2-City.mmdb"), + loader("ipinfo/ip_geolocation_standard_sample.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + getIpinfoGeolocationLookup(), false, false, "filename" ); Map document = new HashMap<>(); - document.put("source_field", "8.8.8.8"); + document.put("source_field", ip); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); processor.execute(ingestDocument); - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo("8.8.8.8")); + assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData.size(), equalTo(7)); - assertThat(geoData.get("ip"), equalTo("8.8.8.8")); - assertThat(geoData.get("country_iso_code"), equalTo("US")); - assertThat(geoData.get("country_name"), equalTo("United States")); - assertThat(geoData.get("continent_code"), equalTo("NA")); - assertThat(geoData.get("continent_name"), equalTo("North America")); - assertThat(geoData.get("timezone"), equalTo("America/Chicago")); - Map location = new HashMap<>(); - location.put("lat", 37.751d); - location.put("lon", -97.822d); - assertThat(geoData.get("location"), equalTo(location)); + Map data = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); + assertThat(data, notNullValue()); + assertThat(data.get("ip"), equalTo(ip)); + assertThat(data.get("city_name"), equalTo("Chicago")); + // see IpinfoIpDataLookupsTests for more tests of the data lookup behavior } public void testNullValueWithIgnoreMissing() throws Exception { GeoIpProcessor processor = new GeoIpProcessor( + GEOIP_TYPE, randomAlphaOfLength(10), null, "source_field", - loader("/GeoLite2-City.mmdb"), + loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + getMaxmindCityLookup(), true, false, "filename" @@ -117,13 +136,14 @@ public void testNullValueWithIgnoreMissing() throws Exception { public void testNonExistentWithIgnoreMissing() throws Exception { GeoIpProcessor processor = new GeoIpProcessor( + GEOIP_TYPE, randomAlphaOfLength(10), null, "source_field", - loader("/GeoLite2-City.mmdb"), + loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + getMaxmindCityLookup(), true, false, "filename" @@ -136,13 +156,14 @@ public void testNonExistentWithIgnoreMissing() throws Exception { public void testNullWithoutIgnoreMissing() { GeoIpProcessor processor = new GeoIpProcessor( + GEOIP_TYPE, randomAlphaOfLength(10), null, "source_field", - loader("/GeoLite2-City.mmdb"), + loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + getMaxmindCityLookup(), false, false, "filename" @@ -158,13 +179,14 @@ public void testNullWithoutIgnoreMissing() { public void testNonExistentWithoutIgnoreMissing() { GeoIpProcessor processor = new GeoIpProcessor( + GEOIP_TYPE, randomAlphaOfLength(10), null, "source_field", - loader("/GeoLite2-City.mmdb"), + loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + getMaxmindCityLookup(), false, false, "filename" @@ -175,342 +197,16 @@ public void testNonExistentWithoutIgnoreMissing() { assertThat(exception.getMessage(), equalTo("field [source_field] not present as part of path [source_field]")); } - public void testCity_withIpV6() throws Exception { - GeoIpProcessor processor = new GeoIpProcessor( - randomAlphaOfLength(10), - null, - "source_field", - loader("/GeoLite2-City.mmdb"), - () -> true, - "target_field", - ALL_PROPERTIES, - false, - false, - "filename" - ); - - String address = "2602:306:33d3:8000::3257:9652"; - Map document = new HashMap<>(); - document.put("source_field", address); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(address)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData.size(), equalTo(10)); - assertThat(geoData.get("ip"), equalTo(address)); - assertThat(geoData.get("country_iso_code"), equalTo("US")); - assertThat(geoData.get("country_name"), equalTo("United States")); - assertThat(geoData.get("continent_code"), equalTo("NA")); - assertThat(geoData.get("continent_name"), equalTo("North America")); - assertThat(geoData.get("region_iso_code"), equalTo("US-FL")); - assertThat(geoData.get("region_name"), equalTo("Florida")); - assertThat(geoData.get("city_name"), equalTo("Homestead")); - assertThat(geoData.get("timezone"), equalTo("America/New_York")); - Map location = new HashMap<>(); - location.put("lat", 25.4573d); - location.put("lon", -80.4572d); - assertThat(geoData.get("location"), equalTo(location)); - } - - public void testCityWithMissingLocation() throws Exception { - GeoIpProcessor processor = new GeoIpProcessor( - randomAlphaOfLength(10), - null, - "source_field", - loader("/GeoLite2-City.mmdb"), - () -> true, - "target_field", - ALL_PROPERTIES, - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", "80.231.5.0"); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo("80.231.5.0")); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData.size(), equalTo(1)); - assertThat(geoData.get("ip"), equalTo("80.231.5.0")); - } - - public void testCountry() throws Exception { - GeoIpProcessor processor = new GeoIpProcessor( - randomAlphaOfLength(10), - null, - "source_field", - loader("/GeoLite2-Country.mmdb"), - () -> true, - "target_field", - ALL_PROPERTIES, - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", "82.170.213.79"); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo("82.170.213.79")); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData.size(), equalTo(5)); - assertThat(geoData.get("ip"), equalTo("82.170.213.79")); - assertThat(geoData.get("country_iso_code"), equalTo("NL")); - assertThat(geoData.get("country_name"), equalTo("Netherlands")); - assertThat(geoData.get("continent_code"), equalTo("EU")); - assertThat(geoData.get("continent_name"), equalTo("Europe")); - } - - public void testCountryWithMissingLocation() throws Exception { - GeoIpProcessor processor = new GeoIpProcessor( - randomAlphaOfLength(10), - null, - "source_field", - loader("/GeoLite2-Country.mmdb"), - () -> true, - "target_field", - ALL_PROPERTIES, - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", "80.231.5.0"); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo("80.231.5.0")); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData.size(), equalTo(1)); - assertThat(geoData.get("ip"), equalTo("80.231.5.0")); - } - - public void testAsn() throws Exception { - String ip = "82.171.64.0"; - GeoIpProcessor processor = new GeoIpProcessor( - randomAlphaOfLength(10), - null, - "source_field", - loader("/GeoLite2-ASN.mmdb"), - () -> true, - "target_field", - ALL_PROPERTIES, - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", ip); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData.size(), equalTo(4)); - assertThat(geoData.get("ip"), equalTo(ip)); - assertThat(geoData.get("asn"), equalTo(1136L)); - assertThat(geoData.get("organization_name"), equalTo("KPN B.V.")); - assertThat(geoData.get("network"), equalTo("82.168.0.0/14")); - } - - public void testAnonymmousIp() throws Exception { - String ip = "81.2.69.1"; - GeoIpProcessor processor = new GeoIpProcessor( - randomAlphaOfLength(10), - null, - "source_field", - loader("/GeoIP2-Anonymous-IP-Test.mmdb"), - () -> true, - "target_field", - ALL_PROPERTIES, - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", ip); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData.size(), equalTo(7)); - assertThat(geoData.get("ip"), equalTo(ip)); - assertThat(geoData.get("hosting_provider"), equalTo(true)); - assertThat(geoData.get("tor_exit_node"), equalTo(true)); - assertThat(geoData.get("anonymous_vpn"), equalTo(true)); - assertThat(geoData.get("anonymous"), equalTo(true)); - assertThat(geoData.get("public_proxy"), equalTo(true)); - assertThat(geoData.get("residential_proxy"), equalTo(true)); - } - - public void testConnectionType() throws Exception { - String ip = "214.78.120.5"; - GeoIpProcessor processor = new GeoIpProcessor( - randomAlphaOfLength(10), - null, - "source_field", - loader("/GeoIP2-Connection-Type-Test.mmdb"), - () -> true, - "target_field", - ALL_PROPERTIES, - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", ip); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData.size(), equalTo(2)); - assertThat(geoData.get("ip"), equalTo(ip)); - assertThat(geoData.get("connection_type"), equalTo("Satellite")); - } - - public void testDomain() throws Exception { - String ip = "69.219.64.2"; - GeoIpProcessor processor = new GeoIpProcessor( - randomAlphaOfLength(10), - null, - "source_field", - loader("/GeoIP2-Domain-Test.mmdb"), - () -> true, - "target_field", - ALL_PROPERTIES, - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", ip); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData.size(), equalTo(2)); - assertThat(geoData.get("ip"), equalTo(ip)); - assertThat(geoData.get("domain"), equalTo("ameritech.net")); - } - - public void testEnterprise() throws Exception { - String ip = "74.209.24.4"; - GeoIpProcessor processor = new GeoIpProcessor( - randomAlphaOfLength(10), - null, - "source_field", - loader("/GeoIP2-Enterprise-Test.mmdb"), - () -> true, - "target_field", - ALL_PROPERTIES, - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", ip); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData.size(), equalTo(24)); - assertThat(geoData.get("ip"), equalTo(ip)); - assertThat(geoData.get("country_iso_code"), equalTo("US")); - assertThat(geoData.get("country_name"), equalTo("United States")); - assertThat(geoData.get("continent_code"), equalTo("NA")); - assertThat(geoData.get("continent_name"), equalTo("North America")); - assertThat(geoData.get("region_iso_code"), equalTo("US-NY")); - assertThat(geoData.get("region_name"), equalTo("New York")); - assertThat(geoData.get("city_name"), equalTo("Chatham")); - assertThat(geoData.get("timezone"), equalTo("America/New_York")); - Map location = new HashMap<>(); - location.put("lat", 42.3478); - location.put("lon", -73.5549); - assertThat(geoData.get("location"), equalTo(location)); - assertThat(geoData.get("asn"), equalTo(14671L)); - assertThat(geoData.get("organization_name"), equalTo("FairPoint Communications")); - assertThat(geoData.get("network"), equalTo("74.209.16.0/20")); - assertThat(geoData.get("hosting_provider"), equalTo(false)); - assertThat(geoData.get("tor_exit_node"), equalTo(false)); - assertThat(geoData.get("anonymous_vpn"), equalTo(false)); - assertThat(geoData.get("anonymous"), equalTo(false)); - assertThat(geoData.get("public_proxy"), equalTo(false)); - assertThat(geoData.get("residential_proxy"), equalTo(false)); - assertThat(geoData.get("domain"), equalTo("frpt.net")); - assertThat(geoData.get("isp"), equalTo("Fairpoint Communications")); - assertThat(geoData.get("isp_organization_name"), equalTo("Fairpoint Communications")); - assertThat(geoData.get("user_type"), equalTo("residential")); - assertThat(geoData.get("connection_type"), equalTo("Cable/DSL")); - } - - public void testIsp() throws Exception { - String ip = "149.101.100.1"; - GeoIpProcessor processor = new GeoIpProcessor( - randomAlphaOfLength(10), - null, - "source_field", - loader("/GeoIP2-ISP-Test.mmdb"), - () -> true, - "target_field", - ALL_PROPERTIES, - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", ip); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData.size(), equalTo(8)); - assertThat(geoData.get("ip"), equalTo(ip)); - assertThat(geoData.get("asn"), equalTo(6167L)); - assertThat(geoData.get("organization_name"), equalTo("CELLCO-PART")); - assertThat(geoData.get("network"), equalTo("149.101.100.0/28")); - assertThat(geoData.get("isp"), equalTo("Verizon Wireless")); - assertThat(geoData.get("isp_organization_name"), equalTo("Verizon Wireless")); - assertThat(geoData.get("mobile_network_code"), equalTo("004")); - assertThat(geoData.get("mobile_country_code"), equalTo("310")); - } - public void testAddressIsNotInTheDatabase() throws Exception { GeoIpProcessor processor = new GeoIpProcessor( + GEOIP_TYPE, randomAlphaOfLength(10), null, "source_field", - loader("/GeoLite2-City.mmdb"), + loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + getMaxmindCityLookup(), false, false, "filename" @@ -524,17 +220,18 @@ public void testAddressIsNotInTheDatabase() throws Exception { } /** - * Don't silently do DNS lookups or anything trappy on bogus data + * Tests that an exception in the IpDataLookup is propagated out of the GeoIpProcessor's execute method */ - public void testInvalid() { + public void testExceptionPropagates() { GeoIpProcessor processor = new GeoIpProcessor( + GEOIP_TYPE, randomAlphaOfLength(10), null, "source_field", - loader("/GeoLite2-City.mmdb"), + loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + getMaxmindCityLookup(), false, false, "filename" @@ -549,13 +246,14 @@ public void testInvalid() { public void testListAllValid() throws Exception { GeoIpProcessor processor = new GeoIpProcessor( + GEOIP_TYPE, randomAlphaOfLength(10), null, "source_field", - loader("/GeoLite2-City.mmdb"), + loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + getMaxmindCityLookup(), false, false, "filename" @@ -567,25 +265,23 @@ public void testListAllValid() throws Exception { processor.execute(ingestDocument); @SuppressWarnings("unchecked") - List> geoData = (List>) ingestDocument.getSourceAndMetadata().get("target_field"); - - Map location = new HashMap<>(); - location.put("lat", 37.751d); - location.put("lon", -97.822d); - assertThat(geoData.get(0).get("location"), equalTo(location)); - - assertThat(geoData.get(1).get("city_name"), equalTo("Hoensbroek")); + List> data = (List>) ingestDocument.getSourceAndMetadata().get("target_field"); + assertThat(data, notNullValue()); + assertThat(data.size(), equalTo(2)); + assertThat(data.get(0).get("location"), equalTo(Map.of("lat", 37.751d, "lon", -97.822d))); + assertThat(data.get(1).get("city_name"), equalTo("Hoensbroek")); } public void testListPartiallyValid() throws Exception { GeoIpProcessor processor = new GeoIpProcessor( + GEOIP_TYPE, randomAlphaOfLength(10), null, "source_field", - loader("/GeoLite2-City.mmdb"), + loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + getMaxmindCityLookup(), false, false, "filename" @@ -597,25 +293,23 @@ public void testListPartiallyValid() throws Exception { processor.execute(ingestDocument); @SuppressWarnings("unchecked") - List> geoData = (List>) ingestDocument.getSourceAndMetadata().get("target_field"); - - Map location = new HashMap<>(); - location.put("lat", 37.751d); - location.put("lon", -97.822d); - assertThat(geoData.get(0).get("location"), equalTo(location)); - - assertThat(geoData.get(1), nullValue()); + List> data = (List>) ingestDocument.getSourceAndMetadata().get("target_field"); + assertThat(data, notNullValue()); + assertThat(data.size(), equalTo(2)); + assertThat(data.get(0).get("location"), equalTo(Map.of("lat", 37.751d, "lon", -97.822d))); + assertThat(data.get(1), nullValue()); } public void testListNoMatches() throws Exception { GeoIpProcessor processor = new GeoIpProcessor( + GEOIP_TYPE, randomAlphaOfLength(10), null, "source_field", - loader("/GeoLite2-City.mmdb"), + loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + getMaxmindCityLookup(), false, false, "filename" @@ -631,11 +325,11 @@ public void testListNoMatches() throws Exception { public void testListDatabaseReferenceCounting() throws Exception { AtomicBoolean closeCheck = new AtomicBoolean(false); - var loader = loader("/GeoLite2-City.mmdb", closeCheck); - GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), null, "source_field", () -> { + var loader = loader("GeoLite2-City.mmdb", closeCheck); + GeoIpProcessor processor = new GeoIpProcessor(GEOIP_TYPE, randomAlphaOfLength(10), null, "source_field", () -> { loader.preLookup(); return loader; - }, () -> true, "target_field", ALL_PROPERTIES, false, false, "filename"); + }, () -> true, "target_field", getMaxmindCityLookup(), false, false, "filename"); Map document = new HashMap<>(); document.put("source_field", List.of("8.8.8.8", "82.171.64.0")); @@ -643,30 +337,28 @@ public void testListDatabaseReferenceCounting() throws Exception { processor.execute(ingestDocument); @SuppressWarnings("unchecked") - List> geoData = (List>) ingestDocument.getSourceAndMetadata().get("target_field"); - - Map location = new HashMap<>(); - location.put("lat", 37.751d); - location.put("lon", -97.822d); - assertThat(geoData.get(0).get("location"), equalTo(location)); - - assertThat(geoData.get(1).get("city_name"), equalTo("Hoensbroek")); + List> data = (List>) ingestDocument.getSourceAndMetadata().get("target_field"); + assertThat(data, notNullValue()); + assertThat(data.size(), equalTo(2)); + assertThat(data.get(0).get("location"), equalTo(Map.of("lat", 37.751d, "lon", -97.822d))); + assertThat(data.get(1).get("city_name"), equalTo("Hoensbroek")); // Check the loader's reference count and attempt to close assertThat(loader.current(), equalTo(0)); - loader.close(); + loader.shutdown(); assertTrue(closeCheck.get()); } public void testListFirstOnly() throws Exception { GeoIpProcessor processor = new GeoIpProcessor( + GEOIP_TYPE, randomAlphaOfLength(10), null, "source_field", - loader("/GeoLite2-City.mmdb"), + loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + getMaxmindCityLookup(), false, true, "filename" @@ -678,23 +370,21 @@ public void testListFirstOnly() throws Exception { processor.execute(ingestDocument); @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - - Map location = new HashMap<>(); - location.put("lat", 37.751d); - location.put("lon", -97.822d); - assertThat(geoData.get("location"), equalTo(location)); + Map data = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); + assertThat(data, notNullValue()); + assertThat(data.get("location"), equalTo(Map.of("lat", 37.751d, "lon", -97.822d))); } public void testListFirstOnlyNoMatches() throws Exception { GeoIpProcessor processor = new GeoIpProcessor( + GEOIP_TYPE, randomAlphaOfLength(10), null, "source_field", - loader("/GeoLite2-City.mmdb"), + loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + getMaxmindCityLookup(), false, true, "filename" @@ -710,13 +400,14 @@ public void testListFirstOnlyNoMatches() throws Exception { public void testInvalidDatabase() throws Exception { GeoIpProcessor processor = new GeoIpProcessor( + GEOIP_TYPE, randomAlphaOfLength(10), null, "source_field", - loader("/GeoLite2-City.mmdb"), + loader("GeoLite2-City.mmdb"), () -> false, "target_field", - ALL_PROPERTIES, + getMaxmindCityLookup(), false, true, "filename" @@ -733,13 +424,14 @@ public void testInvalidDatabase() throws Exception { public void testNoDatabase() throws Exception { GeoIpProcessor processor = new GeoIpProcessor( + GEOIP_TYPE, randomAlphaOfLength(10), null, "source_field", () -> null, () -> true, "target_field", - ALL_PROPERTIES, + getMaxmindCityLookup(), false, false, "GeoLite2-City" @@ -756,13 +448,14 @@ public void testNoDatabase() throws Exception { public void testNoDatabase_ignoreMissing() throws Exception { GeoIpProcessor processor = new GeoIpProcessor( + GEOIP_TYPE, randomAlphaOfLength(10), null, "source_field", () -> null, () -> true, "target_field", - ALL_PROPERTIES, + getMaxmindCityLookup(), true, false, "GeoLite2-City" @@ -776,43 +469,34 @@ public void testNoDatabase_ignoreMissing() throws Exception { assertIngestDocument(originalIngestDocument, ingestDocument); } - private CheckedSupplier loader(final String path) { - var loader = loader(path, null); - return () -> loader; + private static IpDataLookup getMaxmindCityLookup() { + final var database = Database.City; + return MaxmindIpDataLookups.getMaxmindLookup(database).apply(database.properties()); } - private DatabaseReaderLazyLoader loader(final String path, final AtomicBoolean closed) { - final Supplier databaseInputStreamSupplier = () -> GeoIpProcessor.class.getResourceAsStream(path); - final CheckedSupplier loader = () -> new DatabaseReader.Builder(databaseInputStreamSupplier.get()) - .build(); - final GeoIpCache cache = new GeoIpCache(1000); - return new DatabaseReaderLazyLoader(cache, PathUtils.get(path), null, loader) { + private static IpDataLookup getIpinfoGeolocationLookup() { + final var database = Database.CityV2; + return IpinfoIpDataLookups.getIpinfoLookup(database).apply(database.properties()); + } - @Override - long databaseFileSize() throws IOException { - try (InputStream is = databaseInputStreamSupplier.get()) { - long bytesRead = 0; - do { - final byte[] bytes = new byte[1 << 10]; - final int read = is.read(bytes); - if (read == -1) break; - bytesRead += read; - } while (true); - return bytesRead; - } - } + private CheckedSupplier loader(final String path) { + var loader = loader(path, null); + return () -> loader; + } - @Override - InputStream databaseInputStream() { - return databaseInputStreamSupplier.get(); - } + private DatabaseReaderLazyLoader loader(final String databaseName, final AtomicBoolean closed) { + int last = databaseName.lastIndexOf("/"); + final Path path = tmpDir.resolve(last == -1 ? databaseName : databaseName.substring(last + 1)); + copyDatabase(databaseName, path); + final GeoIpCache cache = new GeoIpCache(1000); + return new DatabaseReaderLazyLoader(cache, path, null) { @Override - protected void doClose() throws IOException { + protected void doShutdown() throws IOException { if (closed != null) { closed.set(true); } - super.doClose(); + super.doShutdown(); } }; } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTaskStateSerializationTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTaskStateSerializationTests.java index 80e4836605e23..cd81ed051491d 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTaskStateSerializationTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTaskStateSerializationTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTestUtils.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTestUtils.java index a3d72aca2295c..160671fd39001 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTestUtils.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTestUtils.java @@ -1,13 +1,21 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; +import com.maxmind.db.DatabaseRecord; +import com.maxmind.db.Reader; +import com.maxmind.geoip2.model.CityResponse; +import com.maxmind.geoip2.model.CountryResponse; + +import org.elasticsearch.common.CheckedBiFunction; +import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.core.SuppressForbidden; import java.io.FileNotFoundException; @@ -16,6 +24,7 @@ import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; import java.util.Set; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; @@ -57,4 +66,28 @@ public static void copyDefaultDatabases(final Path directory, ConfigDatabases co configDatabases.updateDatabase(directory.resolve(database), true); } } + + /** + * A static city-specific responseProvider for use with {@link IpDatabase#getResponse(String, CheckedBiFunction)} in + * tests. + *

    + * Like this: {@code CityResponse city = loader.getResponse("some.ip.address", GeoIpTestUtils::getCity);} + */ + public static CityResponse getCity(Reader reader, String ip) throws IOException { + DatabaseRecord record = reader.getRecord(InetAddresses.forString(ip), CityResponse.class); + CityResponse data = record.getData(); + return data == null ? null : new CityResponse(data, ip, record.getNetwork(), List.of("en")); + } + + /** + * A static country-specific responseProvider for use with {@link IpDatabase#getResponse(String, CheckedBiFunction)} in + * tests. + *

    + * Like this: {@code CountryResponse country = loader.getResponse("some.ip.address", GeoIpTestUtils::getCountry);} + */ + public static CountryResponse getCountry(Reader reader, String ip) throws IOException { + DatabaseRecord record = reader.getRecord(InetAddresses.forString(ip), CountryResponse.class); + CountryResponse data = record.getData(); + return data == null ? null : new CountryResponse(data, ip, record.getNetwork(), List.of("en")); + } } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/HttpClientTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/HttpClientTests.java index a3f4395d74755..f4a3cfbde4f4c 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/HttpClientTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/HttpClientTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; @@ -46,6 +47,7 @@ public static void startServer() throws Throwable { server.createContext("/404/", exchange -> { try { exchange.sendResponseHeaders(404, 0); + exchange.close(); } catch (Exception e) { fail(e); } @@ -101,6 +103,7 @@ public boolean checkCredentials(String username, String password) { exchange.getResponseHeaders().add("Location", "/" + destination + "/"); } exchange.sendResponseHeaders(302, 0); + exchange.close(); } catch (Exception e) { fail(e); } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadataTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadataTests.java index eca23cb13cd3d..6a98cd532604b 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadataTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadataTests.java @@ -1,13 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration; import org.elasticsearch.ingest.geoip.direct.DatabaseConfigurationMetadata; @@ -20,6 +22,12 @@ import java.util.Map; public class IngestGeoIpMetadataTests extends AbstractChunkedSerializingTestCase { + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(new IngestGeoIpPlugin().getNamedWriteables()); + } + @Override protected IngestGeoIpMetadata doParseInstance(XContentParser parser) throws IOException { return IngestGeoIpMetadata.fromXContent(parser); diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java new file mode 100644 index 0000000000000..11aa123824d18 --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java @@ -0,0 +1,564 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import com.maxmind.db.DatabaseRecord; +import com.maxmind.db.Networks; +import com.maxmind.db.Reader; + +import org.apache.lucene.util.Constants; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.core.IOUtils; +import org.elasticsearch.core.Strings; +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.test.ESTestCase; +import org.junit.After; +import org.junit.Before; + +import java.io.File; +import java.io.IOException; +import java.net.InetAddress; +import java.nio.file.Path; +import java.util.Map; +import java.util.Set; +import java.util.function.BiConsumer; + +import static java.util.Map.entry; +import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase; +import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.ipinfoTypeCleanup; +import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.parseAsn; +import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.parseBoolean; +import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.parseLocationDouble; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.startsWith; + +public class IpinfoIpDataLookupsTests extends ESTestCase { + + // a temporary directory that mmdb files can be copied to and read from + private Path tmpDir; + + @Before + public void setup() { + tmpDir = createTempDir(); + } + + @After + public void cleanup() throws IOException { + IOUtils.rm(tmpDir); + } + + public void testParseAsn() { + // expected case: "AS123" is 123 + assertThat(parseAsn("AS123"), equalTo(123L)); + // defensive cases: null and empty becomes null, this is not expected fwiw + assertThat(parseAsn(null), nullValue()); + assertThat(parseAsn(""), nullValue()); + // defensive cases: we strip whitespace and ignore case + assertThat(parseAsn(" as 456 "), equalTo(456L)); + // defensive cases: we ignore the absence of the 'AS' prefix + assertThat(parseAsn("123"), equalTo(123L)); + // bottom case: a non-parsable string is null + assertThat(parseAsn("anythingelse"), nullValue()); + } + + public void testParseBoolean() { + // expected cases: "true" is true and "" is false + assertThat(parseBoolean("true"), equalTo(true)); + assertThat(parseBoolean(""), equalTo(false)); + assertThat(parseBoolean("false"), equalTo(false)); // future proofing + // defensive case: null becomes null, this is not expected fwiw + assertThat(parseBoolean(null), nullValue()); + // defensive cases: we strip whitespace and ignore case + assertThat(parseBoolean(" "), equalTo(false)); + assertThat(parseBoolean(" TrUe "), equalTo(true)); + assertThat(parseBoolean(" FaLSE "), equalTo(false)); + // bottom case: a non-parsable string is null + assertThat(parseBoolean(randomAlphaOfLength(8)), nullValue()); + } + + public void testParseLocationDouble() { + // expected case: "123.45" is 123.45 + assertThat(parseLocationDouble("123.45"), equalTo(123.45)); + // defensive cases: null and empty becomes null, this is not expected fwiw + assertThat(parseLocationDouble(null), nullValue()); + assertThat(parseLocationDouble(""), nullValue()); + // defensive cases: we strip whitespace + assertThat(parseLocationDouble(" -123.45 "), equalTo(-123.45)); + // bottom case: a non-parsable string is null + assertThat(parseLocationDouble("anythingelse"), nullValue()); + } + + public void testAsnFree() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "ip_asn_sample.mmdb"; + String ip = "23.32.184.0"; + assertExpectedLookupResults( + databaseName, + ip, + new IpinfoIpDataLookups.Asn(Database.AsnV2.properties()), + Map.ofEntries( + entry("ip", ip), + entry("organization_name", "Akamai Technologies, Inc."), + entry("asn", 16625L), + entry("network", "23.32.184.0/21"), + entry("domain", "akamai.com") + ), + Map.ofEntries(entry("name", "organization_name"), entry("asn", "asn"), entry("network", "network"), entry("domain", "domain")), + Set.of("ip"), + Set.of() + ); + } + + public void testAsnStandard() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "asn_sample.mmdb"; + String ip = "69.19.224.0"; + assertExpectedLookupResults( + databaseName, + ip, + new IpinfoIpDataLookups.Asn(Database.AsnV2.properties()), + Map.ofEntries( + entry("ip", ip), + entry("organization_name", "TPx Communications"), + entry("asn", 14265L), + entry("network", "69.19.224.0/22"), + entry("domain", "tpx.com"), + entry("type", "hosting"), + entry("country_iso_code", "US") + ), + Map.ofEntries( + entry("name", "organization_name"), + entry("asn", "asn"), + entry("network", "network"), + entry("domain", "domain"), + entry("country", "country_iso_code"), + entry("type", "type") + ), + Set.of("ip"), + Set.of() + ); + } + + public void testAsnInvariants() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + Path configDir = tmpDir; + copyDatabase("ipinfo/ip_asn_sample.mmdb", configDir.resolve("ip_asn_sample.mmdb")); + copyDatabase("ipinfo/asn_sample.mmdb", configDir.resolve("asn_sample.mmdb")); + + { + final Set expectedColumns = Set.of("network", "asn", "name", "domain"); + + Path databasePath = configDir.resolve("ip_asn_sample.mmdb"); + assertDatabaseInvariants(databasePath, (ip, row) -> { + assertThat(row.keySet(), equalTo(expectedColumns)); + String asn = (String) row.get("asn"); + assertThat(asn, startsWith("AS")); + assertThat(asn, equalTo(asn.trim())); + Long parsed = parseAsn(asn); + assertThat(parsed, notNullValue()); + assertThat(asn, equalTo("AS" + parsed)); // reverse it + }); + } + + { + final Set expectedColumns = Set.of("network", "asn", "name", "domain", "country", "type"); + + Path databasePath = configDir.resolve("asn_sample.mmdb"); + assertDatabaseInvariants(databasePath, (ip, row) -> { + assertThat(row.keySet(), equalTo(expectedColumns)); + String asn = (String) row.get("asn"); + assertThat(asn, startsWith("AS")); + assertThat(asn, equalTo(asn.trim())); + Long parsed = parseAsn(asn); + assertThat(parsed, notNullValue()); + assertThat(asn, equalTo("AS" + parsed)); // reverse it + }); + } + } + + public void testCountryFree() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "ip_country_sample.mmdb"; + String ip = "20.33.76.0"; + assertExpectedLookupResults( + databaseName, + ip, + new IpinfoIpDataLookups.Country(Database.CountryV2.properties()), + Map.ofEntries( + entry("ip", ip), + entry("country_name", "Ireland"), + entry("country_iso_code", "IE"), + entry("continent_name", "Europe"), + entry("continent_code", "EU") + ), + Map.ofEntries( + entry("continent_name", "continent_name"), + entry("continent", "continent_code"), + entry("country", "country_iso_code"), + entry("country_name", "country_name"), + entry("type", "type") + ), + Set.of("ip"), + Set.of("network") + ); + } + + public void testGeolocationStandard() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "ip_geolocation_standard_sample.mmdb"; + String ip = "62.69.48.19"; + assertExpectedLookupResults( + databaseName, + ip, + new IpinfoIpDataLookups.Geolocation(Database.CityV2.properties()), + Map.ofEntries( + entry("ip", ip), + entry("country_iso_code", "GB"), + entry("region_name", "England"), + entry("city_name", "London"), + entry("timezone", "Europe/London"), + entry("postal_code", "E1W"), + entry("location", Map.of("lat", 51.50853, "lon", -0.12574)) + ), + Map.ofEntries( + entry("country", "country_iso_code"), + entry("region", "region_name"), + entry("city", "city_name"), + entry("timezone", "timezone"), + entry("postal_code", "postal_code"), + entry("lat", "location"), + entry("lng", "location") + ), + Set.of("ip", "location"), + Set.of("geoname_id", "region_code") + ); + } + + public void testGeolocationInvariants() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + Path configDir = tmpDir; + copyDatabase("ipinfo/ip_geolocation_standard_sample.mmdb", configDir.resolve("ip_geolocation_standard_sample.mmdb")); + + { + final Set expectedColumns = Set.of( + "city", + "geoname_id", + "region", + "region_code", + "country", + "postal_code", + "timezone", + "lat", + "lng" + ); + + Path databasePath = configDir.resolve("ip_geolocation_standard_sample.mmdb"); + assertDatabaseInvariants(databasePath, (ip, row) -> { + assertThat(row.keySet(), equalTo(expectedColumns)); + { + String latitude = (String) row.get("lat"); + assertThat(latitude, equalTo(latitude.trim())); + Double parsed = parseLocationDouble(latitude); + assertThat(parsed, notNullValue()); + assertThat(Double.parseDouble(latitude), equalTo(Double.parseDouble(Double.toString(parsed)))); // reverse it + } + { + String longitude = (String) row.get("lng"); + assertThat(longitude, equalTo(longitude.trim())); + Double parsed = parseLocationDouble(longitude); + assertThat(parsed, notNullValue()); + assertThat(Double.parseDouble(longitude), equalTo(Double.parseDouble(Double.toString(parsed)))); // reverse it + } + }); + } + } + + public void testPrivacyDetectionStandard() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "privacy_detection_sample.mmdb"; + String ip = "2.57.109.154"; + assertExpectedLookupResults( + databaseName, + ip, + new IpinfoIpDataLookups.PrivacyDetection(Database.PrivacyDetection.properties()), + Map.ofEntries( + entry("ip", ip), + entry("hosting", false), + entry("proxy", false), + entry("relay", false), + entry("tor", false), + entry("vpn", true) + ), + Map.ofEntries( + entry("hosting", "hosting"), + entry("proxy", "proxy"), + entry("relay", "relay"), + entry("tor", "tor"), + entry("vpn", "vpn") + ), + Set.of("ip"), + Set.of("network", "service") + ); + } + + public void testPrivacyDetectionStandardNonEmptyService() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "privacy_detection_sample.mmdb"; + String ip = "59.29.201.246"; + assertExpectedLookupResults( + databaseName, + ip, + new IpinfoIpDataLookups.PrivacyDetection(Database.PrivacyDetection.properties()), + Map.ofEntries( + entry("ip", ip), + entry("hosting", false), + entry("proxy", false), + entry("service", "VPNGate"), + entry("relay", false), + entry("tor", false), + entry("vpn", true) + ), + Map.ofEntries( + entry("hosting", "hosting"), + entry("proxy", "proxy"), + entry("service", "service"), + entry("relay", "relay"), + entry("tor", "tor"), + entry("vpn", "vpn") + ), + Set.of("ip"), + Set.of("network") + ); + } + + public void testPrivacyDetectionInvariants() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + Path configDir = tmpDir; + copyDatabase("ipinfo/privacy_detection_sample.mmdb", configDir.resolve("privacy_detection_sample.mmdb")); + + { + final Set expectedColumns = Set.of("network", "service", "hosting", "proxy", "relay", "tor", "vpn"); + + Path databasePath = configDir.resolve("privacy_detection_sample.mmdb"); + assertDatabaseInvariants(databasePath, (ip, row) -> { + assertThat(row.keySet(), equalTo(expectedColumns)); + + for (String booleanColumn : Set.of("hosting", "proxy", "relay", "tor", "vpn")) { + String bool = (String) row.get(booleanColumn); + assertThat(bool, anyOf(equalTo("true"), equalTo(""), equalTo("false"))); + assertThat(parseBoolean(bool), notNullValue()); + } + }); + } + } + + public void testIpinfoTypeCleanup() { + Map typesToCleanedTypes = Map.ofEntries( + // database_type strings from upstream: + // abuse.mmdb + entry("ipinfo standard_abuse_mmdb_v4.mmdb", "abuse_v4"), + // asn.mmdb + entry("ipinfo generic_asn_mmdb_v4.mmdb", "asn_v4"), + // carrier.mmdb + entry("ipinfo standard_carrier_mmdb.mmdb", "carrier"), + // location_extended_v2.mmdb + entry("ipinfo extended_location_v2.mmdb", "location_v2"), + // privacy_extended_v2.mmdb + entry("ipinfo extended_privacy_v2.mmdb", "privacy_v2"), + // standard_company.mmdb + entry("ipinfo standard_company.mmdb", "company"), + // standard_ip_hosted_domains_sample.mmdb + entry("ipinfo standard_ip_hosted_domains_sample.mmdb", "hosted_domains"), + // standard_location.mmdb + entry("ipinfo standard_location_mmdb_v4.mmdb", "location_v4"), + // standard_privacy.mmdb + entry("ipinfo standard_privacy.mmdb", "privacy"), + + // database_type strings from test files: + // ip_asn_sample.mmdb + entry("ipinfo ip_asn_sample.mmdb", "asn"), + // ip_country_asn_sample.mmdb + entry("ipinfo ip_country_asn_sample.mmdb", "country_asn"), + // ip_geolocation_sample.mmdb + entry("ipinfo ip_geolocation_sample.mmdb", "geolocation"), + // abuse_contact_sample.mmdb + entry("ipinfo abuse_contact_sample.mmdb", "abuse_contact"), + // asn_sample.mmdb + entry("ipinfo asn_sample.mmdb", "asn"), + // hosted_domains_sample.mmdb + entry("ipinfo hosted_domains_sample.mmdb", "hosted_domains"), + // ip_carrier_sample.mmdb + entry("ipinfo ip_carrier_sample.mmdb", "carrier"), + // ip_company_sample.mmdb + entry("ipinfo ip_company_sample.mmdb", "company"), + // ip_country_sample.mmdb + entry("ipinfo ip_country_sample.mmdb", "country"), + // ip_geolocation_extended_ipv4_sample.mmdb + entry("ipinfo ip_geolocation_extended_ipv4_sample.mmdb", "geolocation_ipv4"), + // ip_geolocation_extended_ipv6_sample.mmdb + entry("ipinfo ip_geolocation_extended_ipv6_sample.mmdb", "geolocation_ipv6"), + // ip_geolocation_extended_sample.mmdb + entry("ipinfo ip_geolocation_extended_sample.mmdb", "geolocation"), + // ip_rdns_domains_sample.mmdb + entry("ipinfo ip_rdns_domains_sample.mmdb", "rdns_domains"), + // ip_rdns_hostnames_sample.mmdb + entry("ipinfo ip_rdns_hostnames_sample.mmdb", "rdns_hostnames"), + // privacy_detection_extended_sample.mmdb + entry("ipinfo privacy_detection_extended_sample.mmdb", "privacy_detection"), + // privacy_detection_sample.mmdb + entry("ipinfo privacy_detection_sample.mmdb", "privacy_detection"), + + // database_type strings from downloaded (free) files: + // asn.mmdb + entry("ipinfo generic_asn_free.mmdb", "asn"), + // country.mmdb + entry("ipinfo generic_country_free.mmdb", "country"), + // country_asn.mmdb + entry("ipinfo generic_country_free_country_asn.mmdb", "country_country_asn") + ); + + for (var entry : typesToCleanedTypes.entrySet()) { + String type = entry.getKey(); + String cleanedType = entry.getValue(); + assertThat(ipinfoTypeCleanup(type), equalTo(cleanedType)); + } + } + + public void testDatabaseTypeParsing() throws IOException { + // this test is a little bit overloaded -- it's testing that we're getting the expected sorts of + // database_type strings from these files, *and* it's also testing that we dispatch on those strings + // correctly and associated those files with the correct high-level Elasticsearch Database type. + // down the road it would probably make sense to split these out and find a better home for some of the + // logic, but for now it's probably more valuable to have the test *somewhere* than to get especially + // pedantic about where precisely it should be. + + copyDatabase("ipinfo/ip_asn_sample.mmdb", tmpDir.resolve("ip_asn_sample.mmdb")); + copyDatabase("ipinfo/ip_geolocation_standard_sample.mmdb", tmpDir.resolve("ip_geolocation_standard_sample.mmdb")); + copyDatabase("ipinfo/asn_sample.mmdb", tmpDir.resolve("asn_sample.mmdb")); + copyDatabase("ipinfo/ip_country_sample.mmdb", tmpDir.resolve("ip_country_sample.mmdb")); + copyDatabase("ipinfo/privacy_detection_sample.mmdb", tmpDir.resolve("privacy_detection_sample.mmdb")); + + assertThat(parseDatabaseFromType("ip_asn_sample.mmdb"), is(Database.AsnV2)); + assertThat(parseDatabaseFromType("ip_geolocation_standard_sample.mmdb"), is(Database.CityV2)); + assertThat(parseDatabaseFromType("asn_sample.mmdb"), is(Database.AsnV2)); + assertThat(parseDatabaseFromType("ip_country_sample.mmdb"), is(Database.CountryV2)); + assertThat(parseDatabaseFromType("privacy_detection_sample.mmdb"), is(Database.PrivacyDetection)); + + // additional cases where we're bailing early on types we don't support + assertThat(IpDataLookupFactories.getDatabase("ipinfo ip_country_asn_sample.mmdb"), nullValue()); + assertThat(IpDataLookupFactories.getDatabase("ipinfo privacy_detection_extended_sample.mmdb"), nullValue()); + } + + private Database parseDatabaseFromType(String databaseFile) throws IOException { + return IpDataLookupFactories.getDatabase(MMDBUtil.getDatabaseType(tmpDir.resolve(databaseFile))); + } + + private static void assertDatabaseInvariants(final Path databasePath, final BiConsumer> rowConsumer) { + try (Reader reader = new Reader(pathToFile(databasePath))) { + Networks networks = reader.networks(Map.class); + while (networks.hasNext()) { + DatabaseRecord dbr = networks.next(); + InetAddress address = dbr.getNetwork().getNetworkAddress(); + @SuppressWarnings("unchecked") + Map result = reader.get(address, Map.class); + try { + rowConsumer.accept(address, result); + } catch (AssertionError e) { + fail(e, "Assert failed for address [%s]", NetworkAddress.format(address)); + } catch (Exception e) { + fail(e, "Exception handling address [%s]", NetworkAddress.format(address)); + } + } + } catch (Exception e) { + fail(e); + } + } + + @SuppressForbidden(reason = "Maxmind API requires java.io.File") + private static File pathToFile(Path databasePath) { + return databasePath.toFile(); + } + + private void assertExpectedLookupResults( + String databaseName, + String ip, + IpDataLookup lookup, + Map expected, + Map keyMappings, + Set knownAdditionalKeys, + Set knownMissingKeys + ) { + try (DatabaseReaderLazyLoader loader = loader(databaseName)) { + Map actual = lookup.getData(loader, ip); + assertThat( + "The set of keys in the result are not the same as the set of expected keys", + actual.keySet(), + containsInAnyOrder(expected.keySet().toArray(new String[0])) + ); + for (Map.Entry entry : expected.entrySet()) { + assertThat("Unexpected value for key [" + entry.getKey() + "]", actual.get(entry.getKey()), equalTo(entry.getValue())); + } + assertActualResultsMatchReader(actual, databaseName, ip, keyMappings, knownAdditionalKeys, knownMissingKeys); + } catch (AssertionError e) { + fail(e, "Assert failed for database [%s] with address [%s]", databaseName, ip); + } catch (Exception e) { + fail(e, "Exception for database [%s] with address [%s]", databaseName, ip); + } + } + + private void assertActualResultsMatchReader( + Map actual, + String databaseName, + String ip, + Map keyMappings, + Set knownAdditionalKeys, + Set knownMissingKeys + ) throws IOException { + Path databasePath = tmpDir.resolve(databaseName); + try (Reader reader = new Reader(pathToFile(databasePath))) { + @SuppressWarnings("unchecked") + Map data = reader.get(InetAddresses.forString(ip), Map.class); + for (String key : data.keySet()) { + if (keyMappings.containsKey(key)) { + assertTrue( + Strings.format( + "The reader returned key [%s] that is expected to map to key [%s], but [%s] did not appear in the " + + "actual data", + key, + keyMappings.get(key), + keyMappings.get(key) + ), + actual.containsKey(keyMappings.get(key)) + ); + } else if (knownMissingKeys.contains(key) == false) { + fail(null, "The reader returned unexpected key [%s]", key); + } + } + for (String key : actual.keySet()) { + if (keyMappings.containsValue(key) == false && knownAdditionalKeys.contains(key) == false) { + fail(null, "Unexpected key [%s] in results", key); + } + } + } + } + + private DatabaseReaderLazyLoader loader(final String databaseName) { + Path path = tmpDir.resolve(databaseName); + copyDatabase("ipinfo/" + databaseName, path); // the ipinfo databases are prefixed on the test classpath + final GeoIpCache cache = new GeoIpCache(1000); + return new DatabaseReaderLazyLoader(cache, path, null); + } +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MMDBUtilTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MMDBUtilTests.java new file mode 100644 index 0000000000000..083e9b5bc32da --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MMDBUtilTests.java @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import org.elasticsearch.core.IOUtils; +import org.elasticsearch.test.ESTestCase; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.zip.GZIPOutputStream; + +import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.hasLength; +import static org.hamcrest.Matchers.is; + +public class MMDBUtilTests extends ESTestCase { + + // a temporary directory that mmdb files can be copied to and read from + private Path tmpDir; + + @Before + public void setup() { + tmpDir = createTempDir(); + } + + @After + public void cleanup() throws IOException { + IOUtils.rm(tmpDir); + } + + public void testGetDatabaseTypeGeoIP2City() throws IOException { + Path database = tmpDir.resolve("GeoIP2-City.mmdb"); + copyDatabase("GeoIP2-City-Test.mmdb", database); + + String type = MMDBUtil.getDatabaseType(database); + assertThat(type, is("GeoIP2-City")); + } + + public void testGetDatabaseTypeGeoLite2City() throws IOException { + Path database = tmpDir.resolve("GeoLite2-City.mmdb"); + copyDatabase("GeoLite2-City-Test.mmdb", database); + + String type = MMDBUtil.getDatabaseType(database); + assertThat(type, is("GeoLite2-City")); + } + + public void testSmallFileWithALongDescription() throws IOException { + Path database = tmpDir.resolve("test-description.mmdb"); + copyDatabase("test-description.mmdb", database); + + // it was once the case that we couldn't read a database_type that was 29 characters or longer + String type = MMDBUtil.getDatabaseType(database); + assertThat(type, endsWith("long database_type")); + assertThat(type, hasLength(60)); // 60 is >= 29, ;) + + // it was once the case that we couldn't process an mmdb that was smaller than 512 bytes + assertThat(Files.size(database), is(444L)); // 444 is <512 + } + + public void testIsGzip() throws IOException { + Path database = tmpDir.resolve("GeoLite2-City.mmdb"); + copyDatabase("GeoLite2-City-Test.mmdb", database); + + Path gzipDatabase = tmpDir.resolve("GeoLite2-City.mmdb.gz"); + + // gzip the test mmdb + try (OutputStream out = new GZIPOutputStream(Files.newOutputStream(gzipDatabase))) { + Files.copy(database, out); + } + + assertThat(MMDBUtil.isGzip(database), is(false)); + assertThat(MMDBUtil.isGzip(gzipDatabase), is(true)); + } +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java index eb958ef0ced80..292a7e3c632d3 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; @@ -23,9 +24,11 @@ import com.maxmind.geoip2.record.MaxMind; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.test.ESTestCase; import java.lang.reflect.Method; +import java.lang.reflect.Modifier; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; import java.net.InetAddress; @@ -56,7 +59,7 @@ * - Fail if we add support for a new mmdb file type (enterprise for example) but don't update the test with which fields we do and do not * support. * - Fail if MaxMind adds a new mmdb file type that we don't know about - * - Fail if we expose a MaxMind type through GeoIpDatabase, but don't update the test to know how to handle it + * - Fail if we expose a MaxMind type through IpDatabase, but don't update the test to know how to handle it */ public class MaxMindSupportTests extends ESTestCase { @@ -77,13 +80,19 @@ public class MaxMindSupportTests extends ESTestCase { "city.name", "continent.code", "continent.name", + "country.inEuropeanUnion", "country.isoCode", "country.name", + "location.accuracyRadius", "location.latitude", "location.longitude", "location.timeZone", "mostSpecificSubdivision.isoCode", - "mostSpecificSubdivision.name" + "mostSpecificSubdivision.name", + "postal.code", + "registeredCountry.inEuropeanUnion", + "registeredCountry.isoCode", + "registeredCountry.name" ); private static final Set CITY_UNSUPPORTED_FIELDS = Set.of( "city.confidence", @@ -93,14 +102,12 @@ public class MaxMindSupportTests extends ESTestCase { "continent.names", "country.confidence", "country.geoNameId", - "country.inEuropeanUnion", "country.names", "leastSpecificSubdivision.confidence", "leastSpecificSubdivision.geoNameId", "leastSpecificSubdivision.isoCode", "leastSpecificSubdivision.name", "leastSpecificSubdivision.names", - "location.accuracyRadius", "location.averageIncome", "location.metroCode", "location.populationDensity", @@ -108,13 +115,9 @@ public class MaxMindSupportTests extends ESTestCase { "mostSpecificSubdivision.confidence", "mostSpecificSubdivision.geoNameId", "mostSpecificSubdivision.names", - "postal.code", "postal.confidence", "registeredCountry.confidence", "registeredCountry.geoNameId", - "registeredCountry.inEuropeanUnion", - "registeredCountry.isoCode", - "registeredCountry.name", "registeredCountry.names", "representedCountry.confidence", "representedCountry.geoNameId", @@ -158,23 +161,23 @@ public class MaxMindSupportTests extends ESTestCase { private static final Set COUNTRY_SUPPORTED_FIELDS = Set.of( "continent.name", + "country.inEuropeanUnion", "country.isoCode", "continent.code", - "country.name" + "country.name", + "registeredCountry.inEuropeanUnion", + "registeredCountry.isoCode", + "registeredCountry.name" ); private static final Set COUNTRY_UNSUPPORTED_FIELDS = Set.of( "continent.geoNameId", "continent.names", "country.confidence", "country.geoNameId", - "country.inEuropeanUnion", "country.names", "maxMind", "registeredCountry.confidence", "registeredCountry.geoNameId", - "registeredCountry.inEuropeanUnion", - "registeredCountry.isoCode", - "registeredCountry.name", "registeredCountry.names", "representedCountry.confidence", "representedCountry.geoNameId", @@ -212,16 +215,25 @@ public class MaxMindSupportTests extends ESTestCase { private static final Set DOMAIN_UNSUPPORTED_FIELDS = Set.of("ipAddress", "network"); private static final Set ENTERPRISE_SUPPORTED_FIELDS = Set.of( + "city.confidence", "city.name", "continent.code", "continent.name", + "country.confidence", + "country.inEuropeanUnion", "country.isoCode", "country.name", + "location.accuracyRadius", "location.latitude", "location.longitude", "location.timeZone", "mostSpecificSubdivision.isoCode", "mostSpecificSubdivision.name", + "postal.code", + "postal.confidence", + "registeredCountry.inEuropeanUnion", + "registeredCountry.isoCode", + "registeredCountry.name", "traits.anonymous", "traits.anonymousVpn", "traits.autonomousSystemNumber", @@ -240,21 +252,17 @@ public class MaxMindSupportTests extends ESTestCase { "traits.userType" ); private static final Set ENTERPRISE_UNSUPPORTED_FIELDS = Set.of( - "city.confidence", "city.geoNameId", "city.names", "continent.geoNameId", "continent.names", - "country.confidence", "country.geoNameId", - "country.inEuropeanUnion", "country.names", "leastSpecificSubdivision.confidence", "leastSpecificSubdivision.geoNameId", "leastSpecificSubdivision.isoCode", "leastSpecificSubdivision.name", "leastSpecificSubdivision.names", - "location.accuracyRadius", "location.averageIncome", "location.metroCode", "location.populationDensity", @@ -262,13 +270,8 @@ public class MaxMindSupportTests extends ESTestCase { "mostSpecificSubdivision.confidence", "mostSpecificSubdivision.geoNameId", "mostSpecificSubdivision.names", - "postal.code", - "postal.confidence", "registeredCountry.confidence", "registeredCountry.geoNameId", - "registeredCountry.inEuropeanUnion", - "registeredCountry.isoCode", - "registeredCountry.name", "registeredCountry.names", "representedCountry.confidence", "representedCountry.geoNameId", @@ -360,8 +363,19 @@ public class MaxMindSupportTests extends ESTestCase { private static final Set> KNOWN_UNSUPPORTED_RESPONSE_CLASSES = Set.of(IpRiskResponse.class); + private static final Set KNOWN_UNSUPPORTED_DATABASE_VARIANTS = Set.of( + Database.AsnV2, + Database.CityV2, + Database.CountryV2, + Database.PrivacyDetection + ); + public void testMaxMindSupport() { for (Database databaseType : Database.values()) { + if (KNOWN_UNSUPPORTED_DATABASE_VARIANTS.contains(databaseType)) { + continue; + } + Class maxMindClass = TYPE_TO_MAX_MIND_CLASS.get(databaseType); Set supportedFields = TYPE_TO_SUPPORTED_FIELDS_MAP.get(databaseType); Set unsupportedFields = TYPE_TO_UNSUPPORTED_FIELDS_MAP.get(databaseType); @@ -478,7 +492,7 @@ public void testUsedMaxMindResponseClassesAreAccountedFor() { supportedMaxMindClasses ); assertThat( - "GeoIpDatabase exposes MaxMind response classes that this test does not know what to do with. Add mappings to " + "MaxmindIpDataLookups exposes MaxMind response classes that this test does not know what to do with. Add mappings to " + "TYPE_TO_MAX_MIND_CLASS for the following: " + usedButNotSupportedMaxMindResponseClasses, usedButNotSupportedMaxMindResponseClasses, @@ -617,18 +631,29 @@ private static String getFormattedList(Set fields) { } /* - * This returns all AbstractResponse classes that are returned from getter methods on GeoIpDatabase. + * This returns all AbstractResponse classes that are declared in transform methods in classes defined in MaxmindIpDataLookups. */ + @SuppressWarnings("unchecked") + @SuppressForbidden(reason = "Need declared classes and methods") private static Set> getUsedMaxMindResponseClasses() { Set> result = new HashSet<>(); - Method[] methods = GeoIpDatabase.class.getMethods(); - for (Method method : methods) { - if (method.getName().startsWith("get")) { - Class returnType = method.getReturnType(); - try { - result.add(returnType.asSubclass(AbstractResponse.class)); - } catch (ClassCastException ignore) { - // This is not what we were looking for, move on + Class[] declaredClasses = MaxmindIpDataLookups.class.getDeclaredClasses(); + for (Class declaredClass : declaredClasses) { + if (Modifier.isAbstract(declaredClass.getModifiers())) { + continue; + } + Method[] declaredMethods = declaredClass.getDeclaredMethods(); + Optional nonAbstractTransformMethod = Arrays.stream(declaredMethods) + .filter( + method -> method.getName().equals("transform") + && method.getParameterTypes().length == 1 + && Modifier.isAbstract(method.getParameterTypes()[0].getModifiers()) == false + ) + .findAny(); + if (nonAbstractTransformMethod.isPresent()) { + Class responseClass = nonAbstractTransformMethod.get().getParameterTypes()[0]; + if (AbstractResponse.class.isAssignableFrom(responseClass)) { + result.add((Class) responseClass); } } } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookupsTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookupsTests.java new file mode 100644 index 0000000000000..57ee2191a590d --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookupsTests.java @@ -0,0 +1,339 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import org.apache.lucene.util.Constants; +import org.elasticsearch.core.IOUtils; +import org.elasticsearch.test.ESTestCase; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Map; + +import static java.util.Map.entry; +import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +public class MaxmindIpDataLookupsTests extends ESTestCase { + + // a temporary directory that mmdb files can be copied to and read from + private Path tmpDir; + + @Before + public void setup() { + tmpDir = createTempDir(); + } + + @After + public void cleanup() throws IOException { + IOUtils.rm(tmpDir); + } + + public void testCity() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoLite2-City.mmdb"; + String ip = "8.8.8.8"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.City(Database.City.properties()), + Map.ofEntries( + entry("ip", ip), + entry("country_in_european_union", false), + entry("country_iso_code", "US"), + entry("country_name", "United States"), + entry("continent_code", "NA"), + entry("continent_name", "North America"), + entry("timezone", "America/Chicago"), + entry("location", Map.of("lat", 37.751d, "lon", -97.822d)), + entry("accuracy_radius", 1000), + entry("registered_country_in_european_union", false), + entry("registered_country_iso_code", "US"), + entry("registered_country_name", "United States") + ) + ); + } + + public void testCity_withIpV6() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoLite2-City.mmdb"; + String ip = "2602:306:33d3:8000::3257:9652"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.City(Database.City.properties()), + Map.ofEntries( + entry("ip", ip), + entry("country_in_european_union", false), + entry("country_iso_code", "US"), + entry("country_name", "United States"), + entry("continent_code", "NA"), + entry("continent_name", "North America"), + entry("region_iso_code", "US-FL"), + entry("region_name", "Florida"), + entry("city_name", "Homestead"), + entry("postal_code", "33035"), + entry("timezone", "America/New_York"), + entry("location", Map.of("lat", 25.4573d, "lon", -80.4572d)), + entry("accuracy_radius", 50), + entry("registered_country_in_european_union", false), + entry("registered_country_iso_code", "US"), + entry("registered_country_name", "United States") + ) + ); + } + + public void testCityWithMissingLocation() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoLite2-City.mmdb"; + String ip = "80.231.5.0"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.City(Database.City.properties()), + Map.ofEntries(entry("ip", ip)) + ); + } + + public void testCountry() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoLite2-Country.mmdb"; + String ip = "82.170.213.79"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.Country(Database.Country.properties()), + Map.ofEntries( + entry("ip", ip), + entry("country_in_european_union", true), + entry("country_iso_code", "NL"), + entry("country_name", "Netherlands"), + entry("continent_code", "EU"), + entry("continent_name", "Europe"), + entry("registered_country_in_european_union", true), + entry("registered_country_iso_code", "NL"), + entry("registered_country_name", "Netherlands") + ) + ); + } + + /** + * Don't silently do DNS lookups or anything trappy on bogus data + */ + public void testInvalid() throws IOException { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoLite2-Country.mmdb"; + String ip = "www.google.com"; + try (DatabaseReaderLazyLoader loader = loader(databaseName)) { + IpDataLookup lookup = new MaxmindIpDataLookups.Country(Database.Country.properties()); + IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> lookup.getData(loader, ip)); + assertThat(e.getMessage(), containsString("not an IP string literal")); + } + } + + public void testCountryWithMissingLocation() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoLite2-Country.mmdb"; + String ip = "80.231.5.0"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.Country(Database.Country.properties()), + Map.ofEntries(entry("ip", ip)) + ); + } + + public void testAsn() throws IOException { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoLite2-ASN.mmdb"; + String ip = "82.171.64.0"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.Asn(Database.Asn.properties()), + Map.ofEntries(entry("ip", ip), entry("organization_name", "KPN B.V."), entry("asn", 1136L), entry("network", "82.168.0.0/14")) + ); + } + + public void testAnonymousIp() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoIP2-Anonymous-IP-Test.mmdb"; + String ip = "81.2.69.1"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.AnonymousIp(Database.AnonymousIp.properties()), + Map.ofEntries( + entry("ip", ip), + entry("hosting_provider", true), + entry("tor_exit_node", true), + entry("anonymous_vpn", true), + entry("anonymous", true), + entry("public_proxy", true), + entry("residential_proxy", true) + ) + ); + } + + public void testConnectionType() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoIP2-Connection-Type-Test.mmdb"; + String ip = "214.78.120.5"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.ConnectionType(Database.ConnectionType.properties()), + Map.ofEntries(entry("ip", ip), entry("connection_type", "Satellite")) + ); + } + + public void testDomain() { + String databaseName = "GeoIP2-Domain-Test.mmdb"; + String ip = "69.219.64.2"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.Domain(Database.Domain.properties()), + Map.ofEntries(entry("ip", ip), entry("domain", "ameritech.net")) + ); + } + + public void testEnterprise() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoIP2-Enterprise-Test.mmdb"; + String ip = "74.209.24.4"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.Enterprise(Database.Enterprise.properties()), + Map.ofEntries( + entry("ip", ip), + entry("country_confidence", 99), + entry("country_in_european_union", false), + entry("country_iso_code", "US"), + entry("country_name", "United States"), + entry("continent_code", "NA"), + entry("continent_name", "North America"), + entry("region_iso_code", "US-NY"), + entry("region_name", "New York"), + entry("city_confidence", 11), + entry("city_name", "Chatham"), + entry("timezone", "America/New_York"), + entry("location", Map.of("lat", 42.3478, "lon", -73.5549)), + entry("accuracy_radius", 27), + entry("postal_code", "12037"), + entry("postal_confidence", 11), + entry("asn", 14671L), + entry("organization_name", "FairPoint Communications"), + entry("network", "74.209.16.0/20"), + entry("hosting_provider", false), + entry("tor_exit_node", false), + entry("anonymous_vpn", false), + entry("anonymous", false), + entry("public_proxy", false), + entry("residential_proxy", false), + entry("domain", "frpt.net"), + entry("isp", "Fairpoint Communications"), + entry("isp_organization_name", "Fairpoint Communications"), + entry("user_type", "residential"), + entry("connection_type", "Cable/DSL"), + entry("registered_country_in_european_union", false), + entry("registered_country_iso_code", "US"), + entry("registered_country_name", "United States") + ) + ); + } + + public void testIsp() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoIP2-ISP-Test.mmdb"; + String ip = "149.101.100.1"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.Isp(Database.Isp.properties()), + Map.ofEntries( + entry("ip", ip), + entry("asn", 6167L), + entry("organization_name", "CELLCO-PART"), + entry("network", "149.101.100.0/28"), + entry("isp", "Verizon Wireless"), + entry("isp_organization_name", "Verizon Wireless"), + entry("mobile_network_code", "004"), + entry("mobile_country_code", "310") + ) + ); + } + + public void testDatabaseTypeParsing() throws IOException { + // this test is a little bit overloaded -- it's testing that we're getting the expected sorts of + // database_type strings from these files, *and* it's also testing that we dispatch on those strings + // correctly and associated those files with the correct high-level Elasticsearch Database type. + // down the road it would probably make sense to split these out and find a better home for some of the + // logic, but for now it's probably more valuable to have the test *somewhere* than to get especially + // pedantic about where precisely it should be. + + copyDatabase("GeoLite2-City-Test.mmdb", tmpDir); + copyDatabase("GeoLite2-Country-Test.mmdb", tmpDir); + copyDatabase("GeoLite2-ASN-Test.mmdb", tmpDir); + copyDatabase("GeoIP2-Anonymous-IP-Test.mmdb", tmpDir); + copyDatabase("GeoIP2-City-Test.mmdb", tmpDir); + copyDatabase("GeoIP2-Country-Test.mmdb", tmpDir); + copyDatabase("GeoIP2-Connection-Type-Test.mmdb", tmpDir); + copyDatabase("GeoIP2-Domain-Test.mmdb", tmpDir); + copyDatabase("GeoIP2-Enterprise-Test.mmdb", tmpDir); + copyDatabase("GeoIP2-ISP-Test.mmdb", tmpDir); + + assertThat(parseDatabaseFromType("GeoLite2-City-Test.mmdb"), is(Database.City)); + assertThat(parseDatabaseFromType("GeoLite2-Country-Test.mmdb"), is(Database.Country)); + assertThat(parseDatabaseFromType("GeoLite2-ASN-Test.mmdb"), is(Database.Asn)); + assertThat(parseDatabaseFromType("GeoIP2-Anonymous-IP-Test.mmdb"), is(Database.AnonymousIp)); + assertThat(parseDatabaseFromType("GeoIP2-City-Test.mmdb"), is(Database.City)); + assertThat(parseDatabaseFromType("GeoIP2-Country-Test.mmdb"), is(Database.Country)); + assertThat(parseDatabaseFromType("GeoIP2-Connection-Type-Test.mmdb"), is(Database.ConnectionType)); + assertThat(parseDatabaseFromType("GeoIP2-Domain-Test.mmdb"), is(Database.Domain)); + assertThat(parseDatabaseFromType("GeoIP2-Enterprise-Test.mmdb"), is(Database.Enterprise)); + assertThat(parseDatabaseFromType("GeoIP2-ISP-Test.mmdb"), is(Database.Isp)); + } + + private Database parseDatabaseFromType(String databaseFile) throws IOException { + return IpDataLookupFactories.getDatabase(MMDBUtil.getDatabaseType(tmpDir.resolve(databaseFile))); + } + + private void assertExpectedLookupResults(String databaseName, String ip, IpDataLookup lookup, Map expected) { + try (DatabaseReaderLazyLoader loader = loader(databaseName)) { + Map actual = lookup.getData(loader, ip); + assertThat( + "The set of keys in the result are not the same as the set of expected keys", + actual.keySet(), + containsInAnyOrder(expected.keySet().toArray(new String[0])) + ); + for (Map.Entry entry : expected.entrySet()) { + assertThat("Unexpected value for key [" + entry.getKey() + "]", actual.get(entry.getKey()), equalTo(entry.getValue())); + } + } catch (AssertionError e) { + fail(e, "Assert failed for database [%s] with address [%s]", databaseName, ip); + } catch (Exception e) { + fail(e, "Exception for database [%s] with address [%s]", databaseName, ip); + } + } + + private DatabaseReaderLazyLoader loader(final String databaseName) { + Path path = tmpDir.resolve(databaseName); + copyDatabase(databaseName, path); + final GeoIpCache cache = new GeoIpCache(1000); + return new DatabaseReaderLazyLoader(cache, path, null); + } +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/TarInputStreamTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/TarInputStreamTests.java index 8ebf4b1cb7404..718e807d1183a 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/TarInputStreamTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/TarInputStreamTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationMetadataTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationMetadataTests.java index f035416d48068..476a30d86ee05 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationMetadataTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationMetadataTests.java @@ -1,14 +1,17 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip.direct; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.ingest.geoip.IngestGeoIpPlugin; import org.elasticsearch.test.AbstractXContentSerializingTestCase; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentParser; @@ -20,6 +23,11 @@ public class DatabaseConfigurationMetadataTests extends AbstractXContentSerializingTestCase { + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(new IngestGeoIpPlugin().getNamedWriteables()); + } + private String id; @Override diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationTests.java index 02c067561b49c..027775da3a2b2 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationTests.java @@ -1,25 +1,37 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip.direct; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.ingest.geoip.IngestGeoIpPlugin; +import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.Ipinfo; +import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.Local; import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.Maxmind; +import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.Web; import org.elasticsearch.test.AbstractXContentSerializingTestCase; import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; import java.util.Set; +import static org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.IPINFO_NAMES; import static org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.MAXMIND_NAMES; public class DatabaseConfigurationTests extends AbstractXContentSerializingTestCase { + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(new IngestGeoIpPlugin().getNamedWriteables()); + } + private String id; @Override @@ -34,26 +46,45 @@ protected DatabaseConfiguration createTestInstance() { } public static DatabaseConfiguration randomDatabaseConfiguration(String id) { - return new DatabaseConfiguration(id, randomFrom(MAXMIND_NAMES), new Maxmind(randomAlphaOfLength(5))); + boolean useIpinfo = randomBoolean(); + DatabaseConfiguration.Provider provider = switch (between(0, 2)) { + case 0 -> useIpinfo ? new Ipinfo() : new Maxmind(randomAlphaOfLength(5)); + case 1 -> new Web(); + case 2 -> new Local(randomAlphaOfLength(10)); + default -> throw new AssertionError("failure, got illegal switch case"); + }; + return new DatabaseConfiguration(id, useIpinfo ? randomFrom(IPINFO_NAMES) : randomFrom(MAXMIND_NAMES), provider); } @Override protected DatabaseConfiguration mutateInstance(DatabaseConfiguration instance) { switch (between(0, 2)) { case 0: - return new DatabaseConfiguration(instance.id() + randomAlphaOfLength(2), instance.name(), instance.maxmind()); + return new DatabaseConfiguration(instance.id() + randomAlphaOfLength(2), instance.name(), instance.provider()); case 1: return new DatabaseConfiguration( instance.id(), - randomValueOtherThan(instance.name(), () -> randomFrom(MAXMIND_NAMES)), - instance.maxmind() + randomValueOtherThan( + instance.name(), + () -> instance.provider() instanceof Ipinfo ? randomFrom(IPINFO_NAMES) : randomFrom(MAXMIND_NAMES) + ), + instance.provider() ); case 2: - return new DatabaseConfiguration( - instance.id(), - instance.name(), - new Maxmind(instance.maxmind().accountId() + randomAlphaOfLength(2)) - ); + DatabaseConfiguration.Provider provider = instance.provider(); + DatabaseConfiguration.Provider modifiedProvider; + if (provider instanceof Maxmind maxmind) { + modifiedProvider = new Maxmind(maxmind.accountId() + randomAlphaOfLength(2)); + } else if (provider instanceof Ipinfo ignored) { + modifiedProvider = new Local(randomAlphaOfLength(20)); // can't modify Ipinfo + } else if (provider instanceof Web) { + modifiedProvider = new Local(randomAlphaOfLength(20)); // can't modify a Web + } else if (provider instanceof Local local) { + modifiedProvider = new Local(local.type() + randomAlphaOfLength(2)); + } else { + throw new AssertionError("Unexpected provider type: " + provider.getClass()); + } + return new DatabaseConfiguration(instance.id(), instance.name(), modifiedProvider); default: throw new AssertionError("failure, got illegal switch case"); } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationActionNodeResponseTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationActionNodeResponseTests.java new file mode 100644 index 0000000000000..12fb08a5a1abf --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationActionNodeResponseTests.java @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip.direct; + +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.io.IOException; +import java.util.List; + +import static java.util.Collections.emptySet; + +public class GetDatabaseConfigurationActionNodeResponseTests extends AbstractWireSerializingTestCase< + GetDatabaseConfigurationAction.NodeResponse> { + @Override + protected Writeable.Reader instanceReader() { + return GetDatabaseConfigurationAction.NodeResponse::new; + } + + @Override + protected GetDatabaseConfigurationAction.NodeResponse createTestInstance() { + return getRandomDatabaseConfigurationActionNodeResponse(); + } + + static GetDatabaseConfigurationAction.NodeResponse getRandomDatabaseConfigurationActionNodeResponse() { + return new GetDatabaseConfigurationAction.NodeResponse(randomDiscoveryNode(), getRandomDatabaseConfigurationMetadata()); + } + + private static DiscoveryNode randomDiscoveryNode() { + return DiscoveryNodeUtils.builder(randomAlphaOfLength(6)).roles(emptySet()).build(); + } + + static List getRandomDatabaseConfigurationMetadata() { + return randomList( + 0, + 20, + () -> new DatabaseConfigurationMetadata( + new DatabaseConfiguration( + randomAlphaOfLength(20), + randomAlphaOfLength(20), + randomFrom( + List.of( + new DatabaseConfiguration.Local(randomAlphaOfLength(10)), + new DatabaseConfiguration.Web(), + new DatabaseConfiguration.Ipinfo(), + new DatabaseConfiguration.Maxmind(randomAlphaOfLength(10)) + ) + ) + ), + randomNonNegativeLong(), + randomNonNegativeLong() + ) + ); + } + + @Override + protected GetDatabaseConfigurationAction.NodeResponse mutateInstance(GetDatabaseConfigurationAction.NodeResponse instance) + throws IOException { + return null; + } + + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry( + List.of( + new NamedWriteableRegistry.Entry( + DatabaseConfiguration.Provider.class, + DatabaseConfiguration.Maxmind.NAME, + DatabaseConfiguration.Maxmind::new + ), + new NamedWriteableRegistry.Entry( + DatabaseConfiguration.Provider.class, + DatabaseConfiguration.Ipinfo.NAME, + DatabaseConfiguration.Ipinfo::new + ), + new NamedWriteableRegistry.Entry( + DatabaseConfiguration.Provider.class, + DatabaseConfiguration.Local.NAME, + DatabaseConfiguration.Local::new + ), + new NamedWriteableRegistry.Entry( + DatabaseConfiguration.Provider.class, + DatabaseConfiguration.Web.NAME, + DatabaseConfiguration.Web::new + ) + ) + ); + } +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationActionResponseTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationActionResponseTests.java new file mode 100644 index 0000000000000..1b48a409d7876 --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/GetDatabaseConfigurationActionResponseTests.java @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip.direct; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.io.IOException; +import java.util.List; + +public class GetDatabaseConfigurationActionResponseTests extends AbstractWireSerializingTestCase { + @Override + protected Writeable.Reader instanceReader() { + return GetDatabaseConfigurationAction.Response::new; + } + + @Override + protected GetDatabaseConfigurationAction.Response createTestInstance() { + return new GetDatabaseConfigurationAction.Response( + GetDatabaseConfigurationActionNodeResponseTests.getRandomDatabaseConfigurationMetadata(), + getTestClusterName(), + getTestNodeResponses(), + getTestFailedNodeExceptions() + ); + } + + @Override + protected GetDatabaseConfigurationAction.Response mutateInstance(GetDatabaseConfigurationAction.Response instance) throws IOException { + return null; + } + + private ClusterName getTestClusterName() { + return new ClusterName(randomAlphaOfLength(30)); + } + + private List getTestNodeResponses() { + return randomList(0, 20, GetDatabaseConfigurationActionNodeResponseTests::getRandomDatabaseConfigurationActionNodeResponse); + } + + private List getTestFailedNodeExceptions() { + return randomList( + 0, + 5, + () -> new FailedNodeException( + randomAlphaOfLength(10), + randomAlphaOfLength(20), + new ElasticsearchException(randomAlphaOfLength(10)) + ) + ); + } + + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry( + List.of( + new NamedWriteableRegistry.Entry( + DatabaseConfiguration.Provider.class, + DatabaseConfiguration.Maxmind.NAME, + DatabaseConfiguration.Maxmind::new + ), + new NamedWriteableRegistry.Entry( + DatabaseConfiguration.Provider.class, + DatabaseConfiguration.Ipinfo.NAME, + DatabaseConfiguration.Ipinfo::new + ), + new NamedWriteableRegistry.Entry( + DatabaseConfiguration.Provider.class, + DatabaseConfiguration.Local.NAME, + DatabaseConfiguration.Local::new + ), + new NamedWriteableRegistry.Entry( + DatabaseConfiguration.Provider.class, + DatabaseConfiguration.Web.NAME, + DatabaseConfiguration.Web::new + ) + ) + ); + } +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationActionTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationActionTests.java new file mode 100644 index 0000000000000..988b50311186d --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationActionTests.java @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip.direct; + +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.test.ESTestCase; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.elasticsearch.ingest.geoip.direct.GetDatabaseConfigurationAction.NodeResponse; +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; + +public class TransportGetDatabaseConfigurationActionTests extends ESTestCase { + public void testDeduplicateNodeResponses() { + { + List nodeResponses = new ArrayList<>(); + Set preExistingNames = Set.of(); + Collection deduplicated = TransportGetDatabaseConfigurationAction.deduplicateNodeResponses( + nodeResponses, + preExistingNames + ); + assertTrue(deduplicated.isEmpty()); + } + { + List nodeResponses = List.of( + generateTestNodeResponse(List.of()), + generateTestNodeResponse(List.of()), + generateTestNodeResponse(List.of()) + ); + Set preExistingNames = Set.of(); + Collection deduplicated = TransportGetDatabaseConfigurationAction.deduplicateNodeResponses( + nodeResponses, + preExistingNames + ); + assertTrue(deduplicated.isEmpty()); + } + { + // 3 nodes with 3 overlapping responses. We expect the deduplicated collection to include 1, 2, 3, and 4. + List nodeResponses = List.of( + generateTestNodeResponse(List.of("1", "2", "3")), + generateTestNodeResponse(List.of("1", "2", "3")), + generateTestNodeResponse(List.of("1", "4")) + ); + Set preExistingNames = Set.of(); + Collection deduplicated = TransportGetDatabaseConfigurationAction.deduplicateNodeResponses( + nodeResponses, + preExistingNames + ); + assertThat(deduplicated.size(), equalTo(4)); + assertThat( + deduplicated.stream().map(database -> database.database().name()).collect(Collectors.toSet()), + equalTo(Set.of("1", "2", "3", "4")) + ); + } + { + /* + * 3 nodes with 3 overlapping responses, but this time we're also passing in a set of pre-existing names that overlap with + * two of them. So we expect the deduplicated collection to include 1 and 4. + */ + List nodeResponses = List.of( + generateTestNodeResponse(List.of("1", "2", "3")), + generateTestNodeResponse(List.of("1", "2", "3")), + generateTestNodeResponse(List.of("1", "4")) + ); + Set preExistingNames = Set.of("2", "3", "5"); + Collection deduplicated = TransportGetDatabaseConfigurationAction.deduplicateNodeResponses( + nodeResponses, + preExistingNames + ); + assertThat(deduplicated.size(), equalTo(2)); + assertThat( + deduplicated.stream().map(database -> database.database().name()).collect(Collectors.toSet()), + equalTo(Set.of("1", "4")) + ); + } + { + /* + * Here 3 nodes report the same database, but with different modified dates and versions. We expect the one with the highest + * modified date to win out. + */ + List nodeResponses = List.of( + generateTestNodeResponseFromDatabases(List.of(generateTestDatabase("1", 1))), + generateTestNodeResponseFromDatabases(List.of(generateTestDatabase("1", 1000))), + generateTestNodeResponseFromDatabases(List.of(generateTestDatabase("1", 3))) + ); + Set preExistingNames = Set.of("2", "3", "5"); + Collection deduplicated = TransportGetDatabaseConfigurationAction.deduplicateNodeResponses( + nodeResponses, + preExistingNames + ); + assertThat(deduplicated.size(), equalTo(1)); + DatabaseConfigurationMetadata result = deduplicated.iterator().next(); + assertThat(result, equalTo(nodeResponses.get(1).getDatabases().get(0))); + } + } + + private NodeResponse generateTestNodeResponse(List databaseNames) { + List databases = databaseNames.stream().map(this::generateTestDatabase).toList(); + return generateTestNodeResponseFromDatabases(databases); + } + + private NodeResponse generateTestNodeResponseFromDatabases(List databases) { + DiscoveryNode discoveryNode = mock(DiscoveryNode.class); + return new NodeResponse(discoveryNode, databases); + } + + private DatabaseConfigurationMetadata generateTestDatabase(String databaseName) { + return generateTestDatabase(databaseName, randomLongBetween(0, Long.MAX_VALUE)); + } + + private DatabaseConfigurationMetadata generateTestDatabase(String databaseName, long modifiedDate) { + DatabaseConfiguration databaseConfiguration = new DatabaseConfiguration( + randomAlphaOfLength(50), + databaseName, + new DatabaseConfiguration.Local(randomAlphaOfLength(20)) + ); + return new DatabaseConfigurationMetadata(databaseConfiguration, randomLongBetween(0, Long.MAX_VALUE), modifiedDate); + } +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationActionTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationActionTests.java index 710c3ee23916d..d53dc69f0d11b 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationActionTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationActionTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip.direct; diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/CacheStatsSerializingTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/CacheStatsSerializingTests.java index fa6b0c366d2d9..0afd018a08f6d 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/CacheStatsSerializingTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/CacheStatsSerializingTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip.stats; diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsSerializingTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsSerializingTests.java index 69e9cc9b5f5e5..8801318c501d3 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsSerializingTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsSerializingTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip.stats; diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsActionNodeResponseSerializingTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsActionNodeResponseSerializingTests.java index 6ff2e589270e1..a6ea5e7a8c0bd 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsActionNodeResponseSerializingTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsActionNodeResponseSerializingTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip.stats; diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsActionNodeResponseTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsActionNodeResponseTests.java index 919f82a4cbc1f..2ed416077b04d 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsActionNodeResponseTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsActionNodeResponseTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip.stats; diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsActionResponseSerializingTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsActionResponseSerializingTests.java index 6e057843b9776..cf64d08c15876 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsActionResponseSerializingTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsActionResponseSerializingTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip.stats; diff --git a/modules/ingest-geoip/src/test/resources/GeoIP2-Country-Test.mmdb b/modules/ingest-geoip/src/test/resources/GeoIP2-Country-Test.mmdb new file mode 100644 index 0000000000000..0b1f6cf50b2a0 Binary files /dev/null and b/modules/ingest-geoip/src/test/resources/GeoIP2-Country-Test.mmdb differ diff --git a/modules/ingest-geoip/src/test/resources/GeoLite2-ASN-Test.mmdb b/modules/ingest-geoip/src/test/resources/GeoLite2-ASN-Test.mmdb new file mode 100644 index 0000000000000..2614d1a7aa235 Binary files /dev/null and b/modules/ingest-geoip/src/test/resources/GeoLite2-ASN-Test.mmdb differ diff --git a/modules/ingest-geoip/src/test/resources/GeoLite2-Country-Test.mmdb b/modules/ingest-geoip/src/test/resources/GeoLite2-Country-Test.mmdb new file mode 100644 index 0000000000000..44b5ff1a3f1be Binary files /dev/null and b/modules/ingest-geoip/src/test/resources/GeoLite2-Country-Test.mmdb differ diff --git a/modules/ingest-geoip/src/test/resources/ipinfo/asn_sample.mmdb b/modules/ingest-geoip/src/test/resources/ipinfo/asn_sample.mmdb new file mode 100644 index 0000000000000..289318a124d75 Binary files /dev/null and b/modules/ingest-geoip/src/test/resources/ipinfo/asn_sample.mmdb differ diff --git a/modules/ingest-geoip/src/test/resources/ipinfo/ip_asn_sample.mmdb b/modules/ingest-geoip/src/test/resources/ipinfo/ip_asn_sample.mmdb new file mode 100644 index 0000000000000..d2bac8452a0f2 Binary files /dev/null and b/modules/ingest-geoip/src/test/resources/ipinfo/ip_asn_sample.mmdb differ diff --git a/modules/ingest-geoip/src/test/resources/ipinfo/ip_country_sample.mmdb b/modules/ingest-geoip/src/test/resources/ipinfo/ip_country_sample.mmdb new file mode 100644 index 0000000000000..caa218f02770b Binary files /dev/null and b/modules/ingest-geoip/src/test/resources/ipinfo/ip_country_sample.mmdb differ diff --git a/modules/ingest-geoip/src/test/resources/ipinfo/ip_geolocation_standard_sample.mmdb b/modules/ingest-geoip/src/test/resources/ipinfo/ip_geolocation_standard_sample.mmdb new file mode 100644 index 0000000000000..205bd77fd53e2 Binary files /dev/null and b/modules/ingest-geoip/src/test/resources/ipinfo/ip_geolocation_standard_sample.mmdb differ diff --git a/modules/ingest-geoip/src/test/resources/ipinfo/privacy_detection_sample.mmdb b/modules/ingest-geoip/src/test/resources/ipinfo/privacy_detection_sample.mmdb new file mode 100644 index 0000000000000..4f2fca5559e14 Binary files /dev/null and b/modules/ingest-geoip/src/test/resources/ipinfo/privacy_detection_sample.mmdb differ diff --git a/modules/ingest-geoip/src/test/resources/test-description.mmdb b/modules/ingest-geoip/src/test/resources/test-description.mmdb new file mode 100644 index 0000000000000..f57af8726d8d5 Binary files /dev/null and b/modules/ingest-geoip/src/test/resources/test-description.mmdb differ diff --git a/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java b/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java index 0f0a0c998bd75..f7943342b5352 100644 --- a/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java +++ b/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.geoip; diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml index 6809443fdfbc3..a1104505bc240 100644 --- a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml +++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml @@ -1,7 +1,19 @@ +--- setup: - requires: - cluster_features: ["geoip.downloader.database.configuration"] - reason: "geoip downloader database configuration APIs added in 8.15" + cluster_features: ["geoip.downloader.database.configuration", "get_database_configuration_action.multi_node"] + reason: "geoip downloader database configuration APIs added in 8.15, and updated in 8.16 to return more results" + +--- +teardown: + - do: + ingest.delete_ip_location_database: + id: "my_database_1" + ignore: 404 + - do: + ingest.delete_ip_location_database: + id: "my_database_2" + ignore: 404 --- "Test adding, getting, and removing geoip databases": @@ -41,6 +53,17 @@ setup: } - match: { acknowledged: true } + - do: + catch: /illegal_argument_exception/ + ingest.put_geoip_database: + id: "_web_TXlDdXN0b21HZW9MaXRlMi1DaXR5Lm1tZGI=" + body: > + { + "name": "GeoIP2-City", + "web": { + } + } + - do: ingest.get_geoip_database: id: "my_database_1" @@ -52,19 +75,37 @@ setup: - do: ingest.get_geoip_database: {} - - length: { databases: 2 } + - length: { databases: 6 } - do: ingest.get_geoip_database: id: "my_database_1,my_database_2" - length: { databases: 2 } + - do: + ingest.get_geoip_database: + id: "_web_TXlDdXN0b21HZW9MaXRlMi1DaXR5Lm1tZGI=" + - length: { databases: 1 } + - match: { databases.0.id: "_web_TXlDdXN0b21HZW9MaXRlMi1DaXR5Lm1tZGI=" } + - gte: { databases.0.modified_date_millis: -1 } + - match: { databases.0.database.name: "MyCustomGeoLite2-City" } + - do: ingest.delete_geoip_database: id: "my_database_1" + - do: + catch: /resource_not_found_exception/ + ingest.delete_geoip_database: + id: "_web_TXlDdXN0b21HZW9MaXRlMi1DaXR5Lm1tZGI=" + - do: ingest.get_geoip_database: {} + - length: { databases: 5 } + + - do: + ingest.get_geoip_database: + id: "my_database_2" - length: { databases: 1 } - match: { databases.0.id: "my_database_2" } - gte: { databases.0.modified_date_millis: 0 } diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/50_ip_lookup_processor.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/50_ip_lookup_processor.yml new file mode 100644 index 0000000000000..fd73c715a5ac5 --- /dev/null +++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/50_ip_lookup_processor.yml @@ -0,0 +1,45 @@ +setup: + - requires: + cluster_features: + - "put_database_configuration_action.ipinfo" + reason: "ipinfo support added in 8.16" + +--- +"Test ip_location processor with defaults": + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "ip_location" : { + "field" : "field1" + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + id: "1" + pipeline: "my_pipeline" + body: {field1: "89.160.20.128"} + + - do: + get: + index: test + id: "1" + - match: { _source.field1: "89.160.20.128" } + - length: { _source.ip_location: 7 } + - match: { _source.ip_location.city_name: "Linköping" } + - match: { _source.ip_location.country_iso_code: "SE" } + - match: { _source.ip_location.location.lon: 15.6167 } + - match: { _source.ip_location.location.lat: 58.4167 } + - match: { _source.ip_location.region_iso_code: "SE-E" } + - match: { _source.ip_location.country_name: "Sweden" } + - match: { _source.ip_location.region_name: "Östergötland County" } + - match: { _source.ip_location.continent_name: "Europe" } diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/60_ip_location_databases.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/60_ip_location_databases.yml new file mode 100644 index 0000000000000..e2e9a1fdb5e28 --- /dev/null +++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/60_ip_location_databases.yml @@ -0,0 +1,137 @@ +--- +setup: + - requires: + cluster_features: + - "put_database_configuration_action.ipinfo" + reason: "ip location downloader database configuration APIs added in 8.16 to support more types" + +--- +teardown: + - do: + ingest.delete_ip_location_database: + id: "my_database_1" + ignore: 404 + - do: + ingest.delete_ip_location_database: + id: "my_database_2" + ignore: 404 + - do: + ingest.delete_ip_location_database: + id: "my_database_3" + ignore: 404 + +--- +"Test adding, getting, and removing ip location databases": + - do: + ingest.put_ip_location_database: + id: "my_database_1" + body: > + { + "name": "GeoIP2-City", + "maxmind": { + "account_id": "1234" + } + } + - match: { acknowledged: true } + + - do: + ingest.put_ip_location_database: + id: "my_database_1" + body: > + { + "name": "GeoIP2-Country", + "maxmind": { + "account_id": "4321" + } + } + - match: { acknowledged: true } + + - do: + ingest.put_ip_location_database: + id: "my_database_2" + body: > + { + "name": "GeoIP2-City", + "maxmind": { + "account_id": "1234" + } + } + - match: { acknowledged: true } + + - do: + catch: /illegal_argument_exception/ + ingest.put_ip_location_database: + id: "_web_TXlDdXN0b21HZW9MaXRlMi1DaXR5Lm1tZGI=" + body: > + { + "name": "GeoIP2-City", + "web": { + } + } + + - do: + ingest.put_ip_location_database: + id: "my_database_3" + body: > + { + "name": "standard_privacy", + "ipinfo": { + } + } + - match: { acknowledged: true } + + - do: + ingest.get_ip_location_database: + id: "my_database_1" + - length: { databases: 1 } + - match: { databases.0.id: "my_database_1" } + - gte: { databases.0.modified_date_millis: 0 } + - match: { databases.0.database.name: "GeoIP2-Country" } + - match: { databases.0.database.maxmind.account_id: "4321" } + + - do: + ingest.get_ip_location_database: {} + - length: { databases: 7 } + + - do: + ingest.get_ip_location_database: + id: "my_database_1,my_database_2" + - length: { databases: 2 } + + - do: + ingest.get_ip_location_database: + id: "_web_TXlDdXN0b21HZW9MaXRlMi1DaXR5Lm1tZGI=" + - length: { databases: 1 } + - match: { databases.0.id: "_web_TXlDdXN0b21HZW9MaXRlMi1DaXR5Lm1tZGI=" } + - gte: { databases.0.modified_date_millis: -1 } + - match: { databases.0.database.name: "MyCustomGeoLite2-City" } + + - do: + ingest.delete_ip_location_database: + id: "my_database_1" + + - do: + catch: /resource_not_found_exception/ + ingest.delete_ip_location_database: + id: "_web_TXlDdXN0b21HZW9MaXRlMi1DaXR5Lm1tZGI=" + + - do: + ingest.get_ip_location_database: {} + - length: { databases: 6 } + + - do: + ingest.get_ip_location_database: + id: "my_database_2" + - length: { databases: 1 } + - match: { databases.0.id: "my_database_2" } + - gte: { databases.0.modified_date_millis: 0 } + - match: { databases.0.database.name: "GeoIP2-City" } + - match: { databases.0.database.maxmind.account_id: "1234" } + + - do: + ingest.get_ip_location_database: + id: "my_database_3" + - length: { databases: 1 } + - match: { databases.0.id: "my_database_3" } + - gte: { databases.0.modified_date_millis: 0 } + - match: { databases.0.database.name: "standard_privacy" } diff --git a/modules/ingest-user-agent/build.gradle b/modules/ingest-user-agent/build.gradle index 64cd38c584820..46eca827dd504 100644 --- a/modules/ingest-user-agent/build.gradle +++ b/modules/ingest-user-agent/build.gradle @@ -1,16 +1,17 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.internal-yaml-rest-test' apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { - description 'Ingest processor that extracts information from a user agent' - classname 'org.elasticsearch.ingest.useragent.IngestUserAgentPlugin' + description = 'Ingest processor that extracts information from a user agent' + classname ='org.elasticsearch.ingest.useragent.IngestUserAgentPlugin' } restResources { diff --git a/modules/ingest-user-agent/src/main/java/module-info.java b/modules/ingest-user-agent/src/main/java/module-info.java index 4e3ad032fd845..e17dab83d5754 100644 --- a/modules/ingest-user-agent/src/main/java/module-info.java +++ b/modules/ingest-user-agent/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.ingest.useragent { diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java index 6117ebc6aa319..d05d8b6abe1ac 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.useragent; diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java index 754c0e7b49617..6262c26cb752f 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.useragent; diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentCache.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentCache.java index 53dee030111cd..22f4ae01ac3f3 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentCache.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentCache.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.useragent; diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java index 515c31735c313..e902ae3c824fe 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.useragent; diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java index abdd8bec4cd2d..742b4c8c7e8e1 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java @@ -1,13 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.useragent; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.util.Maps; @@ -97,19 +99,8 @@ public IngestDocument execute(IngestDocument ingestDocument) { } break; case VERSION: - StringBuilder version = new StringBuilder(); if (uaClient.userAgent() != null && uaClient.userAgent().major() != null) { - version.append(uaClient.userAgent().major()); - if (uaClient.userAgent().minor() != null) { - version.append(".").append(uaClient.userAgent().minor()); - if (uaClient.userAgent().patch() != null) { - version.append(".").append(uaClient.userAgent().patch()); - if (uaClient.userAgent().build() != null) { - version.append(".").append(uaClient.userAgent().build()); - } - } - } - uaDetails.put("version", version.toString()); + uaDetails.put("version", versionToString(uaClient.userAgent())); } break; case OS: @@ -117,20 +108,10 @@ public IngestDocument execute(IngestDocument ingestDocument) { Map osDetails = Maps.newMapWithExpectedSize(3); if (uaClient.operatingSystem().name() != null) { osDetails.put("name", uaClient.operatingSystem().name()); - StringBuilder sb = new StringBuilder(); if (uaClient.operatingSystem().major() != null) { - sb.append(uaClient.operatingSystem().major()); - if (uaClient.operatingSystem().minor() != null) { - sb.append(".").append(uaClient.operatingSystem().minor()); - if (uaClient.operatingSystem().patch() != null) { - sb.append(".").append(uaClient.operatingSystem().patch()); - if (uaClient.operatingSystem().build() != null) { - sb.append(".").append(uaClient.operatingSystem().build()); - } - } - } - osDetails.put("version", sb.toString()); - osDetails.put("full", uaClient.operatingSystem().name() + " " + sb.toString()); + String version = versionToString(uaClient.operatingSystem()); + osDetails.put("version", version); + osDetails.put("full", uaClient.operatingSystem().name() + " " + version); } uaDetails.put("os", osDetails); } @@ -162,6 +143,23 @@ public IngestDocument execute(IngestDocument ingestDocument) { return ingestDocument; } + private static String versionToString(final UserAgentParser.VersionedName version) { + final StringBuilder versionString = new StringBuilder(); + if (Strings.hasLength(version.major())) { + versionString.append(version.major()); + if (Strings.hasLength(version.minor())) { + versionString.append(".").append(version.minor()); + if (Strings.hasLength(version.patch())) { + versionString.append(".").append(version.patch()); + if (Strings.hasLength(version.build())) { + versionString.append(".").append(version.build()); + } + } + } + } + return versionString.toString(); + } + @Override public String getType() { return TYPE; diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java index 582a40fb8a210..75f47a879597e 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.useragent; diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java index b81fa3e37b00e..b08531cf80ec5 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.useragent; diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java index 4da6ac77b50f8..df023f4d13428 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.useragent; @@ -330,4 +331,21 @@ public void testExtractDeviceTypeDisabled() { device.put("name", "Other"); assertThat(target.get("device"), is(device)); } + + // From https://github.com/elastic/elasticsearch/issues/116950 + @SuppressWarnings("unchecked") + public void testFirefoxVersion() { + Map document = new HashMap<>(); + document.put("source_field", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0"); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + + processor.execute(ingestDocument); + Map data = ingestDocument.getSourceAndMetadata(); + + assertThat(data, hasKey("target_field")); + Map target = (Map) data.get("target_field"); + + assertThat(target.get("name"), is("Firefox")); + assertThat(target.get("version"), is("128.0")); + } } diff --git a/modules/ingest-user-agent/src/yamlRestTest/java/org/elasticsearch/ingest/useragent/IngestUserAgentClientYamlTestSuiteIT.java b/modules/ingest-user-agent/src/yamlRestTest/java/org/elasticsearch/ingest/useragent/IngestUserAgentClientYamlTestSuiteIT.java index a942bc6ad8584..f5bcf771346f3 100644 --- a/modules/ingest-user-agent/src/yamlRestTest/java/org/elasticsearch/ingest/useragent/IngestUserAgentClientYamlTestSuiteIT.java +++ b/modules/ingest-user-agent/src/yamlRestTest/java/org/elasticsearch/ingest/useragent/IngestUserAgentClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest.useragent; diff --git a/modules/kibana/build.gradle b/modules/kibana/build.gradle index a4ff51e99069c..deaf2664236ba 100644 --- a/modules/kibana/build.gradle +++ b/modules/kibana/build.gradle @@ -1,16 +1,17 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.internal-cluster-test' apply plugin: 'elasticsearch.internal-java-rest-test' esplugin { - description 'Plugin exposing APIs for Kibana system indices' - classname 'org.elasticsearch.kibana.KibanaPlugin' + description = 'Plugin exposing APIs for Kibana system indices' + classname ='org.elasticsearch.kibana.KibanaPlugin' } dependencies { diff --git a/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java b/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java index b4cb4404525f4..61bd31fea3455 100644 --- a/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java +++ b/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.kibana; @@ -22,6 +23,7 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.threadpool.ThreadPoolStats; import java.util.Arrays; import java.util.Collection; @@ -55,7 +57,8 @@ public class KibanaThreadPoolIT extends ESIntegTestCase { protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { return Settings.builder() .put(super.nodeSettings(nodeOrdinal, otherSettings)) - .put(IndexingPressure.MAX_INDEXING_BYTES.getKey(), "1KB") + .put(IndexingPressure.MAX_PRIMARY_BYTES.getKey(), "1KB") + .put(IndexingPressure.MAX_COORDINATING_BYTES.getKey(), "1KB") .put("thread_pool.search.size", 1) .put("thread_pool.search.queue_size", 1) .put("thread_pool.write.size", 1) @@ -120,8 +123,32 @@ private void assertThreadPoolsBlocked() { () -> client().prepareIndex(USER_INDEX).setSource(Map.of("foo", "bar")).get() ); assertThat(e1.getMessage(), startsWith("rejected execution of TimedRunnable")); - var e2 = expectThrows(EsRejectedExecutionException.class, () -> client().prepareGet(USER_INDEX, "id").get()); - assertThat(e2.getMessage(), startsWith("rejected execution of ActionRunnable")); + + final var getFuture = client().prepareGet(USER_INDEX, "id").execute(); + // response handling is force-executed on GET pool, so we must + // (a) wait for that task to be enqueued, expanding the queue beyond its configured limit, and + // (b) check for the exception in the background + + try { + assertTrue(waitUntil(() -> { + if (getFuture.isDone()) { + return true; + } + for (ThreadPool threadPool : internalCluster().getInstances(ThreadPool.class)) { + for (ThreadPoolStats.Stats stats : threadPool.stats().stats()) { + if (stats.name().equals(ThreadPool.Names.GET) && stats.queue() > 1) { + return true; + } + } + } + return false; + })); + } catch (Exception e) { + fail(e); + } + + new Thread(() -> expectThrows(EsRejectedExecutionException.class, () -> getFuture.actionGet(SAFE_AWAIT_TIMEOUT))).start(); + // intentionally commented out this test until https://github.com/elastic/elasticsearch/issues/97916 is fixed // var e3 = expectThrows( // SearchPhaseExecutionException.class, diff --git a/modules/kibana/src/javaRestTest/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java b/modules/kibana/src/javaRestTest/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java index d066fb6f03999..a87f61d0ddb2c 100644 --- a/modules/kibana/src/javaRestTest/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java +++ b/modules/kibana/src/javaRestTest/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.kibana; diff --git a/modules/kibana/src/main/java/module-info.java b/modules/kibana/src/main/java/module-info.java index 30a2d12601a14..6c5692e7e04a1 100644 --- a/modules/kibana/src/main/java/module-info.java +++ b/modules/kibana/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.kibana { diff --git a/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java b/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java index a6c48fc8b772f..e5877d1f2eb2a 100644 --- a/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java +++ b/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.kibana; diff --git a/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaPluginTests.java b/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaPluginTests.java index 6d9c4c39ad487..aa883c83eecf6 100644 --- a/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaPluginTests.java +++ b/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaPluginTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.kibana; diff --git a/modules/lang-expression/build.gradle b/modules/lang-expression/build.gradle index 66c9dcf6a236f..a7f30c4fbb5cd 100644 --- a/modules/lang-expression/build.gradle +++ b/modules/lang-expression/build.gradle @@ -1,17 +1,18 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.legacy-yaml-rest-test' apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { - description 'Lucene expressions integration for Elasticsearch' - classname 'org.elasticsearch.script.expression.ExpressionPlugin' + description = 'Lucene expressions integration for Elasticsearch' + classname ='org.elasticsearch.script.expression.ExpressionPlugin' } dependencies { diff --git a/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java b/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java index 69e33863b0f2b..570c2a5f3783a 100644 --- a/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java +++ b/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/StoredExpressionIT.java b/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/StoredExpressionIT.java index c41fea6b86ceb..f5a0d70627b7a 100644 --- a/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/StoredExpressionIT.java +++ b/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/StoredExpressionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/main/java/module-info.java b/modules/lang-expression/src/main/java/module-info.java index b42f40d7d0955..691baeb66e875 100644 --- a/modules/lang-expression/src/main/java/module-info.java +++ b/modules/lang-expression/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.script.expression { diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java index 0b8c6c45d7500..75f59888abfbe 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateField.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateField.java index de29788850899..44f9199e5d4ca 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateField.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateField.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java index 360ec5c3046db..9b20bf1bfdd34 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObject.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObject.java index 48fbdc1094dbd..c951404dcf3b1 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObject.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObject.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObjectValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObjectValueSource.java index f9b67c427bba1..340c08fc489b0 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObjectValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObjectValueSource.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java index a3b4c1e3394c7..8fc654bcd7859 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionAggregationScript.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionAggregationScript.java index df08c0908e182..4954bc0b2144f 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionAggregationScript.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionAggregationScript.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionDoubleValuesScript.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionDoubleValuesScript.java index a7935800ec4ba..0952ff8fe856f 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionDoubleValuesScript.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionDoubleValuesScript.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionFieldScript.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionFieldScript.java index eb8d3ea833ecb..d8bc0f1cf1abb 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionFieldScript.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionFieldScript.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionNumberSortScript.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionNumberSortScript.java index 8cbb6cedd39a1..24382395212b5 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionNumberSortScript.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionNumberSortScript.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java index 9c39a9c0297f5..087fa7cb8946f 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScoreScript.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScoreScript.java index 622a1bd4afd25..f6a88e7597af6 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScoreScript.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScoreScript.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java index 7d80a0d401013..b306f104d7ba5 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionTermSetQueryScript.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionTermSetQueryScript.java index 8dec2e93b6d00..c7320850231c0 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionTermSetQueryScript.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionTermSetQueryScript.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataBasedDoubleValuesSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataBasedDoubleValuesSource.java index a9f0abd91e837..3eb6a283a2a92 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataBasedDoubleValuesSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataBasedDoubleValuesSource.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java index 656a420cdfcb6..a59fed4d9f873 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java index 6aaf20acf62cb..8499446d3dd27 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoField.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoField.java index 63ea46238e679..c50dde811dbc5 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoField.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoField.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java index b8c9f95784ad5..b296d40230d90 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java index dec6beb8dc732..18b09ba2694a3 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/NumericField.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/NumericField.java index 387b6fd7d939d..a468643a2d781 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/NumericField.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/NumericField.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValueSource.java index 697eca6226e3c..6f8e06c6368e0 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValueSource.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValues.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValues.java index 494db56201901..461da22c385d2 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValues.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValues.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/main/plugin-metadata/entitlement-policy.yaml b/modules/lang-expression/src/main/plugin-metadata/entitlement-policy.yaml new file mode 100644 index 0000000000000..b05e6e3a7bf7c --- /dev/null +++ b/modules/lang-expression/src/main/plugin-metadata/entitlement-policy.yaml @@ -0,0 +1,2 @@ +org.elasticsearch.script.expression: + - create_class_loader diff --git a/modules/lang-expression/src/main/plugin-metadata/plugin-security.policy b/modules/lang-expression/src/main/plugin-metadata/plugin-security.policy index 95b919b75bf85..344ff666ebdb4 100644 --- a/modules/lang-expression/src/main/plugin-metadata/plugin-security.policy +++ b/modules/lang-expression/src/main/plugin-metadata/plugin-security.policy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ grant { diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionDoubleValuesScriptTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionDoubleValuesScriptTests.java index bce884a422871..8b302de382953 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionDoubleValuesScriptTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionDoubleValuesScriptTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java index e573d32f7b6c2..494302b092098 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java index eedd80bf6d295..b8477edadc223 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java index d88b2d67e663a..a83833f37ab02 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-expression/src/yamlRestTest/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java b/modules/lang-expression/src/yamlRestTest/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java index dcbba01564295..87972cd6f2876 100644 --- a/modules/lang-expression/src/yamlRestTest/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java +++ b/modules/lang-expression/src/yamlRestTest/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.expression; diff --git a/modules/lang-mustache/build.gradle b/modules/lang-mustache/build.gradle index 7059165af2d9f..bd1e084df5d38 100644 --- a/modules/lang-mustache/build.gradle +++ b/modules/lang-mustache/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.legacy-yaml-rest-test' apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' @@ -11,8 +12,8 @@ apply plugin: 'elasticsearch.legacy-java-rest-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { - description 'Mustache scripting integration for Elasticsearch' - classname 'org.elasticsearch.script.mustache.MustachePlugin' + description = 'Mustache scripting integration for Elasticsearch' + classname ='org.elasticsearch.script.mustache.MustachePlugin' } dependencies { diff --git a/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java index c01f6ef46d0c4..6a418d2265a48 100644 --- a/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java +++ b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MultiSearchTemplateIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; diff --git a/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MustacheSettingsIT.java b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MustacheSettingsIT.java new file mode 100644 index 0000000000000..7ab3f1bd3b809 --- /dev/null +++ b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/MustacheSettingsIT.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.script.mustache; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.test.ESSingleNodeTestCase; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; +import static org.hamcrest.Matchers.equalTo; + +public class MustacheSettingsIT extends ESSingleNodeTestCase { + @Override + protected Collection> getPlugins() { + return List.of(MustachePlugin.class); + } + + @Override + protected Settings nodeSettings() { + return Settings.builder().put(MustacheScriptEngine.MUSTACHE_RESULT_SIZE_LIMIT.getKey(), "10b").build(); + } + + public void testResultSizeLimit() throws Exception { + createIndex("test"); + prepareIndex("test").setId("1").setSource(jsonBuilder().startObject().field("text", "value1").endObject()).get(); + indicesAdmin().prepareRefresh().get(); + + String query = """ + { "query": {"match_all": {}}, "size" : "{{my_size}}" }"""; + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices("test"); + var e = expectThrows( + ElasticsearchParseException.class, + () -> new SearchTemplateRequestBuilder(client()).setRequest(searchRequest) + .setScript(query) + .setScriptType(ScriptType.INLINE) + .setScriptParams(Collections.singletonMap("my_size", 1)) + .get() + ); + assertThat(e.getMessage(), equalTo("Mustache script result size limit exceeded")); + } +} diff --git a/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java index bfac939721122..cc0b0122e9cce 100644 --- a/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java +++ b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; @@ -12,12 +13,10 @@ import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptAction; import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest; import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse; -import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest; import org.elasticsearch.action.admin.cluster.storedscripts.TransportDeleteStoredScriptAction; import org.elasticsearch.action.admin.cluster.storedscripts.TransportPutStoredScriptAction; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptType; @@ -38,6 +37,7 @@ import java.util.Map; import java.util.concurrent.ExecutionException; +import static org.elasticsearch.action.admin.cluster.storedscripts.StoredScriptIntegTestUtils.newPutStoredScriptTestRequest; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; @@ -466,12 +466,6 @@ public static void assertHitCount(SearchTemplateRequestBuilder requestBuilder, l } private void putJsonStoredScript(String id, String jsonContent) { - assertAcked( - safeExecute( - TransportPutStoredScriptAction.TYPE, - new PutStoredScriptRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT).id(id) - .content(new BytesArray(jsonContent), XContentType.JSON) - ) - ); + assertAcked(safeExecute(TransportPutStoredScriptAction.TYPE, newPutStoredScriptTestRequest(id, jsonContent))); } } diff --git a/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchUsageStatsIT.java b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchUsageStatsIT.java index e51a4822c67ba..903d073b55783 100644 --- a/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchUsageStatsIT.java +++ b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchUsageStatsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; diff --git a/modules/lang-mustache/src/javaRestTest/java/org/elasticsearch/script/mustache/SearchTemplateWithoutContentIT.java b/modules/lang-mustache/src/javaRestTest/java/org/elasticsearch/script/mustache/SearchTemplateWithoutContentIT.java index 6188b6b2d1537..e70e82bd6c14b 100644 --- a/modules/lang-mustache/src/javaRestTest/java/org/elasticsearch/script/mustache/SearchTemplateWithoutContentIT.java +++ b/modules/lang-mustache/src/javaRestTest/java/org/elasticsearch/script/mustache/SearchTemplateWithoutContentIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; diff --git a/modules/lang-mustache/src/main/java/module-info.java b/modules/lang-mustache/src/main/java/module-info.java index 5d6ec6333cda1..082dcf20a84a3 100644 --- a/modules/lang-mustache/src/main/java/module-info.java +++ b/modules/lang-mustache/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.mustache { diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java index 49ad8302605cf..81263e4a45519 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomReflectionObjectHandler.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomReflectionObjectHandler.java index 491ec6c851342..eb3fc09aba297 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomReflectionObjectHandler.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomReflectionObjectHandler.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java index 2e50d14b24d72..07a00b33407f1 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponse.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponse.java index 11d060002955a..93dedb5cb9645 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponse.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheInvalidParameterException.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheInvalidParameterException.java index 9aaf8cdae89ad..ccb8b506340df 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheInvalidParameterException.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheInvalidParameterException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java index 8c1a410ee8a66..cfdd57db4dd54 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; @@ -16,6 +17,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.features.NodeFeature; @@ -43,7 +45,7 @@ public class MustachePlugin extends Plugin implements ScriptPlugin, ActionPlugin @Override public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { - return new MustacheScriptEngine(); + return new MustacheScriptEngine(settings); } @Override @@ -72,4 +74,9 @@ public List getRestHandlers( new RestRenderSearchTemplateAction() ); } + + @Override + public List> getSettings() { + return List.of(MustacheScriptEngine.MUSTACHE_RESULT_SIZE_LIMIT); + } } diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java index 61102de0ab5a4..e7b1727791510 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; @@ -13,6 +14,13 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.text.SizeLimitingStringWriter; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.MemorySizeValue; import org.elasticsearch.script.GeneralScriptException; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; @@ -46,6 +54,19 @@ public final class MustacheScriptEngine implements ScriptEngine { public static final String NAME = "mustache"; + public static final Setting MUSTACHE_RESULT_SIZE_LIMIT = new Setting<>( + "mustache.max_output_size_bytes", + s -> "1mb", + s -> MemorySizeValue.parseBytesSizeValueOrHeapRatio(s, "mustache.max_output_size_bytes"), + Setting.Property.NodeScope + ); + + private final int sizeLimit; + + public MustacheScriptEngine(Settings settings) { + sizeLimit = (int) MUSTACHE_RESULT_SIZE_LIMIT.get(settings).getBytes(); + } + /** * Compile a template string to (in this case) a Mustache object than can * later be re-used for execution to fill in missing parameter values. @@ -117,10 +138,15 @@ private class MustacheExecutableScript extends TemplateScript { @Override public String execute() { - final StringWriter writer = new StringWriter(); + StringWriter writer = new SizeLimitingStringWriter(sizeLimit); try { template.execute(writer, params); } catch (Exception e) { + // size limit exception can appear at several places in the causal list depending on script & context + if (ExceptionsHelper.unwrap(e, SizeLimitingStringWriter.SizeLimitExceededException.class) != null) { + // don't log, client problem + throw new ElasticsearchParseException("Mustache script result size limit exceeded", e); + } if (shouldLogException(e)) { logger.error(() -> format("Error running %s", template), e); } diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java index f7f68324907ff..5a393b4032e6f 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java @@ -1,14 +1,16 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.rest.BaseRestHandler; @@ -46,8 +48,12 @@ public List routes() { new Route(POST, "/_msearch/template"), new Route(GET, "/{index}/_msearch/template"), new Route(POST, "/{index}/_msearch/template"), - Route.builder(GET, "/{index}/{type}/_msearch/template").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(POST, "/{index}/{type}/_msearch/template").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build() + Route.builder(GET, "/{index}/{type}/_msearch/template") + .deprecated(TYPES_DEPRECATION_MESSAGE, DeprecationLogger.CRITICAL, RestApiVersion.V_7) + .build(), + Route.builder(POST, "/{index}/{type}/_msearch/template") + .deprecated(TYPES_DEPRECATION_MESSAGE, DeprecationLogger.CRITICAL, RestApiVersion.V_7) + .build() ); } diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestRenderSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestRenderSearchTemplateAction.java index 18e18a49f7ddb..72c1bcf1390e7 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestRenderSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestRenderSearchTemplateAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java index ab1f90bc59933..b95c87fb6fdb3 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java @@ -1,15 +1,17 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.rest.BaseRestHandler; @@ -49,10 +51,10 @@ public List routes() { new Route(GET, "/{index}/_search/template"), new Route(POST, "/{index}/_search/template"), Route.builder(GET, "/{index}/{type}/_search/template") - .deprecated(RestSearchAction.TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7) + .deprecated(RestSearchAction.TYPES_DEPRECATION_MESSAGE, DeprecationLogger.CRITICAL, RestApiVersion.V_7) .build(), Route.builder(POST, "/{index}/{type}/_search/template") - .deprecated(RestSearchAction.TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7) + .deprecated(RestSearchAction.TYPES_DEPRECATION_MESSAGE, DeprecationLogger.CRITICAL, RestApiVersion.V_7) .build() ); } diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequest.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequest.java index 8145c586c402d..b535b3556b5be 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequest.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequestBuilder.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequestBuilder.java index 5fb60b6d2a085..7cb39b20d8bcd 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequestBuilder.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateResponse.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateResponse.java index 2e62f6e9c96f4..9bb80d5688b5f 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateResponse.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; @@ -106,7 +107,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws void innerToXContent(XContentBuilder builder, Params params) throws IOException { if (hasResponse()) { - ChunkedToXContent.wrapAsToXContent(p -> response.innerToXContentChunked(p)).toXContent(builder, params); + ChunkedToXContent.wrapAsToXContent(response::innerToXContentChunked).toXContent(builder, params); } else { // we can assume the template is always json as we convert it before compiling it try (InputStream stream = source.streamInput()) { diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportMultiSearchTemplateAction.java index cac278563d944..b7c0223298d59 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportMultiSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportMultiSearchTemplateAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java index 35d3bfdf6446d..9d5cbeb573e2b 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/CustomMustacheFactoryTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/CustomMustacheFactoryTests.java index 4845279c4dd86..eb9c1a6dc3031 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/CustomMustacheFactoryTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/CustomMustacheFactoryTests.java @@ -1,13 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptEngine; import org.elasticsearch.script.TemplateScript; @@ -64,7 +66,7 @@ public void testCreateEncoder() { } public void testJsonEscapeEncoder() { - final ScriptEngine engine = new MustacheScriptEngine(); + final ScriptEngine engine = new MustacheScriptEngine(Settings.EMPTY); final Map params = randomBoolean() ? Map.of(Script.CONTENT_TYPE_OPTION, JSON_MEDIA_TYPE) : Map.of(); TemplateScript.Factory compiled = engine.compile(null, "{\"field\": \"{{value}}\"}", TemplateScript.CONTEXT, params); @@ -74,7 +76,7 @@ public void testJsonEscapeEncoder() { } public void testDefaultEncoder() { - final ScriptEngine engine = new MustacheScriptEngine(); + final ScriptEngine engine = new MustacheScriptEngine(Settings.EMPTY); final Map params = Map.of(Script.CONTENT_TYPE_OPTION, PLAIN_TEXT_MEDIA_TYPE); TemplateScript.Factory compiled = engine.compile(null, "{\"field\": \"{{value}}\"}", TemplateScript.CONTEXT, params); @@ -84,7 +86,7 @@ public void testDefaultEncoder() { } public void testUrlEncoder() { - final ScriptEngine engine = new MustacheScriptEngine(); + final ScriptEngine engine = new MustacheScriptEngine(Settings.EMPTY); final Map params = Map.of(Script.CONTENT_TYPE_OPTION, X_WWW_FORM_URLENCODED_MEDIA_TYPE); TemplateScript.Factory compiled = engine.compile(null, "{\"field\": \"{{value}}\"}", TemplateScript.CONTEXT, params); diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java index caac9d54cfb7c..990a92fb0dbf8 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponseTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponseTests.java index 8e019e0f2cd42..c21f1cffd1aef 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponseTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponseTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java index 7c67938044c49..bc1cd30ad45bf 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java @@ -1,14 +1,20 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; +import com.github.mustachejava.MustacheException; import com.github.mustachejava.MustacheFactory; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.text.SizeLimitingStringWriter; import org.elasticsearch.script.GeneralScriptException; import org.elasticsearch.script.Script; import org.elasticsearch.script.TemplateScript; @@ -23,6 +29,9 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.test.LambdaMatchers.transformedMatch; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.startsWith; @@ -36,7 +45,7 @@ public class MustacheScriptEngineTests extends ESTestCase { @Before public void setup() { - qe = new MustacheScriptEngine(); + qe = new MustacheScriptEngine(Settings.builder().put(MustacheScriptEngine.MUSTACHE_RESULT_SIZE_LIMIT.getKey(), "1kb").build()); factory = CustomMustacheFactory.builder().build(); } @@ -401,6 +410,24 @@ public void testEscapeJson() throws IOException { } } + public void testResultSizeLimit() throws IOException { + String vals = "\"" + "{{val}}".repeat(200) + "\""; + String params = "\"val\":\"aaaaaaaaaa\""; + XContentParser parser = createParser(JsonXContent.jsonXContent, Strings.format("{\"source\":%s,\"params\":{%s}}", vals, params)); + Script script = Script.parse(parser); + var compiled = qe.compile(null, script.getIdOrCode(), TemplateScript.CONTEXT, Map.of()); + TemplateScript templateScript = compiled.newInstance(script.getParams()); + var ex = expectThrows(ElasticsearchParseException.class, templateScript::execute); + assertThat(ex.getCause(), instanceOf(MustacheException.class)); + assertThat( + ex.getCause().getCause(), + allOf( + instanceOf(SizeLimitingStringWriter.SizeLimitExceededException.class), + transformedMatch(Throwable::getMessage, endsWith("has exceeded the size limit [1024]")) + ) + ); + } + private String getChars() { String string = randomRealisticUnicodeOfCodepointLengthBetween(0, 10); for (int i = 0; i < string.length(); i++) { diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java index e06e696928930..335cfe91df87d 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java @@ -1,13 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Strings; import org.elasticsearch.script.ScriptEngine; @@ -38,7 +40,7 @@ public class MustacheTests extends ESTestCase { - private ScriptEngine engine = new MustacheScriptEngine(); + private ScriptEngine engine = new MustacheScriptEngine(Settings.EMPTY); public void testBasics() { String template = """ diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateActionTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateActionTests.java index fffa5295522a3..3613d7390fda2 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateActionTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateActionTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestSearchTemplateActionTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestSearchTemplateActionTests.java index 1f1955e5ca171..0216e750c55e0 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestSearchTemplateActionTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestSearchTemplateActionTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateRequestTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateRequestTests.java index 81b3aa4147b1b..9ce02f051e74e 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateRequestTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateRequestTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateRequestXContentTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateRequestXContentTests.java index b3adef392ffc6..70bf9d288802c 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateRequestXContentTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateRequestXContentTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateResponseTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateResponseTests.java index 26b4a45d4da53..3efcfde684ebc 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateResponseTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateResponseTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; diff --git a/modules/lang-mustache/src/yamlRestTest/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java b/modules/lang-mustache/src/yamlRestTest/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java index af7002705a3c0..38c1c269b7599 100644 --- a/modules/lang-mustache/src/yamlRestTest/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java +++ b/modules/lang-mustache/src/yamlRestTest/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.mustache; diff --git a/modules/lang-painless/build.gradle b/modules/lang-painless/build.gradle index cc557ac2289f6..509fdda808acb 100644 --- a/modules/lang-painless/build.gradle +++ b/modules/lang-painless/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.testclusters.DefaultTestClustersTask; @@ -14,8 +15,8 @@ apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { - description 'An easy, safe and fast scripting language for Elasticsearch' - classname 'org.elasticsearch.painless.PainlessPlugin' + description = 'An easy, safe and fast scripting language for Elasticsearch' + classname ='org.elasticsearch.painless.PainlessPlugin' } testClusters.configureEach { @@ -52,7 +53,7 @@ tasks.named("dependencyLicenses").configure { restResources { restApi { include '_common', 'cluster', 'nodes', 'indices', 'index', 'search', 'get', 'bulk', 'update', - 'scripts_painless_execute', 'put_script', 'delete_script' + 'scripts_painless_execute', 'put_script', 'delete_script', 'capabilities' } } diff --git a/modules/lang-painless/spi/build.gradle b/modules/lang-painless/spi/build.gradle index 96b26eea97720..103326f5059f8 100644 --- a/modules/lang-painless/spi/build.gradle +++ b/modules/lang-painless/spi/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.build' diff --git a/modules/lang-painless/spi/src/main/java/module-info.java b/modules/lang-painless/spi/src/main/java/module-info.java index 881c9c4f8a79a..f90214dc2072f 100644 --- a/modules/lang-painless/spi/src/main/java/module-info.java +++ b/modules/lang-painless/spi/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.painless.spi { diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/PainlessExtension.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/PainlessExtension.java index 4fe5b4a37fcab..466125fa4058b 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/PainlessExtension.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/PainlessExtension.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.spi; diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/PainlessTestScript.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/PainlessTestScript.java index 6f2f49233242f..173105f9e94df 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/PainlessTestScript.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/PainlessTestScript.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.spi; diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java index 1fcf776ac8428..a1f80275ce0d5 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.spi; diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistClass.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistClass.java index 1daad59768a15..b934a0087c118 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistClass.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistClass.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.spi; diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistClassBinding.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistClassBinding.java index c4aa86d12e70f..34b15125db992 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistClassBinding.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistClassBinding.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.spi; diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistConstructor.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistConstructor.java index 88ca2ca3eabb9..50489653a1ae8 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistConstructor.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistConstructor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.spi; diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistField.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistField.java index 872482bcf6281..e6f6723686ce0 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistField.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistField.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.spi; diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistInstanceBinding.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistInstanceBinding.java index 279cd1e60900f..b13c8d6cf960e 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistInstanceBinding.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistInstanceBinding.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.spi; diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistLoader.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistLoader.java index 238f4522fd7a0..37bff97a07ae2 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistLoader.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistLoader.java @@ -1,15 +1,18 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.spi; +import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.painless.spi.annotation.WhitelistAnnotationParser; +import java.io.InputStream; import java.io.InputStreamReader; import java.io.LineNumberReader; import java.lang.reflect.Constructor; @@ -139,7 +142,7 @@ public static Whitelist loadFromResourceFiles(Class resource, String... filep * } * } */ - public static Whitelist loadFromResourceFiles(Class resource, Map parsers, String... filepaths) { + public static Whitelist loadFromResourceFiles(Class owner, Map parsers, String... filepaths) { List whitelistClasses = new ArrayList<>(); List whitelistStatics = new ArrayList<>(); List whitelistClassBindings = new ArrayList<>(); @@ -152,7 +155,7 @@ public static Whitelist loadFromResourceFiles(Class resource, Map resource, Map) resource::getClassLoader); + ClassLoader loader = AccessController.doPrivileged((PrivilegedAction) owner::getClassLoader); return new Whitelist(loader, whitelistClasses, whitelistStatics, whitelistClassBindings, Collections.emptyList()); } + private static InputStream getResourceAsStream(Class owner, String name) { + InputStream stream = owner.getResourceAsStream(name); + if (stream == null) { + String msg = "Whitelist file [" + + owner.getPackageName().replace(".", "/") + + "/" + + name + + "] not found from owning class [" + + owner.getName() + + "]."; + if (owner.getModule().isNamed()) { + msg += " Check that the file exists and the package [" + + owner.getPackageName() + + "] is opened " + + "to module " + + WhitelistLoader.class.getModule().getName(); + } + throw new ResourceNotFoundException(msg); + } + return stream; + } + private static List parseWhitelistAnnotations(Map parsers, String line) { List annotations; diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistMethod.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistMethod.java index 8927d290ecc77..6ecc27061cecb 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistMethod.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistMethod.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.spi; diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/AliasAnnotation.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/AliasAnnotation.java index a35e9be94074b..6e48b6045b4b6 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/AliasAnnotation.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/AliasAnnotation.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.spi.annotation; diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/AliasAnnotationParser.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/AliasAnnotationParser.java index c46b9a048cbc6..a73ccda1615cc 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/AliasAnnotationParser.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/AliasAnnotationParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.spi.annotation; diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/AugmentedAnnotation.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/AugmentedAnnotation.java index 1c4f7f084ff5f..7f4ffcbcefa3a 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/AugmentedAnnotation.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/AugmentedAnnotation.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.spi.annotation; diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/AugmentedAnnotationParser.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/AugmentedAnnotationParser.java index 8d825dbf7434f..aabe01c78739c 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/AugmentedAnnotationParser.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/AugmentedAnnotationParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.spi.annotation; diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/CompileTimeOnlyAnnotation.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/CompileTimeOnlyAnnotation.java index daa6d58820684..85f9fd60929d1 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/CompileTimeOnlyAnnotation.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/CompileTimeOnlyAnnotation.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.spi.annotation; diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/CompileTimeOnlyAnnotationParser.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/CompileTimeOnlyAnnotationParser.java index 75f18aa919443..7b48577487aac 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/CompileTimeOnlyAnnotationParser.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/CompileTimeOnlyAnnotationParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.spi.annotation; diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/DeprecatedAnnotation.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/DeprecatedAnnotation.java index 832f74c67235e..4ea16e99b44d1 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/DeprecatedAnnotation.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/DeprecatedAnnotation.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.spi.annotation; diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/DeprecatedAnnotationParser.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/DeprecatedAnnotationParser.java index 65b23cfbf3345..6c880f891aa61 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/DeprecatedAnnotationParser.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/DeprecatedAnnotationParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.spi.annotation; diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/DynamicTypeAnnotation.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/DynamicTypeAnnotation.java index 4ae75df2f9496..aeb6b816456a5 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/DynamicTypeAnnotation.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/DynamicTypeAnnotation.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.spi.annotation; diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/DynamicTypeAnnotationParser.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/DynamicTypeAnnotationParser.java index 7a04c909bc173..0217dd8ca1f23 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/DynamicTypeAnnotationParser.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/DynamicTypeAnnotationParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.spi.annotation; diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/InjectConstantAnnotation.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/InjectConstantAnnotation.java index 5fb86313665e1..8e8f24e9187de 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/InjectConstantAnnotation.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/InjectConstantAnnotation.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.spi.annotation; diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/InjectConstantAnnotationParser.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/InjectConstantAnnotationParser.java index beb2f8b70e46c..9bbe543577951 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/InjectConstantAnnotationParser.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/InjectConstantAnnotationParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.spi.annotation; diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/NoImportAnnotation.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/NoImportAnnotation.java index 04bfc973f334a..19a00cabe3bce 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/NoImportAnnotation.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/NoImportAnnotation.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.spi.annotation; diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/NoImportAnnotationParser.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/NoImportAnnotationParser.java index dda209ce0faa6..2701cd42a07e7 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/NoImportAnnotationParser.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/NoImportAnnotationParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.spi.annotation; diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/NonDeterministicAnnotation.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/NonDeterministicAnnotation.java index 02d1115aea023..204ef3bd07a88 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/NonDeterministicAnnotation.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/NonDeterministicAnnotation.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.spi.annotation; diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/NonDeterministicAnnotationParser.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/NonDeterministicAnnotationParser.java index 2aa90dbfd037d..f30a9f70664c9 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/NonDeterministicAnnotationParser.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/NonDeterministicAnnotationParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.spi.annotation; diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/WhitelistAnnotationParser.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/WhitelistAnnotationParser.java index 72140608e69c1..4eaa4fa5a57d1 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/WhitelistAnnotationParser.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/WhitelistAnnotationParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.spi.annotation; diff --git a/modules/lang-painless/spi/src/test/java/org/elasticsearch/painless/AnnotationTestObject.java b/modules/lang-painless/spi/src/test/java/org/elasticsearch/painless/AnnotationTestObject.java index f6a8dcd8e9ed5..40258c4a4f3bc 100644 --- a/modules/lang-painless/spi/src/test/java/org/elasticsearch/painless/AnnotationTestObject.java +++ b/modules/lang-painless/spi/src/test/java/org/elasticsearch/painless/AnnotationTestObject.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/spi/src/test/java/org/elasticsearch/painless/WhitelistLoaderTests.java b/modules/lang-painless/spi/src/test/java/org/elasticsearch/painless/WhitelistLoaderTests.java index 2a773dc9672fa..b46bc118e0913 100644 --- a/modules/lang-painless/spi/src/test/java/org/elasticsearch/painless/WhitelistLoaderTests.java +++ b/modules/lang-painless/spi/src/test/java/org/elasticsearch/painless/WhitelistLoaderTests.java @@ -1,13 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; +import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.painless.spi.Whitelist; import org.elasticsearch.painless.spi.WhitelistClass; import org.elasticsearch.painless.spi.WhitelistLoader; @@ -16,10 +18,18 @@ import org.elasticsearch.painless.spi.annotation.NoImportAnnotation; import org.elasticsearch.painless.spi.annotation.WhitelistAnnotationParser; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.compiler.InMemoryJavaCompiler; +import org.elasticsearch.test.jar.JarUtils; +import java.lang.ModuleLayer.Controller; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import java.util.HashMap; import java.util.Map; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; + public class WhitelistLoaderTests extends ESTestCase { public void testUnknownAnnotations() { @@ -95,4 +105,52 @@ public void testAnnotations() { assertEquals(3, count); } + + public void testMissingWhitelistResource() { + var e = expectThrows(ResourceNotFoundException.class, () -> WhitelistLoader.loadFromResourceFiles(Whitelist.class, "missing.txt")); + assertThat( + e.getMessage(), + equalTo( + "Whitelist file [org/elasticsearch/painless/spi/missing.txt] not found" + + " from owning class [org.elasticsearch.painless.spi.Whitelist]." + ) + ); + } + + public void testMissingWhitelistResourceInModule() throws Exception { + Map sources = new HashMap<>(); + sources.put("module-info", "module m {}"); + sources.put("p.TestOwner", "package p; public class TestOwner { }"); + var classToBytes = InMemoryJavaCompiler.compile(sources); + + Path dir = createTempDir(getTestName()); + Path jar = dir.resolve("m.jar"); + Map jarEntries = new HashMap<>(); + jarEntries.put("module-info.class", classToBytes.get("module-info")); + jarEntries.put("p/TestOwner.class", classToBytes.get("p.TestOwner")); + jarEntries.put("p/resource.txt", "# test resource".getBytes(StandardCharsets.UTF_8)); + JarUtils.createJarWithEntries(jar, jarEntries); + + try (var loader = JarUtils.loadJar(jar)) { + Controller controller = JarUtils.loadModule(jar, loader.classloader(), "m"); + Module module = controller.layer().findModule("m").orElseThrow(); + + Class ownerClass = module.getClassLoader().loadClass("p.TestOwner"); + + // first check we get a nice error message when accessing the resource + var e = expectThrows(ResourceNotFoundException.class, () -> WhitelistLoader.loadFromResourceFiles(ownerClass, "resource.txt")); + assertThat( + e.getMessage(), + equalTo( + "Whitelist file [p/resource.txt] not found from owning class [p.TestOwner]." + + " Check that the file exists and the package [p] is opened to module null" + ) + ); + + // now check we can actually read it once the package is opened to us + controller.addOpens(module, "p", WhitelistLoader.class.getModule()); + var whitelist = WhitelistLoader.loadFromResourceFiles(ownerClass, "resource.txt"); + assertThat(whitelist, notNullValue()); + } + } } diff --git a/modules/lang-painless/src/doc/java/org/elasticsearch/painless/ContextApiSpecGenerator.java b/modules/lang-painless/src/doc/java/org/elasticsearch/painless/ContextApiSpecGenerator.java index d4b11d159ae44..ef5cbd5c2a03a 100644 --- a/modules/lang-painless/src/doc/java/org/elasticsearch/painless/ContextApiSpecGenerator.java +++ b/modules/lang-painless/src/doc/java/org/elasticsearch/painless/ContextApiSpecGenerator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/doc/java/org/elasticsearch/painless/ContextDocGenerator.java b/modules/lang-painless/src/doc/java/org/elasticsearch/painless/ContextDocGenerator.java index d99a085d784b5..fb0b8f6dcd499 100644 --- a/modules/lang-painless/src/doc/java/org/elasticsearch/painless/ContextDocGenerator.java +++ b/modules/lang-painless/src/doc/java/org/elasticsearch/painless/ContextDocGenerator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/doc/java/org/elasticsearch/painless/ContextGeneratorCommon.java b/modules/lang-painless/src/doc/java/org/elasticsearch/painless/ContextGeneratorCommon.java index e1a68221c92a7..5a12e54714706 100644 --- a/modules/lang-painless/src/doc/java/org/elasticsearch/painless/ContextGeneratorCommon.java +++ b/modules/lang-painless/src/doc/java/org/elasticsearch/painless/ContextGeneratorCommon.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/doc/java/org/elasticsearch/painless/JavaClassResolver.java b/modules/lang-painless/src/doc/java/org/elasticsearch/painless/JavaClassResolver.java index 137f48398b00b..8f3f7bdc3774d 100644 --- a/modules/lang-painless/src/doc/java/org/elasticsearch/painless/JavaClassResolver.java +++ b/modules/lang-painless/src/doc/java/org/elasticsearch/painless/JavaClassResolver.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/doc/java/org/elasticsearch/painless/JavadocExtractor.java b/modules/lang-painless/src/doc/java/org/elasticsearch/painless/JavadocExtractor.java index 77498b93e91d7..04519ce822cf5 100644 --- a/modules/lang-painless/src/doc/java/org/elasticsearch/painless/JavadocExtractor.java +++ b/modules/lang-painless/src/doc/java/org/elasticsearch/painless/JavadocExtractor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; @@ -48,10 +49,12 @@ public class JavadocExtractor { + " modify it under the terms of the GNU General Public License version 2 only, as published" + " by the Free Software Foundation."; - private static final String ESv2 = "Copyright Elasticsearch B.V. and/or licensed to Elasticsearch" - + " B.V. under one or more contributor license agreements. Licensed under the Elastic License 2.0" - + " and the Server Side Public License, v 1; you may not use this file except in compliance with," - + " at your election, the Elastic License 2.0 or the Server Side Public License, v 1."; + private static final String ESv2 = "Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one" + + " or more contributor license agreements. Licensed under the \"Elastic License" + + " 2.0\", the \"GNU Affero General Public License v3.0 only\", and the \"Server Side" + + " Public License v 1\"; you may not use this file except in compliance with, at" + + " your election, the \"Elastic License 2.0\", the \"GNU Affero General Public" + + " License v3.0 only\", or the \"Server Side Public License, v 1\"."; private static final String[] LICENSES = new String[] { GPLv2, ESv2 }; diff --git a/modules/lang-painless/src/doc/java/org/elasticsearch/painless/PainlessInfoJson.java b/modules/lang-painless/src/doc/java/org/elasticsearch/painless/PainlessInfoJson.java index 5797ac85cede1..2a8d86fb5ac33 100644 --- a/modules/lang-painless/src/doc/java/org/elasticsearch/painless/PainlessInfoJson.java +++ b/modules/lang-painless/src/doc/java/org/elasticsearch/painless/PainlessInfoJson.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/internalClusterTest/java/org/elasticsearch/painless/action/CrossClusterPainlessExecuteIT.java b/modules/lang-painless/src/internalClusterTest/java/org/elasticsearch/painless/action/CrossClusterPainlessExecuteIT.java index 99fb0edd4334f..4669ab25f5d8c 100644 --- a/modules/lang-painless/src/internalClusterTest/java/org/elasticsearch/painless/action/CrossClusterPainlessExecuteIT.java +++ b/modules/lang-painless/src/internalClusterTest/java/org/elasticsearch/painless/action/CrossClusterPainlessExecuteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.action; diff --git a/modules/lang-painless/src/internalClusterTest/java/org/elasticsearch/painless/search/SyntheticSourceIT.java b/modules/lang-painless/src/internalClusterTest/java/org/elasticsearch/painless/search/SyntheticSourceIT.java index 0383999d6f7e5..10b641c7cedaa 100644 --- a/modules/lang-painless/src/internalClusterTest/java/org/elasticsearch/painless/search/SyntheticSourceIT.java +++ b/modules/lang-painless/src/internalClusterTest/java/org/elasticsearch/painless/search/SyntheticSourceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.search; diff --git a/modules/lang-painless/src/main/antlr/PainlessLexer.g4 b/modules/lang-painless/src/main/antlr/PainlessLexer.g4 index 70d476b321a54..4c407b0774ac2 100644 --- a/modules/lang-painless/src/main/antlr/PainlessLexer.g4 +++ b/modules/lang-painless/src/main/antlr/PainlessLexer.g4 @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ lexer grammar PainlessLexer; diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.g4 b/modules/lang-painless/src/main/antlr/PainlessParser.g4 index 41e9e49365aa6..f5b6a5d687cda 100644 --- a/modules/lang-painless/src/main/antlr/PainlessParser.g4 +++ b/modules/lang-painless/src/main/antlr/PainlessParser.g4 @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ parser grammar PainlessParser; diff --git a/modules/lang-painless/src/main/antlr/SuggestLexer.g4 b/modules/lang-painless/src/main/antlr/SuggestLexer.g4 index ff068df700820..f08bd64062d4b 100644 --- a/modules/lang-painless/src/main/antlr/SuggestLexer.g4 +++ b/modules/lang-painless/src/main/antlr/SuggestLexer.g4 @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ lexer grammar SuggestLexer; diff --git a/modules/lang-painless/src/main/java/module-info.java b/modules/lang-painless/src/main/java/module-info.java index 539c2919b81ff..ada64ac7920c3 100644 --- a/modules/lang-painless/src/main/java/module-info.java +++ b/modules/lang-painless/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.painless { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java index 00294dab64790..2c08cf2c06134 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ClassWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ClassWriter.java index d274b294774dc..17cc62ac348d6 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ClassWriter.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ClassWriter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java index 3c31a962d841b..54b538cff51de 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java index 4080507a4e893..bda2e7b7516f2 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java index 8b45e7d0edd55..ca5a71ad027f8 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java index 3d35c5d0ab5e4..8d0b79fe268a6 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java index 1132b1163565b..0e4e223d3a958 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ErrorCauseWrapper.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ErrorCauseWrapper.java index c1e1012eb3381..cd985716d1382 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ErrorCauseWrapper.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ErrorCauseWrapper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java index 4da523bce634b..904cc0411433e 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaBootstrap.java index a2fa5dea229d5..bc7039f7cdd5c 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaBootstrap.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaBootstrap.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Location.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Location.java index d28630ffb24a5..604decf114706 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Location.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Location.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java index 1e88edc788c69..113a9ca3b2a0c 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Operation.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Operation.java index 967a0b43122f0..1c5b0eff1e9b6 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Operation.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Operation.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessError.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessError.java index 7903258069dc0..65c2cdcdfd61f 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessError.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessError.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExplainError.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExplainError.java index a59f0e587d3bf..fbadc7900a342 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExplainError.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExplainError.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java index c37fe8866fec8..fedae2a4535d8 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScript.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScript.java index c56cd397ecf07..d8e6fa62567e5 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScript.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScript.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java index 005148a6fcd5d..fe3b609ff56d0 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java index 8b0b57e6389fc..c660f3cd1d581 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/SimpleChecksAdapter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/SimpleChecksAdapter.java index bcd0f85c54244..e0a656dbe1807 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/SimpleChecksAdapter.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/SimpleChecksAdapter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java index b4e2f2a553830..218d4047be18d 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java index 9207e0a55673e..827edfa2599cf 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextAction.java index b11ebd2e652cb..efcd63be46888 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextAction.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.action; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextClassBindingInfo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextClassBindingInfo.java index 6dfa9e3b80fcb..841f49d7e99ca 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextClassBindingInfo.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextClassBindingInfo.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.action; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextClassInfo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextClassInfo.java index 1abb887418ddc..55d17fb837175 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextClassInfo.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextClassInfo.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.action; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextConstructorInfo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextConstructorInfo.java index e787731fe4915..3f7c52e419fa2 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextConstructorInfo.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextConstructorInfo.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.action; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextFieldInfo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextFieldInfo.java index ea08666437178..c6c8f8264deea 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextFieldInfo.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextFieldInfo.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.action; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextInfo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextInfo.java index 2fb8afdc2d874..854e3a8c869ce 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextInfo.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextInfo.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.action; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextInstanceBindingInfo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextInstanceBindingInfo.java index abc4304c8f370..80cf82ece5652 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextInstanceBindingInfo.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextInstanceBindingInfo.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.action; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextMethodInfo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextMethodInfo.java index a3635d1328bc8..6157d6ae58eb8 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextMethodInfo.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextMethodInfo.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.action; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextTypeInfo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextTypeInfo.java index 2c8d87a1ad6bd..4f711ce2aecdb 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextTypeInfo.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextTypeInfo.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.action; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java index 919f2659e1f92..4f34cbd3cc475 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.action; @@ -107,7 +108,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -237,29 +237,21 @@ static Tuple parseClusterAliasAndIndex(String indexExpression) { return new Tuple<>(null, null); } String trimmed = indexExpression.trim(); - String sep = String.valueOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR); - if (trimmed.startsWith(sep) || trimmed.endsWith(sep)) { - throw new IllegalArgumentException( - "Unable to parse one single valid index name from the provided index: [" + indexExpression + "]" - ); - } - + String[] parts = RemoteClusterAware.splitIndexName(trimmed); // The parser here needs to ensure that the indexExpression is not of the form "remote1:blogs,remote2:blogs" // because (1) only a single index is allowed for Painless Execute and // (2) if this method returns Tuple("remote1", "blogs,remote2:blogs") that will not fail with "index not found". // Instead, it will fail with the inaccurate and confusing error message: // "Cross-cluster calls are not supported in this context but remote indices were requested: [blogs,remote1:blogs]" // which comes later out of the IndexNameExpressionResolver pathway this code uses. - String[] parts = indexExpression.split(sep, 2); - if (parts.length == 1) { - return new Tuple<>(null, parts[0]); - } else if (parts.length == 2 && parts[1].contains(sep) == false) { - return new Tuple<>(parts[0], parts[1]); - } else { + if ((parts[0] != null && parts[1].isEmpty()) + || parts[1].contains(String.valueOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR))) { throw new IllegalArgumentException( "Unable to parse one single valid index name from the provided index: [" + indexExpression + "]" ); } + + return new Tuple<>(parts[0], parts[1]); } public String getClusterAlias() { @@ -555,8 +547,8 @@ protected void doExecute(Task task, Request request, ActionListener li // Visible for testing static void removeClusterAliasFromIndexExpression(Request request) { if (request.index() != null) { - String[] split = request.index().split(String.valueOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR)); - if (split.length > 1) { + String[] split = RemoteClusterAware.splitIndexName(request.index()); + if (split[0] != null) { /* * if the cluster alias is null and the index field has a clusterAlias (clusterAlias:index notation) * that means this is executing on a remote cluster (it was forwarded by the querying cluster). @@ -564,9 +556,6 @@ static void removeClusterAliasFromIndexExpression(Request request) { * We need to strip off the clusterAlias from the index before executing the script locally, * so it will resolve to a local index */ - assert split.length == 2 - : "If the index contains the REMOTE_CLUSTER_INDEX_SEPARATOR it should have only two parts but it has " - + Arrays.toString(split); request.index(split[1]); } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java index 622dd5e46d522..3198e73b5f44c 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.antlr; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedSuggestLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedSuggestLexer.java index f2bfba4f4b6cd..559a3a8afa8ef 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedSuggestLexer.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedSuggestLexer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.antlr; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/ParserErrorStrategy.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/ParserErrorStrategy.java index 4ab2f8296d459..868a7497e4a58 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/ParserErrorStrategy.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/ParserErrorStrategy.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.antlr; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java index 1f8b7b909909f..bb5d04c2c6e93 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.antlr; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/package-info.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/package-info.java index 6bf6fa964e3e3..ddc65a31d012d 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/package-info.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Augmentation.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Augmentation.java index 21f940efda5ac..5dc6fc0bb1244 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Augmentation.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Augmentation.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.api; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/CIDR.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/CIDR.java index c3e39b5905cdc..eef37a6a3a68d 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/CIDR.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/CIDR.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.api; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Debug.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Debug.java index d3b0e41ffd533..79886d6449e94 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Debug.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Debug.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.api; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Json.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Json.java index f121894cf4dc5..2660bbb84abc8 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Json.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Json.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.api; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/LimitedCharSequence.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/LimitedCharSequence.java index bb589a8dd6ccb..8980749dbf8a4 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/LimitedCharSequence.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/LimitedCharSequence.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.api; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/ValueIterator.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/ValueIterator.java index 7f366bc3284a8..99de79dae3853 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/ValueIterator.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/ValueIterator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.api; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/package-info.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/package-info.java index d3b77818cd819..bda38f9e4e520 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/package-info.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ArgumentsNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ArgumentsNode.java index 1e78f06cdf7a7..66937564cc424 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ArgumentsNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ArgumentsNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BinaryImplNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BinaryImplNode.java index 9aeba1b2b4f45..14133b1f014a8 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BinaryImplNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BinaryImplNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BinaryMathNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BinaryMathNode.java index feb019c5b9bd4..f6c6275f4aa19 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BinaryMathNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BinaryMathNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BinaryNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BinaryNode.java index 45af94ec31c0f..449b9c2943e83 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BinaryNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BinaryNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BlockNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BlockNode.java index e664b0f37aaf5..59cfdf2038e56 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BlockNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BlockNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BooleanNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BooleanNode.java index 54f811a1504b1..02befd41decfc 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BooleanNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BooleanNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BreakNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BreakNode.java index 534a0854c71cb..cf14dff88c478 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BreakNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/BreakNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/CastNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/CastNode.java index 15d5e29c2c52f..0b2f0ec3d4854 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/CastNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/CastNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/CatchNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/CatchNode.java index 39a8b1baa0dd8..f1da03727e1a2 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/CatchNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/CatchNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ClassNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ClassNode.java index fedc9a962b6e6..42d1cd5f14a39 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ClassNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ClassNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ComparisonNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ComparisonNode.java index 01267aa8e69a2..9361213081de8 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ComparisonNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ComparisonNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ConditionNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ConditionNode.java index 56fee19048bca..614ce7c26c128 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ConditionNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ConditionNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ConditionalNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ConditionalNode.java index 0c8c817fcd042..10d8ced2f9156 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ConditionalNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ConditionalNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ConstantNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ConstantNode.java index d638b79b93cba..ff3c8233c7d5b 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ConstantNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ConstantNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ContinueNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ContinueNode.java index fba09d4c33457..45f612f2cebe7 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ContinueNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ContinueNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/DeclarationBlockNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/DeclarationBlockNode.java index fdfbd8050a748..c76240d56e2da 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/DeclarationBlockNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/DeclarationBlockNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/DeclarationNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/DeclarationNode.java index f5d634b67acf2..0d592f53be267 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/DeclarationNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/DeclarationNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/DefInterfaceReferenceNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/DefInterfaceReferenceNode.java index b655132da5882..bd1c8dd42b801 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/DefInterfaceReferenceNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/DefInterfaceReferenceNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/DoWhileLoopNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/DoWhileLoopNode.java index 1ac475b90ca02..e981ef2a45996 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/DoWhileLoopNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/DoWhileLoopNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/DupNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/DupNode.java index b45e80c50bf9f..eab06c0100a38 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/DupNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/DupNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ElvisNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ElvisNode.java index 034fad7151217..f2ba0b5867b46 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ElvisNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ElvisNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ExpressionNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ExpressionNode.java index 0fdc885851457..4633f9e930fac 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ExpressionNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ExpressionNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/FieldNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/FieldNode.java index 01126387c131e..be4e7b4a03fdc 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/FieldNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/FieldNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/FlipArrayIndexNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/FlipArrayIndexNode.java index a653a53996be0..8be046a7eed91 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/FlipArrayIndexNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/FlipArrayIndexNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/FlipCollectionIndexNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/FlipCollectionIndexNode.java index 9400120468751..23cf38e51a423 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/FlipCollectionIndexNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/FlipCollectionIndexNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/FlipDefIndexNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/FlipDefIndexNode.java index e691ac42c860f..0c9ae6359baf2 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/FlipDefIndexNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/FlipDefIndexNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ForEachLoopNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ForEachLoopNode.java index 5ee8e73b794e2..257fdb873b674 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ForEachLoopNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ForEachLoopNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ForEachSubArrayNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ForEachSubArrayNode.java index b9e9625815c1d..1860d11c459ac 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ForEachSubArrayNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ForEachSubArrayNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ForEachSubIterableNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ForEachSubIterableNode.java index f34d3beb1faec..ff3b350e21fab 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ForEachSubIterableNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ForEachSubIterableNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ForLoopNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ForLoopNode.java index e7bf1e3cbe21a..39940fc0a7d5f 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ForLoopNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ForLoopNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/FunctionNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/FunctionNode.java index 2fc943fe84830..96fe769180755 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/FunctionNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/FunctionNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/IRNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/IRNode.java index 70417f2a8fd01..9c3807b4c77b8 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/IRNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/IRNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/IfElseNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/IfElseNode.java index 26c996dbb8897..1888272eaf4fc 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/IfElseNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/IfElseNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/IfNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/IfNode.java index c2c6351673e3b..94dfe70714522 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/IfNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/IfNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/InstanceofNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/InstanceofNode.java index 95030bef3d1fb..d4b45404777a6 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/InstanceofNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/InstanceofNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/InvokeCallDefNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/InvokeCallDefNode.java index c2258bd44852d..53e89478b4fa3 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/InvokeCallDefNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/InvokeCallDefNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/InvokeCallMemberNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/InvokeCallMemberNode.java index b9f10f41da5c3..89515c598a36c 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/InvokeCallMemberNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/InvokeCallMemberNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/InvokeCallNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/InvokeCallNode.java index 898a93c5a9496..f116135bb0020 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/InvokeCallNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/InvokeCallNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ListInitializationNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ListInitializationNode.java index 440b2a79575ec..5f508e8cce4ec 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ListInitializationNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ListInitializationNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadBraceDefNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadBraceDefNode.java index a1d87705ef2a0..aaf7852d21163 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadBraceDefNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadBraceDefNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadBraceNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadBraceNode.java index 8c2564f03a388..fb21cdeb43b6d 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadBraceNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadBraceNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadDotArrayLengthNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadDotArrayLengthNode.java index 6e899b3bc2004..905cef7e1b026 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadDotArrayLengthNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadDotArrayLengthNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadDotDefNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadDotDefNode.java index e1f3a0de68dc9..b75d5d2214149 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadDotDefNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadDotDefNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadDotNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadDotNode.java index b4045de850ae8..35d2f32f0aef1 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadDotNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadDotNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadDotShortcutNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadDotShortcutNode.java index 6510f1f63642f..46fdaa317458f 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadDotShortcutNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadDotShortcutNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadFieldMemberNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadFieldMemberNode.java index dc597b343797e..37d10b60a8580 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadFieldMemberNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadFieldMemberNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadListShortcutNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadListShortcutNode.java index bfab31478b438..bf892432ea726 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadListShortcutNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadListShortcutNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadMapShortcutNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadMapShortcutNode.java index 3612e9c5a1c65..31d008b51227c 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadMapShortcutNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadMapShortcutNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadVariableNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadVariableNode.java index 9a2e737e804e2..9aa10e9fe7b08 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadVariableNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/LoadVariableNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/MapInitializationNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/MapInitializationNode.java index 1e9f52076e63c..5e3ac13fb5501 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/MapInitializationNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/MapInitializationNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/NewArrayNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/NewArrayNode.java index 68483d3b8a938..499b3d4d5f8d6 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/NewArrayNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/NewArrayNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/NewObjectNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/NewObjectNode.java index dd369e053c08a..adde55946252c 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/NewObjectNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/NewObjectNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/NullNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/NullNode.java index d5c7e09714382..abaa943e5f39e 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/NullNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/NullNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/NullSafeSubNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/NullSafeSubNode.java index 621a91b9b1c33..e257dc0b210c5 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/NullSafeSubNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/NullSafeSubNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ReturnNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ReturnNode.java index b82e818bf9ff6..aecad40c32a43 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ReturnNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ReturnNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StatementExpressionNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StatementExpressionNode.java index 16ebe13d119f0..5901e096615a9 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StatementExpressionNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StatementExpressionNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StatementNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StatementNode.java index eafcd82355582..17e93656b4c1f 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StatementNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StatementNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StaticNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StaticNode.java index 7989dfca1cb1f..258acfda0969b 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StaticNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StaticNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreBraceDefNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreBraceDefNode.java index 742a8cc20f9ad..76599707baf94 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreBraceDefNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreBraceDefNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreBraceNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreBraceNode.java index abf13c9bbbe15..ae7bc73cf4a45 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreBraceNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreBraceNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreDotDefNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreDotDefNode.java index d602a15df08b6..0a4367159740a 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreDotDefNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreDotDefNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreDotNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreDotNode.java index f3d427beada3a..c1e68efb5e8b7 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreDotNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreDotNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreDotShortcutNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreDotShortcutNode.java index d740f43c45534..72f4ee7447eb0 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreDotShortcutNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreDotShortcutNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreFieldMemberNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreFieldMemberNode.java index ee926a772dcb1..8bf9053b767b5 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreFieldMemberNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreFieldMemberNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreListShortcutNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreListShortcutNode.java index f5ae4fa57d969..d911db07f2cec 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreListShortcutNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreListShortcutNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreMapShortcutNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreMapShortcutNode.java index 8abe6a67f2fd2..649d5f8625050 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreMapShortcutNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreMapShortcutNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreVariableNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreVariableNode.java index 2e911bbeec8ba..74524b23215f3 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreVariableNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StoreVariableNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StringConcatenationNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StringConcatenationNode.java index 1b30ce03cc665..5443aa5dd7ff9 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StringConcatenationNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/StringConcatenationNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ThrowNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ThrowNode.java index 71bb67357caf0..9d4f7ad1097aa 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ThrowNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/ThrowNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/TryNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/TryNode.java index 0b417ba253905..7eb46144b3565 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/TryNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/TryNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/TypedCaptureReferenceNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/TypedCaptureReferenceNode.java index ad02be6bf26d1..9ca154fae3f63 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/TypedCaptureReferenceNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/TypedCaptureReferenceNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/TypedInterfaceReferenceNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/TypedInterfaceReferenceNode.java index c638079e2e194..ff7067dc15e0e 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/TypedInterfaceReferenceNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/TypedInterfaceReferenceNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/UnaryMathNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/UnaryMathNode.java index 39941c81d2fda..c1242ca70f18d 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/UnaryMathNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/UnaryMathNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/UnaryNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/UnaryNode.java index f229f39338072..f50dd8d29ed84 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/UnaryNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/UnaryNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/WhileLoopNode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/WhileLoopNode.java index 33a1c3f25d214..ebd953f53ffc7 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/WhileLoopNode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ir/WhileLoopNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.ir; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessCast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessCast.java index 0a5cf4f997106..dc171afe67285 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessCast.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessCast.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.lookup; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClass.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClass.java index e6f7c1a3bb617..e839138ae412a 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClass.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClass.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.lookup; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClassBinding.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClassBinding.java index d63d4d2db4243..48ce94560da0a 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClassBinding.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClassBinding.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.lookup; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClassBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClassBuilder.java index 65f1293d5772b..46254c5b4c21d 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClassBuilder.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClassBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.lookup; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessConstructor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessConstructor.java index e4b1c7087eca7..e22b9c76d3a53 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessConstructor.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessConstructor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.lookup; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessField.java index 49407fed32d41..5fb28a12c06f6 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessField.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessField.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.lookup; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessInstanceBinding.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessInstanceBinding.java index d0343b0812dd8..fd0025b058c5b 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessInstanceBinding.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessInstanceBinding.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.lookup; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java index 5bf8e5cde2afb..1e41e1990e668 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.lookup; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java index d3f14b08c7dc2..2cc09d0b9308f 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.lookup; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java index 766714a454699..a83b40dbb9008 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.lookup; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java index 093ff34ba80cc..85071f7ebfc8c 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.lookup; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/def.java index 730e9c5ee702d..4e26c629853ad 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/def.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/def.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.lookup; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java index 206ff9d8d42d1..a0ff660c34735 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java index aa3117f79906a..989bc6e8a0e2d 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java index d144ab55e27dc..28016c2f1b8e6 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java index f0e76edd469f0..1445753cfcabe 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java index 7c0625357759c..9b962fa75429a 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBooleanComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBooleanComp.java index 224b158f555d6..95c9fc389083d 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBooleanComp.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBooleanComp.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBooleanConstant.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBooleanConstant.java index f62dcd4c310ee..74f703d3cdc85 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBooleanConstant.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBooleanConstant.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBrace.java index e251a238adb60..c09e16555e16f 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBrace.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBrace.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECall.java index dc9891e9e1ff0..5ce5938935753 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECall.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECall.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java index 09bf6a4297f7a..05d04526d8d96 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java index 0e6d0e5a38bd9..a569325ea1ea3 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java index 0dd7b3435cbfc..a0a27c8020880 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java index 057483695f7a6..f22261b1476fa 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDot.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDot.java index 14474393f83d7..5c0acb1580ca3 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDot.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDot.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EElvis.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EElvis.java index 4988f6e97efd8..f28052310ceec 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EElvis.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EElvis.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java index 9cccf33929f84..58ec53839e29e 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java index fdcb238081c5e..0f15ca8572c23 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java index c464b83120928..69c771d5404db 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java index ad8c521314d11..71c2ff64496e2 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java index 71bbc8072c6ae..1ace8b389b703 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java index 093de4f253a91..89ab8d01b7854 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java index 5b948983b8289..7543a0dc4b16f 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArrayFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArrayFunctionRef.java index 4092afe54b1fb..fe0320be41fbd 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArrayFunctionRef.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArrayFunctionRef.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java index 57e63eded368b..3c357c3b5c22d 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java index aed3a6b8fb371..bf5051356f70c 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java index 649831213f50d..4790291638e86 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ERegex.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ERegex.java index d625dd1e4aa7d..a184592d35a05 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ERegex.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ERegex.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EString.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EString.java index 3ac468c51183f..c7858ddf99af9 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EString.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EString.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ESymbol.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ESymbol.java index b299542c26e2d..332bfefbad367 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ESymbol.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ESymbol.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java index 9a3791f159381..5e5d12d1cf5b1 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java index 567db6d378716..47a99b9518985 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java index ca884cdf233c7..eeebd449e2c35 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java index c156fdc052186..78bb9a3710984 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SClass.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SClass.java index d129d79d86f78..eadf1e41ca600 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SClass.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SClass.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java index b482e03dd0baf..6e67518d01b4b 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java index 137f071ce66de..ad4f41adde6ea 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java index 747ae1fe884c8..14ca91a8df8a6 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java index bce1950e7f29f..dfbcb8be894c3 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java index 1c294366ae0dc..db68711b0502e 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java index e5c9c6362a1af..5b06f472b3a1c 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java index df3f63d748bea..0bb6a8669af9a 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java index 0fd0dd710e9b9..e4d53b2831bf8 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java index fbd8d03c13336..cc17193e537a4 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java index ce09a72943c0b..e53adb05d2516 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java index b19082c99e257..efa46bbf52305 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java index 7df35036528b1..251f5158fcb25 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java index 71f62d19b4258..29ab59da1ad02 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java index ed771b6c4e88f..3a53c151846e7 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.node; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java index 8bde818d1fd7f..895ea0b160196 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/package-info.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/package-info.java index 2116518ec4475..5b9b8e25bf936 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/package-info.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultConstantFoldingOptimizationPhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultConstantFoldingOptimizationPhase.java index 3cba26f7b2da1..8c4cea9104c2b 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultConstantFoldingOptimizationPhase.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultConstantFoldingOptimizationPhase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.phase; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultEqualityMethodOptimizationPhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultEqualityMethodOptimizationPhase.java index 5b1f9ec809bfe..60e22037fbdf2 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultEqualityMethodOptimizationPhase.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultEqualityMethodOptimizationPhase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.phase; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultIRTreeToASMBytesPhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultIRTreeToASMBytesPhase.java index 084d5ee8e6c30..04d74882f54ae 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultIRTreeToASMBytesPhase.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultIRTreeToASMBytesPhase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.phase; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticAnalysisPhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticAnalysisPhase.java index ddb7b060062b5..7dffe1e500f6d 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticAnalysisPhase.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticAnalysisPhase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.phase; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticHeaderPhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticHeaderPhase.java index 61be12039cbe7..7ccd53b7ac7e0 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticHeaderPhase.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticHeaderPhase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.phase; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultStaticConstantExtractionPhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultStaticConstantExtractionPhase.java index e6e3c08849742..0842cbae9bc67 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultStaticConstantExtractionPhase.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultStaticConstantExtractionPhase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.phase; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultStringConcatenationOptimizationPhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultStringConcatenationOptimizationPhase.java index 24a36e73200c5..547d36456ff52 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultStringConcatenationOptimizationPhase.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultStringConcatenationOptimizationPhase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.phase; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java index 5e9ba3601e11c..0426bd03fd513 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.phase; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/IRExpressionModifyingVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/IRExpressionModifyingVisitor.java index b26acb5424746..ccf13a27e07e0 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/IRExpressionModifyingVisitor.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/IRExpressionModifyingVisitor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.phase; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/IRTreeBaseVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/IRTreeBaseVisitor.java index d2de28a26552a..a7e750c54fad4 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/IRTreeBaseVisitor.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/IRTreeBaseVisitor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.phase; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/IRTreeVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/IRTreeVisitor.java index 4ab3b3cc83969..7062de11fa295 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/IRTreeVisitor.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/IRTreeVisitor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.phase; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessSemanticAnalysisPhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessSemanticAnalysisPhase.java index 0f77fb90b5b59..61536c46c0cf4 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessSemanticAnalysisPhase.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessSemanticAnalysisPhase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.phase; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessSemanticHeaderPhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessSemanticHeaderPhase.java index 083f14d88b089..98f2e27b234bd 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessSemanticHeaderPhase.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessSemanticHeaderPhase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.phase; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessUserTreeToIRTreePhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessUserTreeToIRTreePhase.java index 4be4d07e449f1..72ecf1fe850ec 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessUserTreeToIRTreePhase.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessUserTreeToIRTreePhase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.phase; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/UserTreeBaseVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/UserTreeBaseVisitor.java index 7917b106c4202..58d260fe0d16e 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/UserTreeBaseVisitor.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/UserTreeBaseVisitor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.phase; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/UserTreeVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/UserTreeVisitor.java index 82f58a2785312..2ea3a5625a7e0 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/UserTreeVisitor.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/UserTreeVisitor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.phase; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/Decorations.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/Decorations.java index 0b615f33be069..d3fc650e49965 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/Decorations.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/Decorations.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.symbol; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/Decorator.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/Decorator.java index ed6ca0f68c406..4192556b2f439 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/Decorator.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/Decorator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.symbol; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/FunctionTable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/FunctionTable.java index 70e2a211808ae..240acf17b499c 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/FunctionTable.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/FunctionTable.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.symbol; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/IRDecorations.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/IRDecorations.java index 9922e439e44b6..bb63b291c0ec4 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/IRDecorations.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/IRDecorations.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.symbol; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/ScriptScope.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/ScriptScope.java index 1a5465ebaa1c5..65c9c7a1a10c0 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/ScriptScope.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/ScriptScope.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.symbol; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/SemanticScope.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/SemanticScope.java index 2d5dd2668e34e..4900afc719495 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/SemanticScope.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/SemanticScope.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.symbol; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/WriteScope.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/WriteScope.java index 46cfbdbcfa944..35aa30f8d357b 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/WriteScope.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/WriteScope.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.symbol; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/toxcontent/DecorationToXContent.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/toxcontent/DecorationToXContent.java index f4f56dcbc45cd..063e2d8e0b4c6 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/toxcontent/DecorationToXContent.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/toxcontent/DecorationToXContent.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.toxcontent; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/toxcontent/XContentBuilderWrapper.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/toxcontent/XContentBuilderWrapper.java index e6759392452df..6ecb212585c23 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/toxcontent/XContentBuilderWrapper.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/toxcontent/XContentBuilderWrapper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.toxcontent; diff --git a/modules/lang-painless/src/main/plugin-metadata/entitlement-policy.yaml b/modules/lang-painless/src/main/plugin-metadata/entitlement-policy.yaml new file mode 100644 index 0000000000000..d7e4ad872fc32 --- /dev/null +++ b/modules/lang-painless/src/main/plugin-metadata/entitlement-policy.yaml @@ -0,0 +1,2 @@ +org.elasticsearch.painless: + - create_class_loader diff --git a/modules/lang-painless/src/main/plugin-metadata/plugin-security.policy b/modules/lang-painless/src/main/plugin-metadata/plugin-security.policy index 90ed5a1f9278c..794044a2669c4 100644 --- a/modules/lang-painless/src/main/plugin-metadata/plugin-security.policy +++ b/modules/lang-painless/src/main/plugin-metadata/plugin-security.policy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ grant { diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt index 3a2cdcd8efe45..23a77865a2e0a 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.math.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.math.txt index 516c458cd1887..1c9ba75c52793 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.math.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.math.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.nio.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.nio.txt index 40f0438a1442b..7014617372e47 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.nio.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.nio.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # class java.nio.ByteOrder { @@ -78,4 +79,4 @@ class java.nio.ShortBuffer { # TODO: https:#github.com/elastic/elasticsearch/issues/79867 # ShortBuffer get(int, short[]) # ShortBuffer get(int, short[], int, int) -} \ No newline at end of file +} diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.text.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.text.txt index 97f0772e4fa2a..8fd9af37ee822 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.text.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.text.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.chrono.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.chrono.txt index 202bc97331e13..76c5f1970e83a 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.chrono.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.chrono.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.format.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.format.txt index 0c4badd8b5f00..cca86ee73b719 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.format.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.format.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.temporal.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.temporal.txt index de7cd586a3068..b802a65dc44fe 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.temporal.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.temporal.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.txt index cd727b1b29bbe..74fa8622afcdd 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.zone.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.zone.txt index c8cf31fddb7a0..de1407ea824df 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.zone.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.zone.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.function.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.function.txt index 09dbdac35f8d2..f08fcc73f9f7f 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.function.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.function.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.regex.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.regex.txt index 53e276c90a4ea..9c84575fa4bdc 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.regex.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.regex.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.stream.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.stream.txt index 8df7455b294d9..610900a09f44f 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.stream.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.stream.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt index 045905c358cd2..14db6336a880d 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.json.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.json.txt index 4b752feb71bd9..7bd9eee945d50 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.json.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.json.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # class org.elasticsearch.painless.api.Json { diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.net.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.net.txt index 1d16b7951c834..8a55efbed0caa 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.net.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.net.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # class org.elasticsearch.script.field.IPAddress { diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.aggs.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.aggs.txt index a09bce101a155..e23289093964e 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.aggs.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.aggs.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # The whitelist for the fields api diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.aggs_map.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.aggs_map.txt index 84df365c4fe2c..0f4dbe276a781 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.aggs_map.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.aggs_map.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # The whitelist for the fields api diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.boolean_field.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.boolean_field.txt index ffc61be086208..4d662d4d5070b 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.boolean_field.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.boolean_field.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # The whitelist for boolean-valued runtime fields diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.bytesref_sort.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.bytesref_sort.txt index 5071cade77461..b3b0788870223 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.bytesref_sort.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.bytesref_sort.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # The whitelist for the fields api diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.composite_field.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.composite_field.txt index b5c499abb8777..317d0ea3ccd76 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.composite_field.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.composite_field.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # The whitelist for composite runtime fields diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.date_field.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.date_field.txt index dca45c308acd4..e8a66fde81834 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.date_field.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.date_field.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # The whitelist for date-valued runtime fields diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.double_field.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.double_field.txt index 394c333027d06..bc62a028aae0f 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.double_field.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.double_field.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # The whitelist for double-valued runtime fields diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.field.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.field.txt index 9f1c4b2368bc8..b3468da53e7c8 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.field.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.field.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # The whitelist for the fields api diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt index 412c7704e14ee..46d1b5ab2d51c 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # The whitelist for the fields api @@ -131,6 +132,21 @@ class org.elasticsearch.script.field.SeqNoDocValuesField @dynamic_type { class org.elasticsearch.script.field.VersionDocValuesField @dynamic_type { } +class org.elasticsearch.script.field.vectors.RankVectors { + RankVectors EMPTY + float[] getMagnitudes() + + Iterator getVectors() + boolean isEmpty() + int getDims() + int size() +} + +class org.elasticsearch.script.field.vectors.RankVectorsDocValuesField { + RankVectors get() + RankVectors get(RankVectors) +} + class org.elasticsearch.script.field.vectors.DenseVector { DenseVector EMPTY float getMagnitude() diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.filter.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.filter.txt index 15ffc4e68f2ef..dba6f357e6d40 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.filter.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.filter.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # The whitelist for the fields api diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.geo_point_field.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.geo_point_field.txt index 2cf8961b434ca..54155ad6047f1 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.geo_point_field.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.geo_point_field.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # The whitelist for ip-valued runtime fields diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.geometry_field.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.geometry_field.txt index 68bcbf922e869..85ea9027e8470 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.geometry_field.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.geometry_field.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # The whitelist for runtime fields that generate geometries diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.txt index 13678c4216d7a..d7fbebb79841a 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # This file contains a whitelist for the ingest scripts diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ip_field.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ip_field.txt index dd17ccf34f480..1cfe25a7c12e9 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ip_field.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ip_field.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # The whitelist for ip-valued runtime fields diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.keyword_field.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.keyword_field.txt index 7e6e308d055cc..c369d35d3156c 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.keyword_field.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.keyword_field.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # The whitelist for keyword runtime fields diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.long_field.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.long_field.txt index 138f89d78d393..eea40e1785918 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.long_field.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.long_field.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # The whitelist for long-valued runtime fields diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.number_sort.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.number_sort.txt index c25c7729f0cfc..8d7c9765e07b6 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.number_sort.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.number_sort.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # The whitelist for the fields api diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.reindex.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.reindex.txt index 18d658d797b60..c110c98710225 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.reindex.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.reindex.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # This file contains a whitelist for reindex scripts diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt index 0dab7dcbadfb5..fd8c81464b4a8 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # This file contains a whitelist for functions to be used in Score context diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.string_sort.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.string_sort.txt index 1e653e56bbc8a..9df5839fe369f 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.string_sort.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.string_sort.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # The whitelist for the fields api diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.update.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.update.txt index 214fdaae26394..5e8a5d518bf23 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.update.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.update.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # This file contains a whitelist for update scripts diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.update_by_query.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.update_by_query.txt index 6c569a165336b..971dddecbaa57 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.update_by_query.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.update_by_query.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # This file contains a whitelist for update_by_query scripts diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt index b8861c85fbb79..e8d19181ae2f2 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # @@ -122,6 +123,11 @@ class org.elasticsearch.index.mapper.vectors.DenseVectorScriptDocValues { float getMagnitude() } +class org.elasticsearch.index.mapper.vectors.RankVectorsScriptDocValues { + Iterator getVectorValues() + float[] getMagnitudes() +} + class org.apache.lucene.util.BytesRef { byte[] bytes int offset diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java index fe907a1c8640d..75231b8f9e8a5 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AliasTestClass.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AliasTestClass.java index 9ebec9647918c..8edc067bda032 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AliasTestClass.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AliasTestClass.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AliasTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AliasTests.java index 1d74de8549435..367921ec589f3 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AliasTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AliasTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AliasedTestInnerClass.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AliasedTestInnerClass.java index 1c745daca0b40..50d450c42c83f 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AliasedTestInnerClass.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AliasedTestInnerClass.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AnalyzerCasterTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AnalyzerCasterTests.java index 10d09f08300ed..d0ebb3fddd630 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AnalyzerCasterTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AnalyzerCasterTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AndTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AndTests.java index c80d82b6776cf..b3ce6a4357bdd 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AndTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AndTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayLikeObjectTestCase.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayLikeObjectTestCase.java index 6edc5cf1a5db1..cb89f2d693427 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayLikeObjectTestCase.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayLikeObjectTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java index 5ab1a154ade5f..cf06269f93f5f 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java index 6d951299b80c6..3ddfc895bffbd 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java index 1acaa83493ee2..50d7a664912ba 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java index 2b3fb68e4f4c2..6f762abf61422 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java index 04f3f346e86d0..18f5e9d9bdd7a 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java index 14f30a895911b..4417e8194b675 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BindingsTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BindingsTests.java index a751e9c82ec2a..6fc748d6c9d84 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BindingsTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BindingsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BoxedCastTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BoxedCastTests.java index 9b1f02e8eeb9b..2562d145db35f 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BoxedCastTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BoxedCastTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BufferTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BufferTests.java index d77e26120c2f7..103ed1a2289d5 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BufferTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BufferTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/CidrTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/CidrTests.java index 75cd05c932b31..99587fd331fe0 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/CidrTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/CidrTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ComparisonTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ComparisonTests.java index b7bd562a3ee0a..7d1e6557b161c 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ComparisonTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ComparisonTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ConditionalTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ConditionalTests.java index 0e995599eeeec..a82d771de8088 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ConditionalTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ConditionalTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ConstantFoldingTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ConstantFoldingTests.java index b2556368fb38b..4ce010679e916 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ConstantFoldingTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ConstantFoldingTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ContextExampleTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ContextExampleTests.java index 49d03220295b0..4da63c2328050 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ContextExampleTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ContextExampleTests.java @@ -1,10 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DateTimeTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DateTimeTests.java index 77199d30eb4be..addb5f57c51e5 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DateTimeTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DateTimeTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java index 3d539f7e3cb85..aceac7fb4c960 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java index b60d8a0fd7ce8..181b719529777 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java index 9ad29bbe84f7f..b22329b8d6bfb 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefCastTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefCastTests.java index 1400ed95d0b32..d358ffdc4c57a 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefCastTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefCastTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefEncodingTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefEncodingTests.java index f8df4424d826a..530bba0a89db7 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefEncodingTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefEncodingTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java index ffbe91bd46031..d9c36485aece2 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DivisionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DivisionTests.java index 6859bec1b9a25..701f2ebc43576 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DivisionTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DivisionTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DynamicTypeTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DynamicTypeTests.java index e7f370d4b8a7f..6d9f43431fe7c 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DynamicTypeTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DynamicTypeTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ElvisTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ElvisTests.java index a4d65d9d8c7af..3e891a01bd5ad 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ElvisTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ElvisTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/EmitTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/EmitTests.java index eb1626af454b7..edc7414168ee1 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/EmitTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/EmitTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/EqualsTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/EqualsTests.java index 1c622c33edd8d..5ca9c02d3f6db 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/EqualsTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/EqualsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ErrorCauseWrapperTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ErrorCauseWrapperTests.java index d2206b3522fc4..510f5e2ee2cca 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ErrorCauseWrapperTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ErrorCauseWrapperTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FactoryTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FactoryTests.java index 8d15e0b70d299..3757d5ac47cb9 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FactoryTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FeatureTestAugmentationObject.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FeatureTestAugmentationObject.java index 7d923218fdb1a..79355f5ad3812 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FeatureTestAugmentationObject.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FeatureTestAugmentationObject.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FeatureTestObject.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FeatureTestObject.java index 7b13d582a139d..804b4ab65592f 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FeatureTestObject.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FeatureTestObject.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FeatureTestObject2.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FeatureTestObject2.java index 01e7935a7affc..b1b8ba32862c8 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FeatureTestObject2.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FeatureTestObject2.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FeatureTestObject3.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FeatureTestObject3.java index e279b9b646e88..7f88ea6d9853a 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FeatureTestObject3.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FeatureTestObject3.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java index 1cb22494fdb9e..4568a1ae22c31 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionTests.java index ceb49ae3b40bc..dcdcffaff7b77 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/GeneralCastTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/GeneralCastTests.java index 221e21da862f5..ae6f0ade6bce7 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/GeneralCastTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/GeneralCastTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/GetByPathAugmentationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/GetByPathAugmentationTests.java index 3022eb0fccc87..b0a4090e80431 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/GetByPathAugmentationTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/GetByPathAugmentationTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/IncrementTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/IncrementTests.java index 87887477ab87e..9bd4a2ec9037a 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/IncrementTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/IncrementTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/InitializerTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/InitializerTests.java index 2e60138f6eb35..785c8be2c7cd9 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/InitializerTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/InitializerTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/InjectionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/InjectionTests.java index 8c38ea49c26f6..d596eb69f01ef 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/InjectionTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/InjectionTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/IntegerOverflowTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/IntegerOverflowTests.java index 04cd8f48688bd..8da89a9630457 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/IntegerOverflowTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/IntegerOverflowTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/JsonTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/JsonTests.java index abdc867cb8eed..3b3a7b5bc1fb9 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/JsonTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/JsonTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java index 1ac1ce9c3a749..01e9c8b0c5745 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ListTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ListTests.java index 17e222357555e..5a375dad92b11 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ListTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ListTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LookupTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LookupTests.java index 6a403c0692540..260077db5f67c 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LookupTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LookupTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/MapTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/MapTests.java index 6e619401bf84e..371f1262bf482 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/MapTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/MapTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/MultiplicationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/MultiplicationTests.java index 9b3be7025b3f5..d5a58bfe59608 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/MultiplicationTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/MultiplicationTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java index 96181458bd496..6837d94b55079 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NoSemiColonTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NoSemiColonTests.java index 4e8b3da943f40..70bd9e5ed63b4 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NoSemiColonTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NoSemiColonTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NumberTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NumberTests.java index 24975fdc714f8..c2bf99c2c0c78 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NumberTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NumberTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OrTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OrTests.java index 7d0390d292d14..328abc3ca05d3 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OrTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OrTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java index fb685f09c2f50..527e30556c20a 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PostfixTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PostfixTests.java index b34618eebeb35..917e704e59aed 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PostfixTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PostfixTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PromotionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PromotionTests.java index abec712e7a7f9..55e32e1e670bc 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PromotionTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PromotionTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexLimitTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexLimitTests.java index 265fba1327191..fb891473c48af 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexLimitTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexLimitTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java index 3e4e057b73365..14776310dbe2a 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RemainderTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RemainderTests.java index 080b99b6ddad4..af258fda7645a 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RemainderTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RemainderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScoreScriptTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScoreScriptTests.java index 08b55fdf3bcc3..eeb33dfa92bba 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScoreScriptTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScoreScriptTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptEngineTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptEngineTests.java index 7e0379616d7bb..f9cb53593e3c8 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptEngineTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptEngineTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java index dbcb4e791cdd8..35d5e333b5501 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptedMetricAggContextsTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptedMetricAggContextsTests.java index 2d3f09fc7243a..fed598e46fbd9 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptedMetricAggContextsTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptedMetricAggContextsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ShiftTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ShiftTests.java index 0567200412262..eda2d14f671e6 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ShiftTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ShiftTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/SimilarityScriptTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/SimilarityScriptTests.java index dbfd1327fb998..01a9e995450aa 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/SimilarityScriptTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/SimilarityScriptTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StandardCastTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StandardCastTests.java index 04b811fc7de13..9298a789aef42 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StandardCastTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StandardCastTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StatementTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StatementTests.java index 52df613f9b647..404e8794097e8 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StatementTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StatementTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StaticTestObject.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StaticTestObject.java index 2a9476bdb923e..bc3b5c870ab5f 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StaticTestObject.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StaticTestObject.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java index 8d4d55468815f..6e92d08c84abf 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/SubtractionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/SubtractionTests.java index 6682d18b0fa91..06f28a93c19b5 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/SubtractionTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/SubtractionTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/TestFieldScript.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/TestFieldScript.java index b4fe2db59da85..0986d6266d8e3 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/TestFieldScript.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/TestFieldScript.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ThisTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ThisTests.java index cfa62ea33c0ea..7362b009b6296 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ThisTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ThisTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/TryCatchTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/TryCatchTests.java index 3ab989515018e..98bf5928ba63f 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/TryCatchTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/TryCatchTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/UnaryTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/UnaryTests.java index 1eb69fce87964..7879c191a864b 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/UnaryTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/UnaryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/UserFunctionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/UserFunctionTests.java index cb84750f2d096..8753b759a6abc 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/UserFunctionTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/UserFunctionTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java index 664602674b5d3..52b796bf24436 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/XorTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/XorTests.java index 586fb08b171b0..7e224a641791c 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/XorTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/XorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/ContextInfoTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/ContextInfoTests.java index b7aa1c117233c..86bb31ad396f2 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/ContextInfoTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/ContextInfoTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.action; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteApiTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteApiTests.java index 9da040e1d34bc..56dec6cf4a58d 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteApiTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteApiTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.action; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteRequestTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteRequestTests.java index 0126be744bbd9..2249441b20964 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteRequestTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteRequestTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.action; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteResponseTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteResponseTests.java index 20db938bdfa78..3f8d777046c26 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteResponseTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteResponseTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.action; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/SuggestTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/SuggestTests.java index f3e10ac0d1162..746f6c5377656 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/SuggestTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/SuggestTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.action; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/api/CIDRTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/api/CIDRTests.java index 1c1984c0a0d69..bafbdc2dc48f2 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/api/CIDRTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/api/CIDRTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.api; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/api/LimitedCharSequenceTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/api/LimitedCharSequenceTests.java index b657ace7bd038..e27b053a314f5 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/api/LimitedCharSequenceTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/api/LimitedCharSequenceTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless.api; diff --git a/modules/lang-painless/src/yamlRestTest/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java b/modules/lang-painless/src/yamlRestTest/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java index 04c944cbc8874..9fa2b8ea2088a 100644 --- a/modules/lang-painless/src/yamlRestTest/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java +++ b/modules/lang-painless/src/yamlRestTest/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/146_dense_vector_bit_basic.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/146_dense_vector_bit_basic.yml index 4c195a0e32623..cdd65ca0eb296 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/146_dense_vector_bit_basic.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/146_dense_vector_bit_basic.yml @@ -101,9 +101,15 @@ setup: - match: {hits.hits.2._id: "3"} - close_to: {hits.hits.2._score: {value: 3.4641016, error: 0.01}} - --- "Dot Product is not supported": + - skip: + features: [capabilities] + capabilities: + - method: POST + path: /_search + capabilities: [ byte_float_bit_dot_product_with_bugfix ] + reason: Capability required to run test - do: catch: bad_request headers: @@ -131,7 +137,6 @@ setup: source: "dotProduct(params.query_vector, 'vector')" params: query_vector: "006ff30e84" - --- "Cosine Similarity is not supported": - do: @@ -388,3 +393,119 @@ setup: - match: {hits.hits.2._id: "3"} - match: {hits.hits.2._score: 11.0} +--- +"Dot product with float": + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ byte_float_bit_dot_product_with_bugfix ] + test_runner_features: [capabilities, close_to] + reason: Capability required to run test + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + body: + query: + script_score: + query: { match_all: { } } + script: + source: "dotProduct(params.query_vector, 'vector')" + params: + query_vector: [0.23, 1.45, 3.67, 4.89, -0.56, 2.34, 3.21, 1.78, -2.45, 0.98, -0.12, 3.45, 4.56, 2.78, 1.23, 0.67, 3.89, 4.12, -2.34, 1.56, 0.78, 3.21, 4.12, 2.45, -1.67, 0.34, -3.45, 4.56, -2.78, 1.23, -0.67, 3.89, -4.34, 2.12, -1.56, 0.78, -3.21, 4.45, 2.12, 1.67] + + - match: { hits.total: 3 } + + - match: {hits.hits.0._id: "2"} + - close_to: {hits.hits.0._score: {value: 33.78, error: 0.01}} + + - match: {hits.hits.1._id: "3"} + - close_to: {hits.hits.1._score:{value: 22.579, error: 0.01}} + + - match: {hits.hits.2._id: "1"} + - close_to: {hits.hits.2._score: {value: 11.919, error: 0.01}} + + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + body: + query: + script_score: + query: { match_all: { } } + script: + source: "dotProduct(params.query_vector, 'indexed_vector')" + params: + query_vector: [0.23, 1.45, 3.67, 4.89, -0.56, 2.34, 3.21, 1.78, -2.45, 0.98, -0.12, 3.45, 4.56, 2.78, 1.23, 0.67, 3.89, 4.12, -2.34, 1.56, 0.78, 3.21, 4.12, 2.45, -1.67, 0.34, -3.45, 4.56, -2.78, 1.23, -0.67, 3.89, -4.34, 2.12, -1.56, 0.78, -3.21, 4.45, 2.12, 1.67] + + - match: { hits.total: 3 } + + - match: {hits.hits.0._id: "2"} + - close_to: {hits.hits.0._score: {value: 33.78, error: 0.01}} + + - match: {hits.hits.1._id: "3"} + - close_to: {hits.hits.1._score:{value: 22.579, error: 0.01}} + + - match: {hits.hits.2._id: "1"} + - close_to: {hits.hits.2._score: {value: 11.919, error: 0.01}} +--- +"Dot product with byte": + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ byte_float_bit_dot_product_with_bugfix ] + test_runner_features: capabilities + reason: Capability required to run test + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + body: + query: + script_score: + query: { match_all: { } } + script: + source: "dotProduct(params.query_vector, 'vector')" + params: + query_vector: [12, -34, 56, -78, 90, 12, 34, -56, 78, -90, 23, -45, 67, -89, 12, 34, 56, 78, 90, -12, 34, -56, 78, -90, 23, -45, 67, -89, 12, -34, 56, -78, 90, -12, 34, -56, 78, 90, 23, -45] + + - match: { hits.total: 3 } + + - match: {hits.hits.0._id: "3"} + - match: {hits.hits.0._score: 415} + + - match: {hits.hits.1._id: "1"} + - match: {hits.hits.1._score: 168} + + - match: {hits.hits.2._id: "2"} + - match: {hits.hits.2._score: 126} + + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + body: + query: + script_score: + query: { match_all: { } } + script: + source: "dotProduct(params.query_vector, 'indexed_vector')" + params: + query_vector: [12, -34, 56, -78, 90, 12, 34, -56, 78, -90, 23, -45, 67, -89, 12, 34, 56, 78, 90, -12, 34, -56, 78, -90, 23, -45, 67, -89, 12, -34, 56, -78, 90, -12, 34, -56, 78, 90, 23, -45] + + - match: { hits.total: 3 } + + - match: {hits.hits.0._id: "3"} + - match: {hits.hits.0._score: 415} + + - match: {hits.hits.1._id: "1"} + - match: {hits.hits.1._score: 168} + + - match: {hits.hits.2._id: "2"} + - match: {hits.hits.2._score: 126} diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml index ee6803d809087..b908f729a0159 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml @@ -1,4 +1,7 @@ setup: + - requires: + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" - do: indices.create: index: test @@ -180,9 +183,9 @@ setup: body: settings: number_of_shards: 1 + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: boolean: type: boolean @@ -5630,9 +5633,9 @@ version and sequence number synthetic _source: body: settings: number_of_shards: 1 + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: keyword: type: keyword diff --git a/modules/legacy-geo/build.gradle b/modules/legacy-geo/build.gradle index 1815fa27d8cba..d5e108eb3235b 100644 --- a/modules/legacy-geo/build.gradle +++ b/modules/legacy-geo/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.internal.info.BuildParams @@ -11,8 +12,8 @@ import org.elasticsearch.gradle.internal.info.BuildParams apply plugin: 'elasticsearch.internal-cluster-test' esplugin { - description 'Placeholder plugin for geospatial features in ES' - classname 'org.elasticsearch.legacygeo.LegacyGeoPlugin' + description = 'Placeholder plugin for geospatial features in ES' + classname = 'org.elasticsearch.legacygeo.LegacyGeoPlugin' } dependencies { @@ -25,7 +26,7 @@ dependencies { testImplementation project(":test:framework") } -if (BuildParams.isSnapshotBuild() == false) { +if (buildParams.isSnapshotBuild() == false) { tasks.named("test").configure { systemProperty 'es.index_mode_feature_flag_registered', 'true' } diff --git a/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/GeoBoundingBoxQueryLegacyGeoShapeIT.java b/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/GeoBoundingBoxQueryLegacyGeoShapeIT.java index 76c02c59f1ff1..37c31c8af47b0 100644 --- a/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/GeoBoundingBoxQueryLegacyGeoShapeIT.java +++ b/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/GeoBoundingBoxQueryLegacyGeoShapeIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.search; diff --git a/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeIT.java b/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeIT.java index 2b9c35429c328..918c343b79b7b 100644 --- a/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeIT.java +++ b/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.search; diff --git a/modules/legacy-geo/src/main/java/module-info.java b/modules/legacy-geo/src/main/java/module-info.java index 793b5094ff94e..b5c62ae18b066 100644 --- a/modules/legacy-geo/src/main/java/module-info.java +++ b/modules/legacy-geo/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.legacy.geo { diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/GeoShapeType.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/GeoShapeType.java index 0bef69ca26a6b..f576eb0662df9 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/GeoShapeType.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/GeoShapeType.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo; diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/LegacyGeoPlugin.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/LegacyGeoPlugin.java index edd26afe526ca..260f7e6ae62e2 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/LegacyGeoPlugin.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/LegacyGeoPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo; diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/ShapesAvailability.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/ShapesAvailability.java index 41d574fc240d2..449fbaa3e79eb 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/ShapesAvailability.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/ShapesAvailability.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo; diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/XShapeCollection.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/XShapeCollection.java index 891bd4b546a59..7c77353755907 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/XShapeCollection.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/XShapeCollection.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo; diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/CircleBuilder.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/CircleBuilder.java index a7576913901cd..23bab9e7ceb24 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/CircleBuilder.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/CircleBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.builders; diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/CoordinatesBuilder.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/CoordinatesBuilder.java index c537c0d3b4a82..baf8f87c5ab25 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/CoordinatesBuilder.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/CoordinatesBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.builders; diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/EnvelopeBuilder.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/EnvelopeBuilder.java index ab21d6f346003..d8c654b8efdd7 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/EnvelopeBuilder.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/EnvelopeBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.builders; diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/GeometryCollectionBuilder.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/GeometryCollectionBuilder.java index 44879fc01eff3..1c577d27d940c 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/GeometryCollectionBuilder.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/GeometryCollectionBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.builders; diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/LineStringBuilder.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/LineStringBuilder.java index 35574b47f5b93..419b7a4d2bcd7 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/LineStringBuilder.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/LineStringBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.builders; diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/MultiLineStringBuilder.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/MultiLineStringBuilder.java index 9756856b072d7..3391f24fb1b6f 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/MultiLineStringBuilder.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/MultiLineStringBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.builders; diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/MultiPointBuilder.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/MultiPointBuilder.java index 4e6c5620ff1ed..76626885c1dbe 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/MultiPointBuilder.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/MultiPointBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.builders; diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/MultiPolygonBuilder.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/MultiPolygonBuilder.java index be943c3f5066d..8154992c8922e 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/MultiPolygonBuilder.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/MultiPolygonBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.builders; diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/PointBuilder.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/PointBuilder.java index a7aa6885c3bc4..762b6c1ece1ed 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/PointBuilder.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/PointBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.builders; diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/PolygonBuilder.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/PolygonBuilder.java index beef840756e78..f1ef207de3d7a 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/PolygonBuilder.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/PolygonBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.builders; diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/ShapeBuilder.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/ShapeBuilder.java index 31d610fbdc57a..8ba609684dd32 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/ShapeBuilder.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/builders/ShapeBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.builders; diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java index f41d365f305bd..b0634f0f1332f 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.mapper; @@ -45,6 +46,7 @@ import org.elasticsearch.legacygeo.builders.ShapeBuilder; import org.elasticsearch.legacygeo.parsers.ShapeParser; import org.elasticsearch.legacygeo.query.LegacyGeoShapeQueryProcessor; +import org.elasticsearch.lucene.spatial.CoordinateEncoder; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; import org.locationtech.spatial4j.shape.Point; @@ -529,6 +531,17 @@ public PrefixTreeStrategy resolvePrefixTreeStrategy(String strategyName) { protected Function>, List> getFormatter(String format) { return GeometryFormatterFactory.getFormatter(format, ShapeBuilder::buildGeometry); } + + @Override + protected boolean isBoundsExtractionSupported() { + // Extracting bounds for geo shapes is not implemented yet. + return false; + } + + @Override + protected CoordinateEncoder coordinateEncoder() { + return CoordinateEncoder.GEO; + } } private final IndexVersion indexCreatedVersion; diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/parsers/CoordinateNode.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/parsers/CoordinateNode.java index b3c42ae97f45d..fe0b5a834dd1a 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/parsers/CoordinateNode.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/parsers/CoordinateNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.parsers; diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/parsers/GeoJsonParser.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/parsers/GeoJsonParser.java index 743b8b1ec8f0f..da1882f0abe9d 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/parsers/GeoJsonParser.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/parsers/GeoJsonParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.parsers; diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/parsers/GeoWKTParser.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/parsers/GeoWKTParser.java index 805ed8a65597a..7ac53320c866d 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/parsers/GeoWKTParser.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/parsers/GeoWKTParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.parsers; diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/parsers/ShapeParser.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/parsers/ShapeParser.java index 1b22c24bb57c1..cf4bcb656cb63 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/parsers/ShapeParser.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/parsers/ShapeParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.parsers; diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/query/LegacyGeoShapeQueryProcessor.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/query/LegacyGeoShapeQueryProcessor.java index cad709358a19e..0b91f68ea4fae 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/query/LegacyGeoShapeQueryProcessor.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/query/LegacyGeoShapeQueryProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.query; diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/BaseGeoParsingTestCase.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/BaseGeoParsingTestCase.java index 51c0addb5d8da..6a6a2e6d61d39 100644 --- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/BaseGeoParsingTestCase.java +++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/BaseGeoParsingTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo; diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/GeoJsonShapeParserTests.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/GeoJsonShapeParserTests.java index 4fa1d7b7a3108..bd5b289abc588 100644 --- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/GeoJsonShapeParserTests.java +++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/GeoJsonShapeParserTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo; diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/GeoWKTShapeParserTests.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/GeoWKTShapeParserTests.java index 6e8a61277cccf..f944a368b2a6c 100644 --- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/GeoWKTShapeParserTests.java +++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/GeoWKTShapeParserTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo; diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/GeometryIOTests.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/GeometryIOTests.java index 2d484c73526a5..47be506427e12 100644 --- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/GeometryIOTests.java +++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/GeometryIOTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo; diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/ShapeBuilderTests.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/ShapeBuilderTests.java index 0603533a8503f..bdb32c4f2e761 100644 --- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/ShapeBuilderTests.java +++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/ShapeBuilderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo; diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/AbstractShapeBuilderTestCase.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/AbstractShapeBuilderTestCase.java index 972e72481c771..a0a871f54d3c8 100644 --- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/AbstractShapeBuilderTestCase.java +++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/AbstractShapeBuilderTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.builders; diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/CircleBuilderTests.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/CircleBuilderTests.java index 72ab85016aac6..e0cca1e826560 100644 --- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/CircleBuilderTests.java +++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/CircleBuilderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.builders; diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/EnvelopeBuilderTests.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/EnvelopeBuilderTests.java index 28fc05eca424d..1664da96cd30a 100644 --- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/EnvelopeBuilderTests.java +++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/EnvelopeBuilderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.builders; diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/GeometryCollectionBuilderTests.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/GeometryCollectionBuilderTests.java index 8f2c498f55fa0..2336b0ae62bdc 100644 --- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/GeometryCollectionBuilderTests.java +++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/GeometryCollectionBuilderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.builders; diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/LineStringBuilderTests.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/LineStringBuilderTests.java index 86af9ece0cf15..e8af91dab7bf5 100644 --- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/LineStringBuilderTests.java +++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/LineStringBuilderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.builders; diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/MultiLineStringBuilderTests.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/MultiLineStringBuilderTests.java index a222cfe9e6b41..0bbe40655a612 100644 --- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/MultiLineStringBuilderTests.java +++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/MultiLineStringBuilderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.builders; diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/MultiPointBuilderTests.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/MultiPointBuilderTests.java index d6484c42c6256..b6255638b0443 100644 --- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/MultiPointBuilderTests.java +++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/MultiPointBuilderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.builders; diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/MultiPolygonBuilderTests.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/MultiPolygonBuilderTests.java index b6ec2313dfc1e..a01484a91fa59 100644 --- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/MultiPolygonBuilderTests.java +++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/MultiPolygonBuilderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.builders; diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/PointBuilderTests.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/PointBuilderTests.java index 132927831fb40..06bea6dd43ab7 100644 --- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/PointBuilderTests.java +++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/PointBuilderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.builders; diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/PolygonBuilderTests.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/PolygonBuilderTests.java index ba82ad37425be..463f795c13800 100644 --- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/PolygonBuilderTests.java +++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/builders/PolygonBuilderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.builders; diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java index 0a0bb12bedbae..c97b0a28d22de 100644 --- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java +++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.mapper; diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldTypeTests.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldTypeTests.java index dc74b9cd295ce..f5e09f19c1a71 100644 --- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldTypeTests.java +++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldTypeTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.mapper; diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeQueryTests.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeQueryTests.java index 4e06a37ec7f20..d3f26bca72e96 100644 --- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeQueryTests.java +++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeQueryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.search; diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/search/LegacyGeoUtilsTests.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/search/LegacyGeoUtilsTests.java index 59178d821825d..fc0cadd30acab 100644 --- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/search/LegacyGeoUtilsTests.java +++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/search/LegacyGeoUtilsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.search; diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/test/ElasticsearchGeoAssertions.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/test/ElasticsearchGeoAssertions.java index a7f9b5401088b..edf378d73111b 100644 --- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/test/ElasticsearchGeoAssertions.java +++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/test/ElasticsearchGeoAssertions.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.test; diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/test/RandomGeoGenerator.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/test/RandomGeoGenerator.java index d70c07f0e9eac..63e5cd27c13b6 100644 --- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/test/RandomGeoGenerator.java +++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/test/RandomGeoGenerator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.test; diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/test/RandomShapeGenerator.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/test/RandomShapeGenerator.java index 37d1a5a0eee27..954f698e6f66a 100644 --- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/test/RandomShapeGenerator.java +++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/test/RandomShapeGenerator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.test; diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/test/TestLegacyGeoShapeFieldMapperPlugin.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/test/TestLegacyGeoShapeFieldMapperPlugin.java index ea10c570f41ac..674a4d0936d03 100644 --- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/test/TestLegacyGeoShapeFieldMapperPlugin.java +++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/test/TestLegacyGeoShapeFieldMapperPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.legacygeo.test; diff --git a/modules/mapper-extras/build.gradle b/modules/mapper-extras/build.gradle index 62e7744bfa7ff..450432e4a1b67 100644 --- a/modules/mapper-extras/build.gradle +++ b/modules/mapper-extras/build.gradle @@ -1,20 +1,21 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.internal.info.BuildParams -apply plugin: 'elasticsearch.legacy-yaml-rest-test' -apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { - description 'Adds advanced field mappers' - classname 'org.elasticsearch.index.mapper.extras.MapperExtrasPlugin' + description = 'Adds advanced field mappers' + classname ='org.elasticsearch.index.mapper.extras.MapperExtrasPlugin' } restResources { @@ -22,9 +23,3 @@ restResources { include '_common', 'cluster', 'field_caps', 'nodes', 'indices', 'index', 'search', 'get' } } - -if (BuildParams.isSnapshotBuild() == false) { - tasks.named("test").configure { - systemProperty 'es.index_mode_feature_flag_registered', 'true' - } -} diff --git a/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/MatchOnlyTextMapperIT.java b/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/MatchOnlyTextMapperIT.java index c387ff2b2134b..18f8b5ca30bf8 100644 --- a/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/MatchOnlyTextMapperIT.java +++ b/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/MatchOnlyTextMapperIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper; @@ -11,6 +12,7 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; @@ -88,13 +90,14 @@ public void testHighlightingWithMatchOnlyTextFieldSyntheticSource() throws IOExc // load the source. String mappings = """ - { "_source" : { "mode" : "synthetic" }, + { "properties" : { "message" : { "type" : "match_only_text" } } } """; - assertAcked(prepareCreate("test").setMapping(mappings)); + Settings.Builder settings = Settings.builder().put(indexSettings()).put("index.mapping.source.mode", "synthetic"); + assertAcked(prepareCreate("test").setSettings(settings).setMapping(mappings)); BulkRequestBuilder bulk = client().prepareBulk("test").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); for (int i = 0; i < 2000; i++) { bulk.add( diff --git a/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/RankFeaturesMapperIntegrationIT.java b/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/RankFeaturesMapperIntegrationIT.java index c6544bac2b13c..19173c650c24a 100644 --- a/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/RankFeaturesMapperIntegrationIT.java +++ b/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/RankFeaturesMapperIntegrationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper; diff --git a/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java b/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java index 08a3d046b00f7..4fc4fc69e0ee8 100644 --- a/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java +++ b/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper; diff --git a/modules/mapper-extras/src/main/java/module-info.java b/modules/mapper-extras/src/main/java/module-info.java index aeec296b2de70..f892248133791 100644 --- a/modules/mapper-extras/src/main/java/module-info.java +++ b/modules/mapper-extras/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.mapper.extras { diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MapperExtrasPlugin.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MapperExtrasPlugin.java index b159177d2c8b0..847494e66e8dc 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MapperExtrasPlugin.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MapperExtrasPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index a616fe9c20c26..5904169308fab 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; @@ -20,6 +21,7 @@ import org.apache.lucene.queries.intervals.IntervalsSource; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.FuzzyQuery; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.PrefixQuery; @@ -269,7 +271,11 @@ public IntervalsSource termIntervals(BytesRef term, SearchExecutionContext conte @Override public IntervalsSource prefixIntervals(BytesRef term, SearchExecutionContext context) { - return toIntervalsSource(Intervals.prefix(term), new PrefixQuery(new Term(name(), term)), context); + return toIntervalsSource( + Intervals.prefix(term, IndexSearcher.getMaxClauseCount()), + new PrefixQuery(new Term(name(), term)), + context + ); } @Override @@ -284,23 +290,47 @@ public IntervalsSource fuzzyIntervals( new Term(name(), term), maxDistance, prefixLength, - 128, + IndexSearcher.getMaxClauseCount(), transpositions, MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE ); - IntervalsSource fuzzyIntervals = Intervals.multiterm(fuzzyQuery.getAutomata(), term); + IntervalsSource fuzzyIntervals = Intervals.multiterm(fuzzyQuery.getAutomata(), IndexSearcher.getMaxClauseCount(), term); return toIntervalsSource(fuzzyIntervals, fuzzyQuery, context); } @Override public IntervalsSource wildcardIntervals(BytesRef pattern, SearchExecutionContext context) { return toIntervalsSource( - Intervals.wildcard(pattern), + Intervals.wildcard(pattern, IndexSearcher.getMaxClauseCount()), new MatchAllDocsQuery(), // wildcard queries can be expensive, what should the approximation be? context ); } + @Override + public IntervalsSource regexpIntervals(BytesRef pattern, SearchExecutionContext context) { + return toIntervalsSource( + Intervals.regexp(pattern, IndexSearcher.getMaxClauseCount()), + new MatchAllDocsQuery(), // regexp queries can be expensive, what should the approximation be? + context + ); + } + + @Override + public IntervalsSource rangeIntervals( + BytesRef lowerTerm, + BytesRef upperTerm, + boolean includeLower, + boolean includeUpper, + SearchExecutionContext context + ) { + return toIntervalsSource( + Intervals.range(lowerTerm, upperTerm, includeLower, includeUpper, IndexSearcher.getMaxClauseCount()), + new MatchAllDocsQuery(), // range queries can be expensive, what should the approximation be? + context + ); + } + @Override public Query phraseQuery(TokenStream stream, int slop, boolean enablePosIncrements, SearchExecutionContext queryShardContext) throws IOException { @@ -433,7 +463,7 @@ public MatchOnlyTextFieldType fieldType() { @Override protected SyntheticSourceSupport syntheticSourceSupport() { - var loader = new StringStoredFieldFieldLoader(fieldType().storedFieldNameForSyntheticSource(), leafName()) { + var loader = new StringStoredFieldFieldLoader(fieldType().storedFieldNameForSyntheticSource(), fieldType().name(), leafName()) { @Override protected void write(XContentBuilder b, Object value) throws IOException { b.value((String) value); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java index 0b475641e4290..0599ea38f1f46 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureMetaFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureMetaFieldMapper.java index 366145be02d82..ed1cc57b84863 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureMetaFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureMetaFieldMapper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; @@ -47,7 +48,7 @@ public String typeName() { @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - throw new UnsupportedOperationException("Cannot fetch values for internal field [" + typeName() + "]."); + throw new IllegalArgumentException("Cannot fetch values for internal field [" + typeName() + "]."); } @Override diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureQueryBuilder.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureQueryBuilder.java index 0e9b293fe9fd0..61b8322e4732d 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureQueryBuilder.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureQueryBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java index 5b1d35ec03c0e..f51bbe33c96b7 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index ac236d13cc586..b845545133e19 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java index 57ac8fdfbb023..bce6ffb5e0ea3 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SourceConfirmedTextQuery.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SourceConfirmedTextQuery.java index 3d0f26e8cc130..d16034c5de2fd 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SourceConfirmedTextQuery.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SourceConfirmedTextQuery.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SourceIntervalsSource.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SourceIntervalsSource.java index 0b757cc9ab898..1f10459e042d7 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SourceIntervalsSource.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SourceIntervalsSource.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java index fa0a96a548a97..fd0098851c5f8 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/BWCTemplateTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/BWCTemplateTests.java index b4ee066a0e391..fe39eaec37df4 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/BWCTemplateTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/BWCTemplateTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/FieldCapsRankFeatureTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/FieldCapsRankFeatureTests.java index b6ac1a36e9223..3289bec339126 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/FieldCapsRankFeatureTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/FieldCapsRankFeatureTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java index e1fbc2e149441..7ebf4d7e76e88 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; @@ -63,19 +64,19 @@ protected Object getSampleValueForDocument() { } public void testExistsStandardSource() throws IOException { - assertExistsQuery(createMapperService(testMapping(false))); + assertExistsQuery(createMapperService(fieldMapping(b -> b.field("type", "match_only_text")))); } public void testExistsSyntheticSource() throws IOException { - assertExistsQuery(createMapperService(testMapping(true))); + assertExistsQuery(createSytheticSourceMapperService(fieldMapping(b -> b.field("type", "match_only_text")))); } public void testPhraseQueryStandardSource() throws IOException { - assertPhraseQuery(createMapperService(testMapping(false))); + assertPhraseQuery(createMapperService(fieldMapping(b -> b.field("type", "match_only_text")))); } public void testPhraseQuerySyntheticSource() throws IOException { - assertPhraseQuery(createMapperService(testMapping(true))); + assertPhraseQuery(createSytheticSourceMapperService(fieldMapping(b -> b.field("type", "match_only_text")))); } private void assertPhraseQuery(MapperService mapperService) throws IOException { @@ -103,13 +104,6 @@ protected void registerParameters(ParameterChecker checker) throws IOException { ); } - private static XContentBuilder testMapping(boolean syntheticSource) throws IOException { - if (syntheticSource) { - return syntheticSourceMapping(b -> b.startObject("field").field("type", "match_only_text").endObject()); - } - return fieldMapping(b -> b.field("type", "match_only_text")); - } - @Override protected void minimalMapping(XContentBuilder b) throws IOException { b.field("type", "match_only_text"); @@ -255,7 +249,7 @@ public void testDocValues() throws IOException { } public void testDocValuesLoadedFromSynthetic() throws IOException { - MapperService mapper = createMapperService(syntheticSourceFieldMapping(b -> b.field("type", "match_only_text"))); + MapperService mapper = createSytheticSourceMapperService(fieldMapping(b -> b.field("type", "match_only_text"))); assertScriptDocValues(mapper, "foo", equalTo(List.of("foo"))); } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java index 082ae9624779a..6970dd6739ecf 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; @@ -13,6 +14,7 @@ import org.apache.lucene.queries.intervals.IntervalsSource; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.FuzzyQuery; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MultiPhraseQuery; import org.apache.lucene.search.PhraseQuery; @@ -151,30 +153,56 @@ public void testPhrasePrefixQuery() throws IOException { assertNotEquals(new MatchAllDocsQuery(), SourceConfirmedTextQuery.approximate(delegate)); } - public void testTermIntervals() throws IOException { + public void testTermIntervals() { MappedFieldType ft = new MatchOnlyTextFieldType("field"); IntervalsSource termIntervals = ft.termIntervals(new BytesRef("foo"), MOCK_CONTEXT); assertThat(termIntervals, Matchers.instanceOf(SourceIntervalsSource.class)); assertEquals(Intervals.term(new BytesRef("foo")), ((SourceIntervalsSource) termIntervals).getIntervalsSource()); } - public void testPrefixIntervals() throws IOException { + public void testPrefixIntervals() { MappedFieldType ft = new MatchOnlyTextFieldType("field"); IntervalsSource prefixIntervals = ft.prefixIntervals(new BytesRef("foo"), MOCK_CONTEXT); assertThat(prefixIntervals, Matchers.instanceOf(SourceIntervalsSource.class)); - assertEquals(Intervals.prefix(new BytesRef("foo")), ((SourceIntervalsSource) prefixIntervals).getIntervalsSource()); + assertEquals( + Intervals.prefix(new BytesRef("foo"), IndexSearcher.getMaxClauseCount()), + ((SourceIntervalsSource) prefixIntervals).getIntervalsSource() + ); } - public void testWildcardIntervals() throws IOException { + public void testWildcardIntervals() { MappedFieldType ft = new MatchOnlyTextFieldType("field"); IntervalsSource wildcardIntervals = ft.wildcardIntervals(new BytesRef("foo"), MOCK_CONTEXT); assertThat(wildcardIntervals, Matchers.instanceOf(SourceIntervalsSource.class)); - assertEquals(Intervals.wildcard(new BytesRef("foo")), ((SourceIntervalsSource) wildcardIntervals).getIntervalsSource()); + assertEquals( + Intervals.wildcard(new BytesRef("foo"), IndexSearcher.getMaxClauseCount()), + ((SourceIntervalsSource) wildcardIntervals).getIntervalsSource() + ); + } + + public void testRegexpIntervals() { + MappedFieldType ft = new MatchOnlyTextFieldType("field"); + IntervalsSource regexpIntervals = ft.regexpIntervals(new BytesRef("foo"), MOCK_CONTEXT); + assertThat(regexpIntervals, Matchers.instanceOf(SourceIntervalsSource.class)); + assertEquals( + Intervals.regexp(new BytesRef("foo"), IndexSearcher.getMaxClauseCount()), + ((SourceIntervalsSource) regexpIntervals).getIntervalsSource() + ); } - public void testFuzzyIntervals() throws IOException { + public void testFuzzyIntervals() { MappedFieldType ft = new MatchOnlyTextFieldType("field"); IntervalsSource fuzzyIntervals = ft.fuzzyIntervals("foo", 1, 2, true, MOCK_CONTEXT); assertThat(fuzzyIntervals, Matchers.instanceOf(SourceIntervalsSource.class)); } + + public void testRangeIntervals() { + MappedFieldType ft = new MatchOnlyTextFieldType("field"); + IntervalsSource rangeIntervals = ft.rangeIntervals(new BytesRef("foo"), new BytesRef("foo1"), true, true, MOCK_CONTEXT); + assertThat(rangeIntervals, Matchers.instanceOf(SourceIntervalsSource.class)); + assertEquals( + Intervals.range(new BytesRef("foo"), new BytesRef("foo1"), true, true, IndexSearcher.getMaxClauseCount()), + ((SourceIntervalsSource) rangeIntervals).getIntervalsSource() + ); + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapperTests.java index 7a8931cf6b96c..917941b39e888 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapperTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldTypeTests.java index 1c45f4a422900..7ab4beae6f5f3 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldTypeTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureMetaFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureMetaFieldMapperTests.java index 9f559c8f55858..919f9214694b5 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureMetaFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureMetaFieldMapperTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureQueryBuilderTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureQueryBuilderTests.java index 32446c3c27391..a0b8c06f288a5 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureQueryBuilderTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureQueryBuilderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapperTests.java index 544200dd3cb60..6818b7936d124 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapperTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldTypeTests.java index 48a31691ac1bc..98c88d97c9119 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldTypeTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java index 765e72091a1ba..83fe07170d6e7 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; @@ -526,7 +527,13 @@ protected Number randomNumber() { public void testEncodeDecodeExactScalingFactor() { double v = randomValue(); - assertThat(encodeDecode(1 / v, v), equalTo(1 / v)); + double expected = 1 / v; + // We don't produce infinities while decoding. See #testDecodeHandlingInfinity(). + if (Double.isInfinite(expected)) { + var sign = expected == Double.POSITIVE_INFINITY ? 1 : -1; + expected = sign * Double.MAX_VALUE; + } + assertThat(encodeDecode(1 / v, v), equalTo(expected)); } /** diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java index 222f0f05d548d..0c69ec6258589 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeAnalyzerTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeAnalyzerTests.java index 1c1da8fc5f178..bc871bce6e9ac 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeAnalyzerTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeAnalyzerTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapperTests.java index a527d341871db..3e2bb422c2062 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapperTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldTypeTests.java index 6485defe9f4c7..ad28b5bbdfb99 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldTypeTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SourceConfirmedTextQueryTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SourceConfirmedTextQueryTests.java index 81e1dd7099860..84139409e8bc6 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SourceConfirmedTextQueryTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SourceConfirmedTextQueryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SourceIntervalsSourceTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SourceIntervalsSourceTests.java index ba14c891c471c..0fef801b22009 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SourceIntervalsSourceTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SourceIntervalsSourceTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java index d34d9c3178c78..1a4619e00f826 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.extras; @@ -219,7 +220,7 @@ public SyntheticSourceExample example(int maxValues) { var textArray = values.stream().map(Value::text).toList(); - var blockExpectedList = values.stream().map(Value::tokenCount).filter(Objects::nonNull).toList(); + var blockExpectedList = values.stream().map(Value::tokenCount).filter(Objects::nonNull).sorted().toList(); var blockExpected = blockExpectedList.size() == 1 ? blockExpectedList.get(0) : blockExpectedList; return new SyntheticSourceExample(textArray, textArray, blockExpected, this::mapping); @@ -229,7 +230,7 @@ private record Value(String text, Integer tokenCount) {} private Value generateValue() { if (rarely()) { - return new Value(null, null); + return new Value(null, nullValue); } var text = randomList(0, 10, () -> randomAlphaOfLengthBetween(0, 10)).stream().collect(Collectors.joining(" ")); diff --git a/modules/mapper-extras/src/yamlRestTest/java/org/elasticsearch/index/mapper/MapperExtrasClientYamlTestSuiteIT.java b/modules/mapper-extras/src/yamlRestTest/java/org/elasticsearch/index/mapper/MapperExtrasClientYamlTestSuiteIT.java index 44a01edaa3ff2..80953af5a4cbb 100644 --- a/modules/mapper-extras/src/yamlRestTest/java/org/elasticsearch/index/mapper/MapperExtrasClientYamlTestSuiteIT.java +++ b/modules/mapper-extras/src/yamlRestTest/java/org/elasticsearch/index/mapper/MapperExtrasClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper; @@ -11,8 +12,10 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; /** Runs yaml rest tests */ public class MapperExtrasClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { @@ -25,4 +28,12 @@ public MapperExtrasClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate t public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("mapper-extras").build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/match_only_text/10_basic.yml b/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/match_only_text/10_basic.yml index b4ee226f72692..821ab46b1bd64 100644 --- a/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/match_only_text/10_basic.yml +++ b/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/match_only_text/10_basic.yml @@ -278,16 +278,17 @@ setup: --- synthetic_source: - requires: - cluster_features: ["gte_v8.4.0"] - reason: synthetic source introduced in 8.4.0 + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" - do: indices.create: index: synthetic_source_test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: foo: type: match_only_text @@ -355,16 +356,17 @@ tsdb: --- synthetic_source with copy_to: - requires: - cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"] - reason: requires copy_to support in synthetic source + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" - do: indices.create: index: synthetic_source_test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: foo: type: match_only_text diff --git a/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/rank_feature/30_synthetic_source.yml b/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/rank_feature/30_synthetic_source.yml index 1e0b90ebb9e0f..ccf3c689bdfd6 100644 --- a/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/rank_feature/30_synthetic_source.yml +++ b/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/rank_feature/30_synthetic_source.yml @@ -1,15 +1,16 @@ setup: - requires: - cluster_features: ["mapper.source.synthetic_source_fallback"] - reason: introduced in 8.15.0 + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" - do: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: pagerank: type: rank_feature diff --git a/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/rank_features/20_synthetic_source.yml b/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/rank_features/20_synthetic_source.yml index c64e35cc2cea4..870e673044c0b 100644 --- a/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/rank_features/20_synthetic_source.yml +++ b/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/rank_features/20_synthetic_source.yml @@ -1,15 +1,16 @@ setup: - requires: - cluster_features: ["mapper.source.synthetic_source_fallback"] - reason: introduced in 8.15.0 + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" - do: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: tags: type: rank_features diff --git a/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/search-as-you-type/30_synthetic_source.yml b/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/search-as-you-type/30_synthetic_source.yml index 75397bd9e0fe9..0ca2306256064 100644 --- a/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/search-as-you-type/30_synthetic_source.yml +++ b/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/search-as-you-type/30_synthetic_source.yml @@ -1,15 +1,16 @@ setup: - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" - do: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: a_field: type: search_as_you_type diff --git a/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/token_count/10_basic.yml b/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/token_count/10_basic.yml index 03b72a2623497..d92b807ebd6ce 100644 --- a/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/token_count/10_basic.yml +++ b/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/token_count/10_basic.yml @@ -36,15 +36,17 @@ --- "Synthetic source": - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" + - do: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: count: type: token_count diff --git a/modules/parent-join/build.gradle b/modules/parent-join/build.gradle index 844478c83e7c7..549f700437cec 100644 --- a/modules/parent-join/build.gradle +++ b/modules/parent-join/build.gradle @@ -1,17 +1,18 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ -apply plugin: 'elasticsearch.legacy-yaml-rest-test' -apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { - description 'This module adds the support parent-child queries and aggregations' - classname 'org.elasticsearch.join.ParentJoinPlugin' + description = 'This module adds the support parent-child queries and aggregations' + classname = 'org.elasticsearch.join.ParentJoinPlugin' } restResources { diff --git a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/AbstractParentChildTestCase.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/AbstractParentChildTestCase.java index 03c64174af2c5..9f276a3643a3c 100644 --- a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/AbstractParentChildTestCase.java +++ b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/AbstractParentChildTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.aggregations; diff --git a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/ChildrenIT.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/ChildrenIT.java index 595d845d40b3d..ad8e252e3fd63 100644 --- a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/ChildrenIT.java +++ b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/ChildrenIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.aggregations; diff --git a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/ParentIT.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/ParentIT.java index 65c162e0b78bc..4c59d5010c396 100644 --- a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/ParentIT.java +++ b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/aggregations/ParentIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.aggregations; diff --git a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ChildQuerySearchIT.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ChildQuerySearchIT.java index dc0f3ea8bb8c6..872165014f5a4 100644 --- a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ChildQuerySearchIT.java +++ b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ChildQuerySearchIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.query; diff --git a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/InnerHitsIT.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/InnerHitsIT.java index f851678b6c9d6..0ae10b297f709 100644 --- a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/InnerHitsIT.java +++ b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/InnerHitsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.query; diff --git a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ParentChildTestCase.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ParentChildTestCase.java index 02f24a67dda02..b91b52ee89b41 100644 --- a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ParentChildTestCase.java +++ b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ParentChildTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.query; diff --git a/modules/parent-join/src/main/java/module-info.java b/modules/parent-join/src/main/java/module-info.java index b3ac78e1fe2b6..ecc0782383ddb 100644 --- a/modules/parent-join/src/main/java/module-info.java +++ b/modules/parent-join/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.join { diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/ParentJoinPlugin.java b/modules/parent-join/src/main/java/org/elasticsearch/join/ParentJoinPlugin.java index 675771c5f7541..f40dc6e05bf09 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/ParentJoinPlugin.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/ParentJoinPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join; diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/Children.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/Children.java index c994f80e9d333..9fafb79724042 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/Children.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/Children.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.aggregations; diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregationBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregationBuilder.java index 0de7b74759828..71945ad395157 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregationBuilder.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregationBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.aggregations; diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregatorFactory.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregatorFactory.java index 05316353387f5..4744fd67a6bbd 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregatorFactory.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregatorFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.aggregations; diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregator.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregator.java index 3f32807a4da66..12489ad37aabd 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregator.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregator.java @@ -1,13 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.aggregations; import org.apache.lucene.search.Query; +import org.elasticsearch.common.util.LongArray; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.CardinalityUpperBound; @@ -43,7 +45,7 @@ public ChildrenToParentAggregator( } @Override - public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws IOException { + public InternalAggregation[] buildAggregations(LongArray owningBucketOrds) throws IOException { return buildAggregationsForSingleBucket( owningBucketOrds, (owningBucketOrd, subAggregationResults) -> new InternalParent( diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/InternalChildren.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/InternalChildren.java index a29469dc49caf..f67596d1cf0ce 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/InternalChildren.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/InternalChildren.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.aggregations; diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/InternalParent.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/InternalParent.java index 18954f9699cbb..4a74b502e1b1d 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/InternalParent.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/InternalParent.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.aggregations; diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/JoinAggregationBuilders.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/JoinAggregationBuilders.java index 1009e6d7454d5..7d98568d116ba 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/JoinAggregationBuilders.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/JoinAggregationBuilders.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.aggregations; diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/JoinAggregationInspectionHelper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/JoinAggregationInspectionHelper.java index b4a452f737b9f..54738e99b6f3f 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/JoinAggregationInspectionHelper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/JoinAggregationInspectionHelper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.aggregations; diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/Parent.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/Parent.java index d566c6422e934..378c79e558f4c 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/Parent.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/Parent.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.aggregations; diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregationBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregationBuilder.java index c7999f27834a9..881b22c8dfa22 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregationBuilder.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregationBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.aggregations; diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregatorFactory.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregatorFactory.java index 89c5b40fee9a7..7faeea25a0bf9 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregatorFactory.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregatorFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.aggregations; diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentJoinAggregator.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentJoinAggregator.java index 9c6a788ea2f77..122286533ec02 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentJoinAggregator.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentJoinAggregator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.aggregations; @@ -20,6 +21,7 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.BitArray; +import org.elasticsearch.common.util.LongArray; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.search.aggregations.AggregationExecutionContext; @@ -114,7 +116,7 @@ public void postCollection() throws IOException { } @Override - protected void prepareSubAggs(long[] ordsToCollect) throws IOException { + protected void prepareSubAggs(LongArray ordsToCollect) throws IOException { IndexReader indexReader = searcher().getIndexReader(); for (LeafReaderContext ctx : indexReader.leaves()) { Scorer childDocsScorer = outFilter.scorer(ctx); @@ -157,9 +159,10 @@ public int docID() { * structure that maps a primitive long to a list of primitive * longs. */ - for (long owningBucketOrd : ordsToCollect) { - if (collectionStrategy.exists(owningBucketOrd, globalOrdinal)) { - collectBucket(sub, docId, owningBucketOrd); + for (long ord = 0; ord < ordsToCollect.size(); ord++) { + long ordToCollect = ordsToCollect.get(ord); + if (collectionStrategy.exists(ordToCollect, globalOrdinal)) { + collectBucket(sub, docId, ordToCollect); } } } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregator.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregator.java index 7496b19a5f053..939107f87715d 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregator.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregator.java @@ -1,13 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.aggregations; import org.apache.lucene.search.Query; +import org.elasticsearch.common.util.LongArray; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.CardinalityUpperBound; @@ -39,7 +41,7 @@ public ParentToChildrenAggregator( } @Override - public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws IOException { + public InternalAggregation[] buildAggregations(LongArray owningBucketOrds) throws IOException { return buildAggregationsForSingleBucket( owningBucketOrds, (owningBucketOrd, subAggregationResults) -> new InternalChildren( diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/Joiner.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/Joiner.java index 9a4911fb8c4d0..49712d4e731ea 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/Joiner.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/Joiner.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.mapper; diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java index f6392f32a88d6..46cec9a0b7a5f 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.mapper; diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index ccb67f5c51acf..fd503bf75178c 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.mapper; diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/Relations.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/Relations.java index c9189433baf07..b3d97010e3ec5 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/Relations.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/Relations.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.mapper; diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java index 0892690b79c0a..e2c93f4cad37a 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.query; diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasParentQueryBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasParentQueryBuilder.java index 962916cc04645..fc15bb55f8703 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasParentQueryBuilder.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasParentQueryBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.query; diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/JoinQueryBuilders.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/JoinQueryBuilders.java index ddfd402080703..f0e6a6de11ed7 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/query/JoinQueryBuilders.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/JoinQueryBuilders.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.query; diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java index 95db4d92874a4..9ecf4ed821e2a 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.query; diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentIdQueryBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentIdQueryBuilder.java index 89850862cd63f..8ed22edec42e2 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentIdQueryBuilder.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentIdQueryBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.query; diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenTests.java index 9d12d1bc3d72f..70f0b1dc7cc2c 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.aggregations; diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java index 5a5ccb640f03d..03a1677e60f47 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.aggregations; diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/InternalChildrenTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/InternalChildrenTests.java index 3cb674ae6ddee..1bbf0df86cca0 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/InternalChildrenTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/InternalChildrenTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.aggregations; diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/InternalParentTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/InternalParentTests.java index 31ae09ab5a14b..c3c6dbbcf86ae 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/InternalParentTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/InternalParentTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.aggregations; diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentTests.java index 75fb02eab2dbb..6abdad53ad829 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.aggregations; diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregatorTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregatorTests.java index 0bb64dcccbf11..91ec0e3c67691 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregatorTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregatorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.aggregations; diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/JoinFieldTypeTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/JoinFieldTypeTests.java index b98c848b47380..aa58270f0320f 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/JoinFieldTypeTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/JoinFieldTypeTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.mapper; diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentIdFieldTypeTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentIdFieldTypeTests.java index 8771aecca0909..0eb41c850536d 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentIdFieldTypeTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentIdFieldTypeTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.mapper; diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java index ccbb011d5c755..1d609d302117d 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.mapper; diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java index ca6e5a8078f6f..d4fe49ec8c773 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.query; diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasParentQueryBuilderTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasParentQueryBuilderTests.java index c442e3d2cb0ea..5aa8281a3d3c6 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasParentQueryBuilderTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasParentQueryBuilderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.query; diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentIdQueryBuilderTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentIdQueryBuilderTests.java index 641213143d6c4..f9aadb3e5712f 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentIdQueryBuilderTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentIdQueryBuilderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join.query; diff --git a/modules/parent-join/src/yamlRestTest/java/org/elasticsearch/join/ParentChildClientYamlTestSuiteIT.java b/modules/parent-join/src/yamlRestTest/java/org/elasticsearch/join/ParentChildClientYamlTestSuiteIT.java index de88ba4ab3b00..ecfc464f0739c 100644 --- a/modules/parent-join/src/yamlRestTest/java/org/elasticsearch/join/ParentChildClientYamlTestSuiteIT.java +++ b/modules/parent-join/src/yamlRestTest/java/org/elasticsearch/join/ParentChildClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.join; @@ -11,8 +12,10 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; public class ParentChildClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { public ParentChildClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { @@ -23,4 +26,12 @@ public ParentChildClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate te public static Iterable parameters() throws Exception { return createParameters(); } + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("parent-join").build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/modules/parent-join/src/yamlRestTest/resources/rest-api-spec/test/60_synthetic_source.yml b/modules/parent-join/src/yamlRestTest/resources/rest-api-spec/test/60_synthetic_source.yml index 12d0f1bbae6c7..6a8b6cfbd1a2e 100644 --- a/modules/parent-join/src/yamlRestTest/resources/rest-api-spec/test/60_synthetic_source.yml +++ b/modules/parent-join/src/yamlRestTest/resources/rest-api-spec/test/60_synthetic_source.yml @@ -1,15 +1,16 @@ supported: - requires: - cluster_features: ["mapper.source.synthetic_source_fallback"] - reason: introduced in 8.15.0 + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" - do: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: join_field: type: join diff --git a/modules/percolator/build.gradle b/modules/percolator/build.gradle index b9b257a42e051..5fbe90648f686 100644 --- a/modules/percolator/build.gradle +++ b/modules/percolator/build.gradle @@ -1,17 +1,18 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ -apply plugin: 'elasticsearch.legacy-yaml-rest-test' -apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { - description 'Percolator module adds capability to index queries and query these queries by specifying documents' - classname 'org.elasticsearch.percolator.PercolatorPlugin' + description = 'Percolator module adds capability to index queries and query these queries by specifying documents' + classname = 'org.elasticsearch.percolator.PercolatorPlugin' } dependencies { diff --git a/modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java index 0fe7de9bbb23c..8a7f1405f8f4e 100644 --- a/modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java +++ b/modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.percolator; @@ -1358,7 +1359,7 @@ public void testKnnQueryNotSupportedInPercolator() throws IOException { """); indicesAdmin().prepareCreate("index1").setMapping(mappings).get(); ensureGreen(); - QueryBuilder knnVectorQueryBuilder = new KnnVectorQueryBuilder("my_vector", new float[] { 1, 1, 1, 1, 1 }, 10, 10, null); + QueryBuilder knnVectorQueryBuilder = new KnnVectorQueryBuilder("my_vector", new float[] { 1, 1, 1, 1, 1 }, 10, 10, null, null); IndexRequestBuilder indexRequestBuilder = prepareIndex("index1").setId("knn_query1") .setSource(jsonBuilder().startObject().field("my_query", knnVectorQueryBuilder).endObject()); diff --git a/modules/percolator/src/main/java/module-info.java b/modules/percolator/src/main/java/module-info.java index c6dce13d1a6dd..51848d92a5e6c 100644 --- a/modules/percolator/src/main/java/module-info.java +++ b/modules/percolator/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.percolator { diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/BinaryRange.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/BinaryRange.java index a2eb600e2f344..a70a8d50c3158 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/BinaryRange.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/BinaryRange.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.percolator; diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java index 408b3f204de1a..255131b51a57a 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.percolator; diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index 27d8695f9ae6a..0ee93474234ec 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.percolator; diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index 576ea4dbd5d23..409b6fd70c3c7 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.percolator; diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java index da8d7de27d317..3efd968c598d1 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.percolator; diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java index fe4bfc7741c87..c363746856681 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.percolator; diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java index a407414d5c8c6..d2ab2d16a75b6 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.percolator; diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java index 84fb889c6dd5a..da4b10956dcf8 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.percolator; diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java index 05a935229246d..31e893ace72fd 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.percolator; diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java index ed9ae582f42eb..d6908f58a901a 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.percolator; diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java index 8ae24dfc47475..075d4d429fb39 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.percolator; diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateWithNestedQueryBuilderTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateWithNestedQueryBuilderTests.java index cbfab4a61aeb0..6e2127349ffd2 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateWithNestedQueryBuilderTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateWithNestedQueryBuilderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.percolator; diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java index 4adc7f9b5ba27..100cda66acdcc 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.percolator; diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java index fd5f07a718176..944d36afeb4b7 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.percolator; diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java index 1ebf0b4a28ed6..140f1f7012709 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.percolator; diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java index a4c3ce6e7a3f7..d9ea30dbd1b68 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.percolator; diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java index b3b47d909b046..a9c3e09e7f4ed 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.percolator; diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java index 917c1699cfdb9..67527a4d27436 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.percolator; diff --git a/modules/percolator/src/yamlRestTest/java/org/elasticsearch/percolator/PercolatorClientYamlTestSuiteIT.java b/modules/percolator/src/yamlRestTest/java/org/elasticsearch/percolator/PercolatorClientYamlTestSuiteIT.java index 61fbc81519cef..d71e758525085 100644 --- a/modules/percolator/src/yamlRestTest/java/org/elasticsearch/percolator/PercolatorClientYamlTestSuiteIT.java +++ b/modules/percolator/src/yamlRestTest/java/org/elasticsearch/percolator/PercolatorClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.percolator; @@ -11,8 +12,10 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; public class PercolatorClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { public PercolatorClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { @@ -23,4 +26,12 @@ public PercolatorClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate tes public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("percolator").build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/modules/percolator/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml b/modules/percolator/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml index a5576d203314f..53fca8b664e59 100644 --- a/modules/percolator/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml +++ b/modules/percolator/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml @@ -130,16 +130,17 @@ --- "Synthetic source": - requires: - cluster_features: ["mapper.source.synthetic_source_fallback"] - reason: introduced in 8.15.0 + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" - do: indices.create: index: queries_index body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: query: type: percolator diff --git a/modules/rank-eval/build.gradle b/modules/rank-eval/build.gradle index 1268a40dd5bd3..eee7b1b16c1db 100644 --- a/modules/rank-eval/build.gradle +++ b/modules/rank-eval/build.gradle @@ -1,17 +1,18 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.legacy-yaml-rest-test' apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { - description 'The Rank Eval module adds APIs to evaluate ranking quality.' - classname 'org.elasticsearch.index.rankeval.RankEvalPlugin' + description = 'The Rank Eval module adds APIs to evaluate ranking quality.' + classname = 'org.elasticsearch.index.rankeval.RankEvalPlugin' } restResources { @@ -24,3 +25,7 @@ testClusters.configureEach { // Modules who's integration is explicitly tested in integration tests module ':modules:lang-mustache' } + +tasks.named("yamlRestTestV7CompatTransform").configure({ task -> + task.skipTest("rank_eval/30_failures/Response format", "warning does not exist for compatibility") +}) diff --git a/modules/rank-eval/src/internalClusterTest/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java b/modules/rank-eval/src/internalClusterTest/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java index 244c3f2188237..e2b7588148472 100644 --- a/modules/rank-eval/src/internalClusterTest/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java +++ b/modules/rank-eval/src/internalClusterTest/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/main/java/module-info.java b/modules/rank-eval/src/main/java/module-info.java index e18e229ea548d..d559ad0433482 100644 --- a/modules/rank-eval/src/main/java/module-info.java +++ b/modules/rank-eval/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.rankeval { diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java index 6b57acb8589ae..8570a0b9f4ee2 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java index f837cb71d509e..4bf80058950c6 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvaluationMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvaluationMetric.java index 65340d225461e..d2d41cfb23465 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvaluationMetric.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvaluationMetric.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRank.java index 17fedc67ef48e..7a7b06ecc8a17 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRank.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java index 86714257f66d1..7bf3baec615b3 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MetricDetail.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MetricDetail.java index 3c0d9b527fa4d..2253216546940 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MetricDetail.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MetricDetail.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtK.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtK.java index 4034dbda7c7ee..4c1c7928623c5 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtK.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtK.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalNamedXContentProvider.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalNamedXContentProvider.java index 9239793066490..6d38ed1c7be07 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalNamedXContentProvider.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalNamedXContentProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java index d3ed0a66f0abe..58b955eb358a0 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java index ce63bcba0345c..8e0d838e602e7 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequestBuilder.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequestBuilder.java index db80a7b598118..d5f6d2909da00 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequestBuilder.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java index fe6da7fe1ce68..bde75af92b9e7 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java index 757a891f1c357..851a585081d48 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java index cf4ac5ca70149..de3e6c2352cc3 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java index 3b199c343ec6c..10785832c5ca8 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java index a6e31ce7f37cf..1658743537415 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RecallAtK.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RecallAtK.java index d3d0a3110ee58..488a1ceef5cfe 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RecallAtK.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RecallAtK.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java index dc47d409e4da2..cde45f3885df5 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index 258eb4ba3e22a..99fb6c3823442 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java index 232a22239d901..a0b339a4e7b48 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java index 3c42c40302b02..d04f21d8b5648 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java index 9b1319ae35055..7c7a5e4fe531c 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java index 1a42ba9606938..7ae10ea2e9959 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java index 10f22bc5a0b3f..96a5665622fa0 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java index 1429fff975cf8..bc5bafe538a51 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java index ed4d627cd2ed2..3c412394eea2c 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java index b180aefc2dde3..9248c4ff2de27 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java index 068719c49c32e..518cdfbe00399 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java index da8872d9b348f..54186468a3e82 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java index 415d270ef4fdb..158eb59f9d507 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RecallAtKTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RecallAtKTests.java index d7acd4a253c2c..0131ecaa31f07 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RecallAtKTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RecallAtKTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TransportRankEvalActionTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TransportRankEvalActionTests.java index 89a850b43e9cd..faf15c7454ca0 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TransportRankEvalActionTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TransportRankEvalActionTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/yamlRestTest/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java b/modules/rank-eval/src/yamlRestTest/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java index c154c617f59f4..bfaafd898082d 100644 --- a/modules/rank-eval/src/yamlRestTest/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java +++ b/modules/rank-eval/src/yamlRestTest/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.rankeval; diff --git a/modules/rank-eval/src/yamlRestTest/resources/rest-api-spec/test/rank_eval/30_failures.yml b/modules/rank-eval/src/yamlRestTest/resources/rest-api-spec/test/rank_eval/30_failures.yml index c88a769b8687b..2a2b062cb9cdc 100644 --- a/modules/rank-eval/src/yamlRestTest/resources/rest-api-spec/test/rank_eval/30_failures.yml +++ b/modules/rank-eval/src/yamlRestTest/resources/rest-api-spec/test/rank_eval/30_failures.yml @@ -21,7 +21,7 @@ }, { "id" : "invalid_query", - "request": { "query": { "range" : { "bar" : { "from" : "Basel", "time_zone": "+01:00" }}}}, + "request": { "query": { "range" : { "bar" : { "gte" : "Basel", "time_zone": "+01:00" }}}}, "ratings": [{"_index": "foo", "_id": "doc1", "rating": 1}] } ], diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index 9cd7963224cf8..9b221902ef301 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.Architecture @@ -15,14 +16,14 @@ import org.elasticsearch.gradle.transform.UnzipTransform apply plugin: 'elasticsearch.test-with-dependencies' apply plugin: 'elasticsearch.jdk-download' -apply plugin: 'elasticsearch.legacy-yaml-rest-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' apply plugin: 'elasticsearch.legacy-java-rest-test' -apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { - description 'The Reindex module adds APIs to reindex from one index to another or update documents in place.' - classname 'org.elasticsearch.reindex.ReindexPlugin' + description = 'The Reindex module adds APIs to reindex from one index to another or update documents in place.' + classname = 'org.elasticsearch.reindex.ReindexPlugin' } testClusters.configureEach { @@ -32,15 +33,18 @@ testClusters.configureEach { module ':modules:rest-root' // Whitelist reindexing from the local node so we can test reindex-from-remote. setting 'reindex.remote.whitelist', '127.0.0.1:*' - requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0") } dependencies { api project(":client:rest") - api project(":libs:elasticsearch-ssl-config") + api project(":libs:ssl-config") // for parent/child testing testImplementation project(':modules:parent-join') testImplementation project(':modules:rest-root') + + clusterModules project(':modules:lang-painless') + clusterModules project(':modules:parent-join') + clusterModules project(":modules:rest-root") } restResources { @@ -131,8 +135,9 @@ if (OS.current() == OS.WINDOWS) { oldEsDependency.getAttributes().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE); TaskProvider fixture = tasks.register("oldEs${version}Fixture", AntFixture) { dependsOn project.configurations.oldesFixture, jdks.legacy, oldEsDependency - executable = "${BuildParams.runtimeJavaHome}/bin/java" - env 'CLASSPATH', "${-> project.configurations.oldesFixture.asPath}" + executable = "${buildParams.runtimeJavaHome.get()}/bin/java" + def oldesFixtureConfiguration = project.configurations.oldesFixture + env 'CLASSPATH', "${-> oldesFixtureConfiguration.asPath}" // old versions of Elasticsearch need JAVA_HOME env 'JAVA_HOME', jdks.legacy.javaHomePath // If we are running on certain arm systems we need to explicitly set the stack size to overcome JDK page size bug diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java index 071031d2ffd19..19a96bfd6e2fc 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.client.documentation; diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/BulkByScrollUsesAllScrollDocumentsAfterConflictsIntegTests.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/BulkByScrollUsesAllScrollDocumentsAfterConflictsIntegTests.java index 37de70ded462f..04d8bae9fda2f 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/BulkByScrollUsesAllScrollDocumentsAfterConflictsIntegTests.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/BulkByScrollUsesAllScrollDocumentsAfterConflictsIntegTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.reindex; diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/CrossClusterReindexIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/CrossClusterReindexIT.java index e0396039029c5..a76ddf13e4595 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/CrossClusterReindexIT.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/CrossClusterReindexIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.reindex; diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/ReindexPluginMetricsIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/ReindexPluginMetricsIT.java index e7d26b0808a48..d9580c13eada6 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/ReindexPluginMetricsIT.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/ReindexPluginMetricsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.reindex; diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java index 3bbc8e4b969ee..4b21db497ead7 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.migration; diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java index bf34c322c1a95..a4aa0514bb47a 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.migration; @@ -207,7 +208,7 @@ public void testMigrateInternalManagedSystemIndex() throws Exception { assertIndexHasCorrectProperties( finalMetadata, - ".int-man-old-reindexed-for-8", + ".int-man-old-reindexed-for-9", INTERNAL_MANAGED_FLAG_VALUE, true, true, @@ -215,7 +216,7 @@ public void testMigrateInternalManagedSystemIndex() throws Exception { ); assertIndexHasCorrectProperties( finalMetadata, - ".int-unman-old-reindexed-for-8", + ".int-unman-old-reindexed-for-9", INTERNAL_UNMANAGED_FLAG_VALUE, false, true, @@ -223,7 +224,7 @@ public void testMigrateInternalManagedSystemIndex() throws Exception { ); assertIndexHasCorrectProperties( finalMetadata, - ".ext-man-old-reindexed-for-8", + ".ext-man-old-reindexed-for-9", EXTERNAL_MANAGED_FLAG_VALUE, true, false, @@ -231,7 +232,7 @@ public void testMigrateInternalManagedSystemIndex() throws Exception { ); assertIndexHasCorrectProperties( finalMetadata, - ".ext-unman-old-reindexed-for-8", + ".ext-unman-old-reindexed-for-9", EXTERNAL_UNMANAGED_FLAG_VALUE, false, false, diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/MultiFeatureMigrationIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/MultiFeatureMigrationIT.java index ebe4b1835b103..bcce51fe97545 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/MultiFeatureMigrationIT.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/MultiFeatureMigrationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.migration; @@ -218,7 +219,7 @@ public void testMultipleFeatureMigration() throws Exception { // Finally, verify that all the indices exist and have the properties we expect. assertIndexHasCorrectProperties( finalMetadata, - ".int-man-old-reindexed-for-8", + ".int-man-old-reindexed-for-9", INTERNAL_MANAGED_FLAG_VALUE, true, true, @@ -226,7 +227,7 @@ public void testMultipleFeatureMigration() throws Exception { ); assertIndexHasCorrectProperties( finalMetadata, - ".int-unman-old-reindexed-for-8", + ".int-unman-old-reindexed-for-9", INTERNAL_UNMANAGED_FLAG_VALUE, false, true, @@ -234,7 +235,7 @@ public void testMultipleFeatureMigration() throws Exception { ); assertIndexHasCorrectProperties( finalMetadata, - ".ext-man-old-reindexed-for-8", + ".ext-man-old-reindexed-for-9", EXTERNAL_MANAGED_FLAG_VALUE, true, false, @@ -242,7 +243,7 @@ public void testMultipleFeatureMigration() throws Exception { ); assertIndexHasCorrectProperties( finalMetadata, - ".ext-unman-old-reindexed-for-8", + ".ext-unman-old-reindexed-for-9", EXTERNAL_UNMANAGED_FLAG_VALUE, false, false, @@ -251,7 +252,7 @@ public void testMultipleFeatureMigration() throws Exception { assertIndexHasCorrectProperties( finalMetadata, - ".second-int-man-old-reindexed-for-8", + ".second-int-man-old-reindexed-for-9", SECOND_FEATURE_IDX_FLAG_VALUE, true, true, diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/SystemIndexMigrationIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/SystemIndexMigrationIT.java index 6484d483bbcd8..018b5e5361711 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/SystemIndexMigrationIT.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/SystemIndexMigrationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.migration; diff --git a/modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/ManyDocumentsIT.java b/modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/ManyDocumentsIT.java index 701e44295a2d4..bd050fe40667d 100644 --- a/modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/ManyDocumentsIT.java +++ b/modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/ManyDocumentsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.reindex; diff --git a/modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/ReindexWithoutContentIT.java b/modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/ReindexWithoutContentIT.java index 773a577e87407..a0212a937f27b 100644 --- a/modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/ReindexWithoutContentIT.java +++ b/modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/ReindexWithoutContentIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.reindex; @@ -17,7 +18,6 @@ import static org.hamcrest.CoreMatchers.containsString; public class ReindexWithoutContentIT extends ESRestTestCase { - public void testReindexMissingBody() throws IOException { ResponseException responseException = expectThrows( ResponseException.class, diff --git a/modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java b/modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java index c34e0933bbca4..62815a12605d0 100644 --- a/modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java +++ b/modules/reindex/src/javaRestTest/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.reindex.remote; diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollAction.java index 7a2166e5860b4..6a5cf8a95bf97 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBaseReindexRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBaseReindexRestHandler.java index 6643e2e9d20ea..f9721fd0bcc13 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBaseReindexRestHandler.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBaseReindexRestHandler.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBulkByQueryRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBulkByQueryRestHandler.java index 8cf7d2200ad36..48a892f796f92 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBulkByQueryRestHandler.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBulkByQueryRestHandler.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/AsyncDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/AsyncDeleteByQueryAction.java index 45f381a7218c2..cbe5e0745cdf3 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/AsyncDeleteByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/AsyncDeleteByQueryAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelper.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelper.java index 82c42f5d680ff..0b5be5e0f0b5b 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelper.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/BulkIndexByScrollResponseContentListener.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/BulkIndexByScrollResponseContentListener.java index 2c597dba6e22f..b9958283a248d 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/BulkIndexByScrollResponseContentListener.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/BulkIndexByScrollResponseContentListener.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/DeleteByQueryMetrics.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/DeleteByQueryMetrics.java index 2cedf0d5f5823..77e79177b7647 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/DeleteByQueryMetrics.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/DeleteByQueryMetrics.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexMetrics.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexMetrics.java index 3025357aa6538..d985a21815103 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexMetrics.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexMetrics.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java index 3169d4c4ee1fb..eb49f63101d4e 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexSslConfig.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexSslConfig.java index ba2b42c38ef20..914311e1190c1 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexSslConfig.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexSslConfig.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexValidator.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexValidator.java index 69e21b10ac3a4..4b960e97ce0e0 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexValidator.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexValidator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; @@ -155,19 +156,10 @@ static void validateAgainstAliases( } private static SearchRequest skipRemoteIndexNames(SearchRequest source) { - return new SearchRequest(source).indices( - Arrays.stream(source.indices()).filter(name -> isRemoteExpression(name) == false).toArray(String[]::new) - ); - } - - private static boolean isRemoteExpression(String expression) { // An index expression that references a remote cluster uses ":" to separate the cluster-alias from the index portion of the // expression, e.g., cluster0:index-name - // in the same time date-math `expression` can also contain ':' symbol inside its name - // to distinguish between those two, given `expression` is pre-evaluated using date-math resolver - // after evaluation date-math `expression` should not contain ':' symbol - // otherwise if `expression` is legit remote name, ':' symbol remains - return IndexNameExpressionResolver.resolveDateMathExpression(expression) - .contains(String.valueOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR)); + return new SearchRequest(source).indices( + Arrays.stream(source.indices()).filter(name -> RemoteClusterAware.isRemoteIndexName(name) == false).toArray(String[]::new) + ); } } diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index cb393a42f52a1..371d3488c3099 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/RestDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/RestDeleteByQueryAction.java index ff0ef1282b2d9..d3bdb3e007447 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/RestDeleteByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/RestDeleteByQueryAction.java @@ -1,14 +1,16 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.reindex.DeleteByQueryAction; @@ -42,7 +44,7 @@ public List routes() { return List.of( new Route(POST, "/{index}/_delete_by_query"), Route.builder(POST, "/{index}/{type}/_delete_by_query") - .deprecated(RestSearchAction.TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7) + .deprecated(RestSearchAction.TYPES_DEPRECATION_MESSAGE, DeprecationLogger.CRITICAL, RestApiVersion.V_7) .build() ); diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/RestReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/RestReindexAction.java index a693b0babaa9f..925d84c1d03eb 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/RestReindexAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/RestReindexAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/RestRethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/RestRethrottleAction.java index 5d89fdb45cbde..d75865d6e4d3a 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/RestRethrottleAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/RestRethrottleAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/RestUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/RestUpdateByQueryAction.java index 2a6146b9fad1c..47b6c578ee41b 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/RestUpdateByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/RestUpdateByQueryAction.java @@ -1,14 +1,16 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.reindex.UpdateByQueryAction; @@ -43,7 +45,7 @@ public List routes() { return List.of( new Route(POST, "/{index}/_update_by_query"), Route.builder(POST, "/{index}/{type}/_update_by_query") - .deprecated(RestSearchAction.TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7) + .deprecated(RestSearchAction.TYPES_DEPRECATION_MESSAGE, DeprecationLogger.CRITICAL, RestApiVersion.V_7) .build() ); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/RethrottleRequest.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/RethrottleRequest.java index 03f33fba7ef85..cb9e5d4e04654 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/RethrottleRequest.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/RethrottleRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/RethrottleRequestBuilder.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/RethrottleRequestBuilder.java index bf5c68882e3dc..e2435ba9f6865 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/RethrottleRequestBuilder.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/RethrottleRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportDeleteByQueryAction.java index 53381c33d7f78..ead9c06288ee0 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportDeleteByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportDeleteByQueryAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java index 821a137ac7566..033a4d4b28fe5 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportRethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportRethrottleAction.java index 4d09253e046d4..53efebf8465d7 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportRethrottleAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportRethrottleAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportUpdateByQueryAction.java index 997d4d32fe042..1d6061f48bb46 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportUpdateByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportUpdateByQueryAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/UpdateByQueryMetrics.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/UpdateByQueryMetrics.java index 6ca52769a1ba9..8d67422fbc7e0 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/UpdateByQueryMetrics.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/UpdateByQueryMetrics.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/package-info.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/package-info.java index 4f4f63ad88412..a07ac361e5551 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/package-info.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteRequestBuilders.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteRequestBuilders.java index d6942fa87687f..ddf73e313e830 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteRequestBuilders.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteRequestBuilders.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex.remote; diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteResponseParsers.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteResponseParsers.java index 76c24aef01f6c..b924f8c311115 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteResponseParsers.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteResponseParsers.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex.remote; diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteScrollableHitSource.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteScrollableHitSource.java index b6728d09d89f8..29204b5bb0163 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteScrollableHitSource.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/RemoteScrollableHitSource.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex.remote; diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/package-info.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/package-info.java index 10d7e29d40389..52d6c74d7483c 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/package-info.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/remote/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/modules/reindex/src/main/plugin-metadata/plugin-security.policy b/modules/reindex/src/main/plugin-metadata/plugin-security.policy index e12973b8eef67..016cc6365b6ee 100644 --- a/modules/reindex/src/main/plugin-metadata/plugin-security.policy +++ b/modules/reindex/src/main/plugin-metadata/plugin-security.policy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ grant { diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollActionMetadataTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollActionMetadataTestCase.java index 5db3e434b54e6..2680b7e838479 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollActionMetadataTestCase.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollActionMetadataTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java index aa338b3c87eca..1ac85677b5ccb 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java index 47505919ba7d2..cb3738938a354 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; @@ -198,7 +199,6 @@ public void testStartRetriesOnRejectionButFailsOnTooManyRejections() throws Exce DummyAsyncBulkByScrollAction action = new DummyActionWithoutBackoff(); action.start(); assertBusy(() -> assertEquals(testRequest.getMaxRetries() + 1, client.searchAttempts.get())); - assertBusy(() -> assertTrue(listener.isDone())); ExecutionException e = expectThrows(ExecutionException.class, () -> listener.get()); assertThat(ExceptionsHelper.stackTrace(e), containsString(EsRejectedExecutionException.class.getSimpleName())); assertNull("There shouldn't be a search attempt pending that we didn't reject", client.lastSearch.get()); diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelperTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelperTests.java index f368139c21253..ebb4471566fbd 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelperTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelperTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; @@ -14,8 +15,8 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.util.Collections; -import static java.util.Collections.emptyList; import static org.elasticsearch.reindex.BulkByScrollParallelizationHelper.sliceIntoSubRequests; import static org.elasticsearch.search.RandomSearchRequestGenerator.randomSearchRequest; import static org.elasticsearch.search.RandomSearchRequestGenerator.randomSearchSourceBuilder; @@ -23,7 +24,7 @@ public class BulkByScrollParallelizationHelperTests extends ESTestCase { public void testSliceIntoSubRequests() throws IOException { SearchRequest searchRequest = randomSearchRequest( - () -> randomSearchSourceBuilder(() -> null, () -> null, () -> null, () -> null, () -> emptyList(), () -> null, () -> null) + () -> randomSearchSourceBuilder(() -> null, () -> null, () -> null, Collections::emptyList, () -> null, () -> null) ); if (searchRequest.source() != null) { // Clear the slice builder if there is one set. We can't call sliceIntoSubRequests if it is. diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkIndexByScrollResponseMatcher.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkIndexByScrollResponseMatcher.java index f38563ed7d48b..ce5736b47a9e1 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkIndexByScrollResponseMatcher.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkIndexByScrollResponseMatcher.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkIndexByScrollResponseTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkIndexByScrollResponseTests.java index 1441dce7478bf..32d5f6a50a39b 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkIndexByScrollResponseTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkIndexByScrollResponseTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/CancelTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/CancelTests.java index a2911090ab931..e7b05f9e347ea 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/CancelTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/CancelTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java index d628a013a37bb..ee65b980c5fc8 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryBasicTests.java index 17dd1503e6c89..3d37a238a42c1 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryBasicTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryConcurrentTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryConcurrentTests.java index 190616b9980f0..32bafc0298847 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryConcurrentTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryConcurrentTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryMetricsTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryMetricsTests.java index 58adc6aebaa9b..2177c4f4ba7d4 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryMetricsTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryMetricsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexBasicTests.java index 21f6427dcb632..d23ce8eb49f4e 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexBasicTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFailureTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFailureTests.java index 1da998831ecc2..1a98d15ce6375 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFailureTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFailureTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFromRemoteBuildRestClientTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFromRemoteBuildRestClientTests.java index 18c57fe342de3..7ec7174dbcfbb 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFromRemoteBuildRestClientTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFromRemoteBuildRestClientTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFromRemoteWhitelistTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFromRemoteWhitelistTests.java index 83adc2586c172..3219c45926f38 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFromRemoteWhitelistTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFromRemoteWhitelistTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFromRemoteWithAuthTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFromRemoteWithAuthTests.java index c641e04f84134..8b42de8eb3870 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFromRemoteWithAuthTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFromRemoteWithAuthTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexIdTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexIdTests.java index 644787446547e..31addc6e11cbc 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexIdTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexIdTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexMetadataTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexMetadataTests.java index a24536a7502c0..d931d37d28104 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexMetadataTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexMetadataTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexMetricsTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexMetricsTests.java index 4711530585817..1356dc50dadb0 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexMetricsTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexMetricsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexRestClientSslTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexRestClientSslTests.java index dc25fa2d2830f..766c3ff695f84 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexRestClientSslTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexRestClientSslTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexScriptTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexScriptTests.java index 380510b814f06..6706c8ab85482 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexScriptTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexScriptTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexSingleNodeTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexSingleNodeTests.java index 24753c2b9ae6a..c58cdf2490f05 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexSingleNodeTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexSingleNodeTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexSourceTargetValidationTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexSourceTargetValidationTests.java index 38f289d894bd6..6b45b5d5339e2 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexSourceTargetValidationTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexSourceTargetValidationTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexTestCase.java index 129d23c010917..4ad503ff98880 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexTestCase.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexVersioningTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexVersioningTests.java index 8e42b29468b5c..2321198c0356c 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexVersioningTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexVersioningTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/RestDeleteByQueryActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/RestDeleteByQueryActionTests.java index 013eace19f1b5..505b12833fb5e 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/RestDeleteByQueryActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/RestDeleteByQueryActionTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/RestReindexActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/RestReindexActionTests.java index 2e1810482bb5f..61c4bd5ad3aea 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/RestReindexActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/RestReindexActionTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/RestUpdateByQueryActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/RestUpdateByQueryActionTests.java index b83f11a91d1b8..0d35b30c86a5a 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/RestUpdateByQueryActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/RestUpdateByQueryActionTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/RethrottleTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/RethrottleTests.java index 5f1af05571585..a06d98d862c86 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/RethrottleTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/RethrottleTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/RetryTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/RetryTests.java index 36fe08afc1ef5..9c30ba07069f9 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/RetryTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/RetryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/RoundTripTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/RoundTripTests.java index 0697ae5f8bd71..48856aa4007e1 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/RoundTripTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/RoundTripTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/TransportRethrottleActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/TransportRethrottleActionTests.java index ec8e0ce87ee56..5e468c39b68f5 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/TransportRethrottleActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/TransportRethrottleActionTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryBasicTests.java index 6b1f3a21a1aad..36eb1a77d1d0e 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryBasicTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryMetadataTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryMetadataTests.java index cdad8316702d0..d20704f07b57e 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryMetadataTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryMetadataTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryMetricsTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryMetricsTests.java index 548d18d202984..036019f9fae73 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryMetricsTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryMetricsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryVersionTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryVersionTests.java index 4a4ff372cec10..39c6b43db45d5 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryVersionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryVersionTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWhileModifyingTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWhileModifyingTests.java index 5c2e82f6d4256..213001296cbf0 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWhileModifyingTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWhileModifyingTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWithScriptTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWithScriptTests.java index c4d591f804750..7eb4a8dc57192 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWithScriptTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWithScriptTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteInfoTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteInfoTests.java index 84ff9d28b5211..1ea8b6a1a5c3c 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteInfoTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteInfoTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex.remote; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteRequestBuildersTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteRequestBuildersTests.java index dfd62c1482369..81d6500acc038 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteRequestBuildersTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteRequestBuildersTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex.remote; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteResponseParsersTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteResponseParsersTests.java index 1c4dd381729aa..b58305067aae2 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteResponseParsersTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteResponseParsersTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex.remote; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteScrollableHitSourceTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteScrollableHitSourceTests.java index 4b76045c5dc63..3224a8429792d 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteScrollableHitSourceTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteScrollableHitSourceTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reindex.remote; diff --git a/modules/reindex/src/yamlRestTest/java/org/elasticsearch/index/reindex/ReindexClientYamlTestSuiteIT.java b/modules/reindex/src/yamlRestTest/java/org/elasticsearch/index/reindex/ReindexClientYamlTestSuiteIT.java index 833fe72a851d7..0378a63ed5481 100644 --- a/modules/reindex/src/yamlRestTest/java/org/elasticsearch/index/reindex/ReindexClientYamlTestSuiteIT.java +++ b/modules/reindex/src/yamlRestTest/java/org/elasticsearch/index/reindex/ReindexClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.reindex; @@ -11,8 +12,10 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; public class ReindexClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { public ReindexClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { @@ -23,4 +26,18 @@ public ReindexClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCa public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .module("reindex") + .module("lang-painless") + .module("parent-join") + .module("rest-root") + .setting("reindex.remote.whitelist", "127.0.0.1:*") + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/110_synthetic_source.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/110_synthetic_source.yml index 9ae2153f89ca5..6d65e42aa223d 100644 --- a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/110_synthetic_source.yml +++ b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/110_synthetic_source.yml @@ -1,11 +1,16 @@ setup: + - requires: + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" + - do: indices.create: index: synthetic body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: kwd: type: keyword diff --git a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/100_synthetic_source.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/100_synthetic_source.yml index 4329bf8ed471a..b9484dcf7db29 100644 --- a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/100_synthetic_source.yml +++ b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/100_synthetic_source.yml @@ -1,11 +1,16 @@ update: + - requires: + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" + - do: indices.create: index: synthetic body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: kwd: type: keyword diff --git a/modules/repository-azure/build.gradle b/modules/repository-azure/build.gradle index 6334e5ae6a195..7894332bb6326 100644 --- a/modules/repository-azure/build.gradle +++ b/modules/repository-azure/build.gradle @@ -5,17 +5,18 @@ import org.elasticsearch.gradle.internal.test.RestIntegTestTask /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.internal-yaml-rest-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { - description 'The Azure Repository plugin adds support for Azure storage repositories.' - classname 'org.elasticsearch.repositories.azure.AzureRepositoryPlugin' + description = 'The Azure Repository plugin adds support for Azure storage repositories.' + classname ='org.elasticsearch.repositories.azure.AzureRepositoryPlugin' } versions << [ @@ -54,15 +55,19 @@ dependencies { api "io.projectreactor.netty:reactor-netty-core:${versions.azureReactorNetty}" api "io.projectreactor.netty:reactor-netty-http:${versions.azureReactorNetty}" api "io.projectreactor:reactor-core:3.4.38" - api "org.reactivestreams:reactive-streams:1.0.4" + api "org.reactivestreams:reactive-streams:${versions.reactive_streams}" // Others api "com.fasterxml.woodstox:woodstox-core:6.7.0" api "com.github.stephenc.jcip:jcip-annotations:1.0-1" api "com.nimbusds:content-type:2.3" api "com.nimbusds:lang-tag:1.7" - api "com.nimbusds:nimbus-jose-jwt:9.37.3" - api "com.nimbusds:oauth2-oidc-sdk:11.9.1" + api("com.nimbusds:nimbus-jose-jwt:9.37.3"){ + exclude group: 'com.google.crypto.tink', module: 'tink' // it's an optional dependency on which we don't rely + } + api("com.nimbusds:oauth2-oidc-sdk:11.9.1"){ + exclude group: 'com.google.crypto.tink', module: 'tink' // it's an optional dependency on which we don't rely + } api "jakarta.activation:jakarta.activation-api:1.2.1" api "jakarta.xml.bind:jakarta.xml.bind-api:2.3.3" api "net.java.dev.jna:jna-platform:${versions.jna}" // Maven says 5.14.0 but this aligns with the Elasticsearch-wide version @@ -72,8 +77,6 @@ dependencies { api "org.codehaus.woodstox:stax2-api:4.2.2" api "org.ow2.asm:asm:9.3" - runtimeOnly "com.google.crypto.tink:tink:1.14.0" - runtimeOnly "com.google.protobuf:protobuf-java:4.27.0" runtimeOnly "com.google.code.gson:gson:2.11.0" runtimeOnly "org.cryptomator:siv-mode:1.5.2" @@ -173,13 +176,11 @@ tasks.named("thirdPartyAudit").configure { // 'org.slf4j.ext.EventData' - bring back when https://github.com/elastic/elasticsearch/issues/93714 is done // Optional dependency of tink - 'com.google.api.client.http.HttpHeaders', - 'com.google.api.client.http.HttpRequest', - 'com.google.api.client.http.HttpRequestFactory', - 'com.google.api.client.http.HttpResponse', - 'com.google.api.client.http.HttpTransport', - 'com.google.api.client.http.javanet.NetHttpTransport', - 'com.google.api.client.http.javanet.NetHttpTransport$Builder', + 'com.google.crypto.tink.subtle.Ed25519Sign', + 'com.google.crypto.tink.subtle.Ed25519Sign$KeyPair', + 'com.google.crypto.tink.subtle.Ed25519Verify', + 'com.google.crypto.tink.subtle.X25519', + 'com.google.crypto.tink.subtle.XChaCha20Poly1305', // Optional dependency of nimbus-jose-jwt and oauth2-oidc-sdk 'org.bouncycastle.asn1.pkcs.PrivateKeyInfo', @@ -251,14 +252,6 @@ tasks.named("thirdPartyAudit").configure { 'javax.activation.MailcapCommandMap', 'javax.activation.MimetypesFileTypeMap', 'reactor.core.publisher.Traces$SharedSecretsCallSiteSupplierFactory$TracingException', - - 'com.google.protobuf.MessageSchema', - 'com.google.protobuf.UnsafeUtil', - 'com.google.protobuf.UnsafeUtil$1', - 'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor', - 'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor', - 'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor', - 'com.google.protobuf.UnsafeUtil$MemoryAccessor', ) } @@ -327,7 +320,7 @@ tasks.register("workloadIdentityYamlRestTest", RestIntegTestTask) { // omitting key and sas_token so that we use a bearer token from workload identity } -if (BuildParams.inFipsJvm) { +if (buildParams.inFipsJvm) { // Cannot override the trust store in FIPS mode, and these tasks require a HTTPS fixture tasks.named("managedIdentityYamlRestTest").configure { enabled = false } tasks.named("workloadIdentityYamlRestTest").configure { enabled = false } diff --git a/modules/repository-azure/licenses/protobuf-java-LICENSE.txt b/modules/repository-azure/licenses/protobuf-java-LICENSE.txt deleted file mode 100644 index 19b305b00060a..0000000000000 --- a/modules/repository-azure/licenses/protobuf-java-LICENSE.txt +++ /dev/null @@ -1,32 +0,0 @@ -Copyright 2008 Google Inc. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Code generated by the Protocol Buffer compiler is owned by the owner -of the input file used when generating it. This code is not -standalone and requires a support library to be linked with it. This -support library is itself covered by the above license. diff --git a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java index 15d47f6bec800..1b7628cc0ad8e 100644 --- a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java +++ b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.azure; diff --git a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureRepositoryMissingCredentialsIT.java b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureRepositoryMissingCredentialsIT.java index 0ce8b7a3e8ea4..947f73c2ce580 100644 --- a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureRepositoryMissingCredentialsIT.java +++ b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureRepositoryMissingCredentialsIT.java @@ -1,14 +1,14 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.azure; -import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest; import org.elasticsearch.action.admin.cluster.repositories.put.TransportPutRepositoryAction; import org.elasticsearch.action.support.master.AcknowledgedResponse; @@ -40,17 +40,13 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { public void testMissingCredentialsException() { assertThat( - asInstanceOf( + safeAwaitAndUnwrapFailure( RepositoryVerificationException.class, - ExceptionsHelper.unwrapCause( - safeAwaitFailure( - AcknowledgedResponse.class, - l -> client().execute( - TransportPutRepositoryAction.TYPE, - new PutRepositoryRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "test-repo").type("azure"), - l - ) - ) + AcknowledgedResponse.class, + l -> client().execute( + TransportPutRepositoryAction.TYPE, + new PutRepositoryRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "test-repo").type("azure"), + l ) ).getCause().getMessage(), allOf( diff --git a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java index 80bd788da9a9f..b2b6eba4a7c03 100644 --- a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java +++ b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.azure; @@ -58,31 +59,6 @@ public class AzureStorageCleanupThirdPartyTests extends AbstractThirdPartyReposi AzureHttpFixture.sharedKeyForAccountPredicate(AZURE_ACCOUNT) ); - @Override - public void testCreateSnapshot() { - super.testCreateSnapshot(); - } - - @Override - public void testIndexLatest() throws Exception { - super.testIndexLatest(); - } - - @Override - public void testListChildren() { - super.testListChildren(); - } - - @Override - public void testCleanup() throws Exception { - super.testCleanup(); - } - - @Override - public void testReadFromPositionWithLength() { - super.testReadFromPositionWithLength(); - } - @Override protected Collection> getPlugins() { return pluginList(AzureRepositoryPlugin.class); diff --git a/modules/repository-azure/src/main/java/module-info.java b/modules/repository-azure/src/main/java/module-info.java index bef2a175a112f..cd6be56b71543 100644 --- a/modules/repository-azure/src/main/java/module-info.java +++ b/modules/repository-azure/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.repository.azure { diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java index eb643b3c1460d..b01f4c22b44e5 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.azure; @@ -133,7 +134,7 @@ public DeleteResult delete(OperationPurpose purpose) throws IOException { @Override public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobNames) throws IOException { - blobStore.deleteBlobsIgnoringIfNotExists(purpose, new Iterator<>() { + blobStore.deleteBlobs(new Iterator<>() { @Override public boolean hasNext() { return blobNames.hasNext(); diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobServiceClient.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobServiceClient.java index 7e5c230453e18..6cb52950b280a 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobServiceClient.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobServiceClient.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.azure; diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java index 06e556e5a8788..6d96a1281af88 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.azure; @@ -46,7 +47,6 @@ import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.blobstore.DeleteResult; -import org.elasticsearch.common.blobstore.OperationPurpose; import org.elasticsearch.common.blobstore.OptionalBytesReference; import org.elasticsearch.common.blobstore.support.BlobContainerUtils; import org.elasticsearch.common.blobstore.support.BlobMetadata; @@ -264,8 +264,7 @@ private static void filterDeleteExceptionsAndRethrow(Exception e, IOException ex throw exception; } - @Override - public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobs) throws IOException { + void deleteBlobs(Iterator blobs) throws IOException { if (blobs.hasNext() == false) { return; } diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java index 911d1c167db7f..e21c1384db271 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.azure; diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java index c8c0b15db5ebe..826832971790c 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.azure; diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java index f6b1c2775926c..c3cd5e78c5dbe 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.azure; diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java index 09088004759a8..17c719b97e448 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.azure; diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java index 2333a1fdb9e93..508efba57923c 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.azure; diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/CancellableRateLimitedFluxIterator.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/CancellableRateLimitedFluxIterator.java index ed2e0bf9be0b2..6230aacbf80a7 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/CancellableRateLimitedFluxIterator.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/CancellableRateLimitedFluxIterator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.azure; diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/LocationMode.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/LocationMode.java index aa20112c07098..d7e3d652029b2 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/LocationMode.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/LocationMode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.azure; diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/SocketAccess.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/SocketAccess.java index cc97b399d8204..cb93858b533a4 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/SocketAccess.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/SocketAccess.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.azure; diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/executors/PrivilegedExecutor.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/executors/PrivilegedExecutor.java index 74ade64eb0bc7..fa3937dd6b406 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/executors/PrivilegedExecutor.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/executors/PrivilegedExecutor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.azure.executors; diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/executors/ReactorScheduledExecutorService.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/executors/ReactorScheduledExecutorService.java index 5f18a417dcccd..6610e2eb3256a 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/executors/ReactorScheduledExecutorService.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/executors/ReactorScheduledExecutorService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.azure.executors; diff --git a/modules/repository-azure/src/main/plugin-metadata/plugin-security.policy b/modules/repository-azure/src/main/plugin-metadata/plugin-security.policy index 8bc2c35fd170b..8a7c623597376 100644 --- a/modules/repository-azure/src/main/plugin-metadata/plugin-security.policy +++ b/modules/repository-azure/src/main/plugin-metadata/plugin-security.policy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ grant { diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureServerTestCase.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureServerTestCase.java index facfd1328ad4e..303f82cf34f9a 100644 --- a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureServerTestCase.java +++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureServerTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.azure; diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java index 9fed6708de53c..9e9313249e967 100644 --- a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java +++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.azure; diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureClientProviderTests.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureClientProviderTests.java index 79fa2afe5d49e..d595cded53dc7 100644 --- a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureClientProviderTests.java +++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureClientProviderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.azure; diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java index 40cd1ff49ac1f..7037dd4eaf111 100644 --- a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java +++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.azure; diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSasTokenTests.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSasTokenTests.java index ec74918f601cc..b10472688e3b0 100644 --- a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSasTokenTests.java +++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSasTokenTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.azure; diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java index 3f9fba35d9f4a..85382c4f022f0 100644 --- a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java +++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.azure; diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/CancellableRateLimitedFluxIteratorTests.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/CancellableRateLimitedFluxIteratorTests.java index efab7e51b9689..ebe8e8204f6ba 100644 --- a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/CancellableRateLimitedFluxIteratorTests.java +++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/CancellableRateLimitedFluxIteratorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.azure; diff --git a/modules/repository-azure/src/yamlRestTest/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java b/modules/repository-azure/src/yamlRestTest/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java index 515b1aa2b6d6a..64dde0248ad2c 100644 --- a/modules/repository-azure/src/yamlRestTest/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java +++ b/modules/repository-azure/src/yamlRestTest/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.azure; diff --git a/modules/repository-gcs/build.gradle b/modules/repository-gcs/build.gradle index 32236ab0fd8af..2a2efb9c4fb3d 100644 --- a/modules/repository-gcs/build.gradle +++ b/modules/repository-gcs/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ @@ -17,8 +18,8 @@ apply plugin: 'elasticsearch.internal-yaml-rest-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { - description 'The GCS repository plugin adds Google Cloud Storage support for repositories.' - classname 'org.elasticsearch.repositories.gcs.GoogleCloudStoragePlugin' + description = 'The GCS repository plugin adds Google Cloud Storage support for repositories.' + classname ='org.elasticsearch.repositories.gcs.GoogleCloudStoragePlugin' } dependencies { @@ -177,7 +178,7 @@ tasks.named("thirdPartyAudit").configure { ) - if(BuildParams.graalVmRuntime == false) { + if(buildParams.graalVmRuntime == false) { ignoreMissingClasses( 'org.graalvm.nativeimage.hosted.Feature', 'org.graalvm.nativeimage.hosted.Feature$BeforeAnalysisAccess', @@ -239,7 +240,7 @@ def gcsThirdPartyTest = tasks.register("gcsThirdPartyUnitTest", Test) { systemProperty 'tests.security.manager', false systemProperty 'test.google.bucket', gcsBucket systemProperty 'test.google.fixture', Boolean.toString(useFixture) - nonInputProperties.systemProperty 'test.google.base', gcsBasePath + "_third_party_tests_" + BuildParams.testSeed + nonInputProperties.systemProperty 'test.google.base', gcsBasePath + "_third_party_tests_" + buildParams.testSeed if (useFixture == false) { nonInputProperties.systemProperty 'test.google.account', "${-> encodedCredentials.call()}" } diff --git a/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/ForwardedViaProxyHandler.java b/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/ForwardedViaProxyHandler.java index 8b6e9d7c4a892..c8eaea95ac3f2 100644 --- a/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/ForwardedViaProxyHandler.java +++ b/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/ForwardedViaProxyHandler.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.gcs; diff --git a/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GcsProxyIntegrationTests.java b/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GcsProxyIntegrationTests.java index e29e547718d45..d15284f143522 100644 --- a/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GcsProxyIntegrationTests.java +++ b/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GcsProxyIntegrationTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.gcs; diff --git a/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java index e6625ea1b7f15..6fa8cb9be5628 100644 --- a/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java +++ b/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.gcs; diff --git a/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java b/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java index 3c13360029bce..f99c488a86b07 100644 --- a/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java +++ b/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.gcs; diff --git a/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/WebProxyServer.java b/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/WebProxyServer.java index bbfe738ad0932..a2c3942e93128 100644 --- a/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/WebProxyServer.java +++ b/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/WebProxyServer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.gcs; diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java index 9e587eb4dc543..f502e4171069e 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.gcs; @@ -102,12 +103,12 @@ public void writeBlobAtomic(OperationPurpose purpose, String blobName, BytesRefe @Override public DeleteResult delete(OperationPurpose purpose) throws IOException { - return blobStore.deleteDirectory(purpose, path().buildAsString()); + return blobStore.deleteDirectory(path().buildAsString()); } @Override public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobNames) throws IOException { - blobStore.deleteBlobsIgnoringIfNotExists(purpose, new Iterator<>() { + blobStore.deleteBlobs(new Iterator<>() { @Override public boolean hasNext() { return blobNames.hasNext(); diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java index 72df453a4e8f9..c92b9581d3601 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.gcs; @@ -27,7 +28,6 @@ import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.blobstore.DeleteResult; -import org.elasticsearch.common.blobstore.OperationPurpose; import org.elasticsearch.common.blobstore.OptionalBytesReference; import org.elasticsearch.common.blobstore.support.BlobContainerUtils; import org.elasticsearch.common.blobstore.support.BlobMetadata; @@ -489,10 +489,9 @@ private void writeBlobMultipart(BlobInfo blobInfo, byte[] buffer, int offset, in /** * Deletes the given path and all its children. * - * @param purpose The purpose of the delete operation * @param pathStr Name of path to delete */ - DeleteResult deleteDirectory(OperationPurpose purpose, String pathStr) throws IOException { + DeleteResult deleteDirectory(String pathStr) throws IOException { return SocketAccess.doPrivilegedIOException(() -> { DeleteResult deleteResult = DeleteResult.ZERO; Page page = client().list(bucketName, BlobListOption.prefix(pathStr)); @@ -500,7 +499,7 @@ DeleteResult deleteDirectory(OperationPurpose purpose, String pathStr) throws IO final AtomicLong blobsDeleted = new AtomicLong(0L); final AtomicLong bytesDeleted = new AtomicLong(0L); final Iterator blobs = page.getValues().iterator(); - deleteBlobsIgnoringIfNotExists(purpose, new Iterator<>() { + deleteBlobs(new Iterator<>() { @Override public boolean hasNext() { return blobs.hasNext(); @@ -524,11 +523,9 @@ public String next() { /** * Deletes multiple blobs from the specific bucket using a batch request * - * @param purpose the purpose of the delete operation * @param blobNames names of the blobs to delete */ - @Override - public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobNames) throws IOException { + void deleteBlobs(Iterator blobNames) throws IOException { if (blobNames.hasNext() == false) { return; } diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettings.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettings.java index e34380a401be8..a97748785c3a1 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettings.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettings.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.gcs; diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageHttpStatsCollector.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageHttpStatsCollector.java index 03bca5020f449..e7e21360de545 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageHttpStatsCollector.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageHttpStatsCollector.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.gcs; diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageOperationsStats.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageOperationsStats.java index 957b049ad82c3..71f05555cc7d2 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageOperationsStats.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageOperationsStats.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.gcs; diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java index 4f05289899a7f..14281b7288067 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.gcs; diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java index e2338371cf837..b3e7f06ff2d1a 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.gcs; diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRetryingInputStream.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRetryingInputStream.java index 5689123872cc3..9777c542bc605 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRetryingInputStream.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRetryingInputStream.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.gcs; diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java index 760c1c57e0496..6a4eeeeabbb6f 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.gcs; diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/SocketAccess.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/SocketAccess.java index 34b42a6891826..8b682fbef9bef 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/SocketAccess.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/SocketAccess.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.gcs; diff --git a/modules/repository-gcs/src/main/plugin-metadata/entitlement-policy.yaml b/modules/repository-gcs/src/main/plugin-metadata/entitlement-policy.yaml new file mode 100644 index 0000000000000..a1ff54f02d969 --- /dev/null +++ b/modules/repository-gcs/src/main/plugin-metadata/entitlement-policy.yaml @@ -0,0 +1,2 @@ +ALL-UNNAMED: + - set_https_connection_properties # required by google-http-client diff --git a/modules/repository-gcs/src/main/plugin-metadata/plugin-security.policy b/modules/repository-gcs/src/main/plugin-metadata/plugin-security.policy index f21a46521a7f7..36149b5d4ecd5 100644 --- a/modules/repository-gcs/src/main/plugin-metadata/plugin-security.policy +++ b/modules/repository-gcs/src/main/plugin-metadata/plugin-security.policy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ grant { diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java index 7cfe95bc69f23..110c31b212ea1 100644 --- a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java +++ b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.gcs; diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreContainerTests.java b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreContainerTests.java index 5a950ad2a9ecc..43724538aabea 100644 --- a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreContainerTests.java +++ b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreContainerTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.gcs; diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettingsTests.java b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettingsTests.java index bb8705cf7908c..3f42e00dce8a3 100644 --- a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettingsTests.java +++ b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettingsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.gcs; diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePluginTests.java b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePluginTests.java index bacad22a6b913..bd34018dd7821 100644 --- a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePluginTests.java +++ b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePluginTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.gcs; diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRetryingInputStreamTests.java b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRetryingInputStreamTests.java index 639b52ae238c5..2f632a32261ed 100644 --- a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRetryingInputStreamTests.java +++ b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRetryingInputStreamTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.gcs; diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java index 45f017600a16c..c6f6b0315f595 100644 --- a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java +++ b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.gcs; diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockHttpProxyServer.java b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockHttpProxyServer.java index 36c2117ff4459..d5cf6bc2c243e 100644 --- a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockHttpProxyServer.java +++ b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockHttpProxyServer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.gcs; diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockHttpProxyServerTests.java b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockHttpProxyServerTests.java index f6082d4ae9417..b15c370c3c726 100644 --- a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockHttpProxyServerTests.java +++ b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockHttpProxyServerTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.gcs; diff --git a/modules/repository-gcs/src/yamlRestTest/java/org/elasticsearch/repositories/gcs/DefaultCredentialsRepositoryGcsClientYamlTestSuiteIT.java b/modules/repository-gcs/src/yamlRestTest/java/org/elasticsearch/repositories/gcs/DefaultCredentialsRepositoryGcsClientYamlTestSuiteIT.java index 612a2eac17214..e427b94a41313 100644 --- a/modules/repository-gcs/src/yamlRestTest/java/org/elasticsearch/repositories/gcs/DefaultCredentialsRepositoryGcsClientYamlTestSuiteIT.java +++ b/modules/repository-gcs/src/yamlRestTest/java/org/elasticsearch/repositories/gcs/DefaultCredentialsRepositoryGcsClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.gcs; diff --git a/modules/repository-gcs/src/yamlRestTest/java/org/elasticsearch/repositories/gcs/LargeBlobRepositoryGcsClientYamlTestSuiteIT.java b/modules/repository-gcs/src/yamlRestTest/java/org/elasticsearch/repositories/gcs/LargeBlobRepositoryGcsClientYamlTestSuiteIT.java index 64468bff81f75..8c626999a3fe6 100644 --- a/modules/repository-gcs/src/yamlRestTest/java/org/elasticsearch/repositories/gcs/LargeBlobRepositoryGcsClientYamlTestSuiteIT.java +++ b/modules/repository-gcs/src/yamlRestTest/java/org/elasticsearch/repositories/gcs/LargeBlobRepositoryGcsClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.gcs; diff --git a/modules/repository-gcs/src/yamlRestTest/java/org/elasticsearch/repositories/gcs/RepositoryGcsClientYamlTestSuiteIT.java b/modules/repository-gcs/src/yamlRestTest/java/org/elasticsearch/repositories/gcs/RepositoryGcsClientYamlTestSuiteIT.java index 33a0cfae2e5ba..8c8e9a624a9eb 100644 --- a/modules/repository-gcs/src/yamlRestTest/java/org/elasticsearch/repositories/gcs/RepositoryGcsClientYamlTestSuiteIT.java +++ b/modules/repository-gcs/src/yamlRestTest/java/org/elasticsearch/repositories/gcs/RepositoryGcsClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.gcs; diff --git a/modules/repository-s3/build.gradle b/modules/repository-s3/build.gradle index 1732fd39794b9..5664fb6272e49 100644 --- a/modules/repository-s3/build.gradle +++ b/modules/repository-s3/build.gradle @@ -4,28 +4,26 @@ import org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.internal-yaml-rest-test' apply plugin: 'elasticsearch.internal-cluster-test' +apply plugin: 'elasticsearch.internal-java-rest-test' esplugin { - description 'The S3 repository plugin adds S3 repositories' - classname 'org.elasticsearch.repositories.s3.S3RepositoryPlugin' + description = 'The S3 repository plugin adds S3 repositories' + classname ='org.elasticsearch.repositories.s3.S3RepositoryPlugin' } -versions << [ - 'aws': '1.12.270' -] - dependencies { - api "com.amazonaws:aws-java-sdk-s3:${versions.aws}" - api "com.amazonaws:aws-java-sdk-core:${versions.aws}" - api "com.amazonaws:aws-java-sdk-sts:${versions.aws}" - api "com.amazonaws:jmespath-java:${versions.aws}" + api "com.amazonaws:aws-java-sdk-s3:${versions.awsv1sdk}" + api "com.amazonaws:aws-java-sdk-core:${versions.awsv1sdk}" + api "com.amazonaws:aws-java-sdk-sts:${versions.awsv1sdk}" + api "com.amazonaws:jmespath-java:${versions.awsv1sdk}" api "org.apache.httpcomponents:httpclient:${versions.httpclient}" api "org.apache.httpcomponents:httpcore:${versions.httpcore}" api "commons-logging:commons-logging:${versions.commonslogging}" @@ -42,13 +40,26 @@ dependencies { api 'javax.xml.bind:jaxb-api:2.2.2' testImplementation project(':test:fixtures:s3-fixture') - yamlRestTestImplementation project(":test:framework") - yamlRestTestImplementation project(':test:fixtures:s3-fixture') - yamlRestTestImplementation project(':test:fixtures:minio-fixture') + internalClusterTestImplementation project(':test:fixtures:minio-fixture') + internalClusterTestRuntimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}" + yamlRestTestImplementation project(':modules:repository-s3') + yamlRestTestImplementation project(':test:fixtures:aws-fixture-utils') + yamlRestTestImplementation project(':test:fixtures:s3-fixture') + yamlRestTestImplementation project(':test:fixtures:testcontainer-utils') + yamlRestTestImplementation project(':test:framework') yamlRestTestRuntimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}" - internalClusterTestRuntimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}" + + javaRestTestImplementation project(':modules:repository-s3') + javaRestTestImplementation project(':test:fixtures:aws-fixture-utils') + javaRestTestImplementation project(':test:fixtures:aws-sts-fixture') + javaRestTestImplementation project(':test:fixtures:ec2-imds-fixture') + javaRestTestImplementation project(':test:fixtures:minio-fixture') + javaRestTestImplementation project(':test:fixtures:s3-fixture') + javaRestTestImplementation project(':test:fixtures:testcontainer-utils') + javaRestTestImplementation project(':test:framework') + javaRestTestRuntimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}" } restResources { @@ -75,90 +86,25 @@ def testRepositoryCreds = tasks.register("testRepositoryCreds", Test) { testClassesDirs = sourceSets.test.output.classesDirs } -tasks.named('check').configure { - dependsOn(testRepositoryCreds) -} - tasks.named('test').configure { // this is tested explicitly in separate test tasks exclude '**/RepositoryCredentialsTests.class' } boolean useFixture = false - -// We test against two repositories, one which uses the usual two-part "permanent" credentials and -// the other which uses three-part "temporary" or "session" credentials. - String s3PermanentAccessKey = System.getenv("amazon_s3_access_key") String s3PermanentSecretKey = System.getenv("amazon_s3_secret_key") String s3PermanentBucket = System.getenv("amazon_s3_bucket") String s3PermanentBasePath = System.getenv("amazon_s3_base_path") -String s3TemporaryAccessKey = System.getenv("amazon_s3_access_key_temporary") -String s3TemporarySecretKey = System.getenv("amazon_s3_secret_key_temporary") -String s3TemporarySessionToken = System.getenv("amazon_s3_session_token_temporary") -String s3TemporaryBucket = System.getenv("amazon_s3_bucket_temporary") -String s3TemporaryBasePath = System.getenv("amazon_s3_base_path_temporary") - -String s3EC2Bucket = System.getenv("amazon_s3_bucket_ec2") -String s3EC2BasePath = System.getenv("amazon_s3_base_path_ec2") - -String s3ECSBucket = System.getenv("amazon_s3_bucket_ecs") -String s3ECSBasePath = System.getenv("amazon_s3_base_path_ecs") - -String s3STSBucket = System.getenv("amazon_s3_bucket_sts") -String s3STSBasePath = System.getenv("amazon_s3_base_path_sts") - -boolean s3DisableChunkedEncoding = BuildParams.random.nextBoolean() - -// If all these variables are missing then we are testing against the internal fixture instead, which has the following -// credentials hard-coded in. +// If all these variables are missing then we are testing against the internal fixture instead, which has the following credentials hard-coded in. if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath) { + useFixture = true s3PermanentAccessKey = 's3_test_access_key' s3PermanentSecretKey = 's3_test_secret_key' s3PermanentBucket = 'bucket' s3PermanentBasePath = 'base_path' - useFixture = true -} -if (!s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken) { - s3TemporaryAccessKey = 'session_token_access_key' - s3TemporarySecretKey = 'session_token_secret_key' - s3TemporaryBucket = 'session_token_bucket' - s3TemporaryBasePath = 'session_token_base_path' -} - -if (!s3EC2Bucket && !s3EC2BasePath && !s3ECSBucket && !s3ECSBasePath) { - s3EC2Bucket = 'ec2_bucket' - s3EC2BasePath = 'ec2_base_path' - s3ECSBucket = 'ecs_bucket' - s3ECSBasePath = 'ecs_base_path' -} - -if (!s3STSBucket && !s3STSBasePath) { - s3STSBucket = 'sts_bucket' - s3STSBasePath = 'sts_base_path' -} - -tasks.named("processYamlRestTestResources").configure { - from("src/test/resources") { - include "aws-web-identity-token-file" - } - Map expansions = [ - 'permanent_bucket' : s3PermanentBucket, - 'permanent_base_path' : s3PermanentBasePath + "_integration_tests", - 'temporary_bucket' : s3TemporaryBucket, - 'temporary_base_path' : s3TemporaryBasePath + "_integration_tests", - 'ec2_bucket' : s3EC2Bucket, - 'ec2_base_path' : s3EC2BasePath, - 'ecs_bucket' : s3ECSBucket, - 'ecs_base_path' : s3ECSBasePath, - 'sts_bucket' : s3STSBucket, - 'sts_base_path' : s3STSBasePath, - 'disable_chunked_encoding': s3DisableChunkedEncoding - ] - inputs.properties(expansions) - filter("tokens" : expansions.collectEntries {k, v -> [k, v.toString()]} /* must be a map of strings */, ReplaceTokens.class) } tasks.named("internalClusterTest").configure { @@ -168,22 +114,7 @@ tasks.named("internalClusterTest").configure { systemProperty 'es.insecure_network_trace_enabled', 'true' } -tasks.named("yamlRestTest").configure { - systemProperty("s3PermanentAccessKey", s3PermanentAccessKey) - systemProperty("s3PermanentSecretKey", s3PermanentSecretKey) - systemProperty("s3TemporaryAccessKey", s3TemporaryAccessKey) - systemProperty("s3TemporarySecretKey", s3TemporarySecretKey) - systemProperty("s3EC2AccessKey", s3PermanentAccessKey) - - // ideally we could resolve an env path in cluster config as resource similar to configuring a config file - // not sure how common this is, but it would be nice to support - File awsWebIdentityTokenExternalLocation = file('src/test/resources/aws-web-identity-token-file') - // The web identity token can be read only from the plugin config directory because of security restrictions - // Ideally we would create a symlink, but extraConfigFile doesn't support it - nonInputProperties.systemProperty("awsWebIdentityTokenExternalLocation", awsWebIdentityTokenExternalLocation.getAbsolutePath()) -} - -// 3rd Party Tests +// 3rd Party Tests, i.e. testing against a real S3 repository tasks.register("s3ThirdPartyTest", Test) { SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); SourceSet internalTestSourceSet = sourceSets.getByName(InternalClusterTestPlugin.SOURCE_SET_NAME) @@ -191,13 +122,13 @@ tasks.register("s3ThirdPartyTest", Test) { setClasspath(internalTestSourceSet.getRuntimeClasspath()) include '**/S3RepositoryThirdPartyTests.class' systemProperty("tests.use.fixture", Boolean.toString(useFixture)) - - // test container accesses ~/.testcontainers.properties read - systemProperty "tests.security.manager", "false" systemProperty 'test.s3.account', s3PermanentAccessKey systemProperty 'test.s3.key', s3PermanentSecretKey systemProperty 'test.s3.bucket', s3PermanentBucket - nonInputProperties.systemProperty 'test.s3.base', s3PermanentBasePath + "_third_party_tests_" + BuildParams.testSeed + nonInputProperties.systemProperty 'test.s3.base', s3PermanentBasePath + "_third_party_tests_" + buildParams.testSeed + + // test container accesses ~/.testcontainers.properties read + systemProperty "tests.security.manager", "false" } tasks.named("thirdPartyAudit").configure { @@ -234,5 +165,6 @@ tasks.named("thirdPartyAudit").configure { tasks.named("check").configure { dependsOn(tasks.withType(Test)) + dependsOn(testRepositoryCreds) } diff --git a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryMetricsTests.java b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryMetricsTests.java index 31fa47fb7b196..e55668adea101 100644 --- a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryMetricsTests.java +++ b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryMetricsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.s3; diff --git a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java index 7916bb5942fc7..12220dc51ea8f 100644 --- a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.s3; @@ -187,7 +188,10 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { } @Override - @TestIssueLogging(issueUrl = "https://github.com/elastic/elasticsearch/issues/88841", value = "com.amazonaws.request:DEBUG") + @TestIssueLogging( + issueUrl = "https://github.com/elastic/elasticsearch/issues/88841", + value = "com.amazonaws.request:DEBUG,com.amazonaws.http.AmazonHttpClient:TRACE" + ) public void testRequestStats() throws Exception { super.testRequestStats(); } @@ -233,8 +237,10 @@ public void testAbortRequestStats() throws Exception { assertEquals(assertionErrorMsg, mockCalls, sdkRequestCounts); } - @TestIssueLogging(issueUrl = "https://github.com/elastic/elasticsearch/issues/101608", value = "com.amazonaws.request:DEBUG") - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/101608") + @TestIssueLogging( + issueUrl = "https://github.com/elastic/elasticsearch/issues/101608", + value = "com.amazonaws.request:DEBUG,com.amazonaws.http.AmazonHttpClient:TRACE" + ) public void testMetrics() throws Exception { // Create the repository and perform some activities final String repository = createRepository(randomRepositoryName(), false); diff --git a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java index 65c5189504a19..4cebedebfba07 100644 --- a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java +++ b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.s3; @@ -60,7 +61,12 @@ public class S3RepositoryThirdPartyTests extends AbstractThirdPartyRepositoryTes static final boolean USE_FIXTURE = Booleans.parseBoolean(System.getProperty("tests.use.fixture", "true")); @ClassRule - public static MinioTestContainer minio = new MinioTestContainer(USE_FIXTURE); + public static MinioTestContainer minio = new MinioTestContainer( + USE_FIXTURE, + System.getProperty("test.s3.account"), + System.getProperty("test.s3.key"), + System.getProperty("test.s3.bucket") + ); @Override protected Collection> getPlugins() { diff --git a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/AbstractRepositoryS3RestTestCase.java b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/AbstractRepositoryS3RestTestCase.java new file mode 100644 index 0000000000000..67ada622efeea --- /dev/null +++ b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/AbstractRepositoryS3RestTestCase.java @@ -0,0 +1,381 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.repositories.s3; + +import io.netty.handler.codec.http.HttpMethod; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.cluster.routing.Murmur3HashFunction; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.repositories.blobstore.BlobStoreRepository; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ObjectPath; + +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Set; +import java.util.function.UnaryOperator; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public abstract class AbstractRepositoryS3RestTestCase extends ESRestTestCase { + + public record TestRepository(String repositoryName, String clientName, String bucketName, String basePath) { + + public Closeable register() throws IOException { + return register(UnaryOperator.identity()); + } + + public Closeable register(UnaryOperator settingsUnaryOperator) throws IOException { + assertOK(client().performRequest(getRegisterRequest(settingsUnaryOperator))); + return () -> assertOK(client().performRequest(new Request("DELETE", "/_snapshot/" + repositoryName()))); + } + + private Request getRegisterRequest(UnaryOperator settingsUnaryOperator) throws IOException { + return newXContentRequest( + HttpMethod.PUT, + "/_snapshot/" + repositoryName(), + (b, p) -> b.field("type", S3Repository.TYPE) + .startObject("settings") + .value( + settingsUnaryOperator.apply( + Settings.builder() + .put("bucket", bucketName()) + .put("base_path", basePath()) + .put("client", clientName()) + .put("canned_acl", "private") + .put("storage_class", "standard") + .put("disable_chunked_encoding", randomBoolean()) + .build() + ) + ) + .endObject() + ); + } + } + + protected abstract String getBucketName(); + + protected abstract String getBasePath(); + + protected abstract String getClientName(); + + protected static String getIdentifierPrefix(String testSuiteName) { + return testSuiteName + "-" + Integer.toString(Murmur3HashFunction.hash(testSuiteName + System.getProperty("tests.seed")), 16) + "-"; + } + + private TestRepository newTestRepository() { + return new TestRepository(randomIdentifier(), getClientName(), getBucketName(), getBasePath()); + } + + private static UnaryOperator readonlyOperator(Boolean readonly) { + return readonly == null + ? UnaryOperator.identity() + : s -> Settings.builder().put(s).put(BlobStoreRepository.READONLY_SETTING_KEY, readonly).build(); + } + + public void testGetRepository() throws IOException { + testGetRepository(null); + } + + public void testGetRepositoryReadonlyTrue() throws IOException { + testGetRepository(Boolean.TRUE); + } + + public void testGetRepositoryReadonlyFalse() throws IOException { + testGetRepository(Boolean.FALSE); + } + + private void testGetRepository(Boolean readonly) throws IOException { + final var repository = newTestRepository(); + try (var ignored = repository.register(readonlyOperator(readonly))) { + final var repositoryName = repository.repositoryName(); + final var responseObjectPath = assertOKAndCreateObjectPath( + client().performRequest(new Request("GET", "/_snapshot/" + repositoryName)) + ); + + assertEquals("s3", responseObjectPath.evaluate(repositoryName + ".type")); + assertNotNull(responseObjectPath.evaluate(repositoryName + ".settings")); + assertEquals(repository.bucketName(), responseObjectPath.evaluate(repositoryName + ".settings.bucket")); + assertEquals(repository.clientName(), responseObjectPath.evaluate(repositoryName + ".settings.client")); + assertEquals(repository.basePath(), responseObjectPath.evaluate(repositoryName + ".settings.base_path")); + assertEquals("private", responseObjectPath.evaluate(repositoryName + ".settings.canned_acl")); + assertEquals("standard", responseObjectPath.evaluate(repositoryName + ".settings.storage_class")); + assertNull(responseObjectPath.evaluate(repositoryName + ".settings.access_key")); + assertNull(responseObjectPath.evaluate(repositoryName + ".settings.secret_key")); + assertNull(responseObjectPath.evaluate(repositoryName + ".settings.session_token")); + + if (readonly == null) { + assertNull(responseObjectPath.evaluate(repositoryName + ".settings." + BlobStoreRepository.READONLY_SETTING_KEY)); + } else { + assertEquals( + Boolean.toString(readonly), + responseObjectPath.evaluate(repositoryName + ".settings." + BlobStoreRepository.READONLY_SETTING_KEY) + ); + } + } + } + + public void testNonexistentBucket() throws Exception { + testNonexistentBucket(null); + } + + public void testNonexistentBucketReadonlyTrue() throws Exception { + testNonexistentBucket(Boolean.TRUE); + } + + public void testNonexistentBucketReadonlyFalse() throws Exception { + testNonexistentBucket(Boolean.FALSE); + } + + private void testNonexistentBucket(Boolean readonly) throws Exception { + final var repository = new TestRepository( + randomIdentifier(), + getClientName(), + randomValueOtherThan(getBucketName(), ESTestCase::randomIdentifier), + getBasePath() + ); + final var registerRequest = repository.getRegisterRequest(readonlyOperator(readonly)); + + final var responseException = expectThrows(ResponseException.class, () -> client().performRequest(registerRequest)); + assertEquals(RestStatus.INTERNAL_SERVER_ERROR.getStatus(), responseException.getResponse().getStatusLine().getStatusCode()); + final var responseObjectPath = ObjectPath.createFromResponse(responseException.getResponse()); + assertThat(responseObjectPath.evaluate("error.type"), equalTo("repository_verification_exception")); + assertThat(responseObjectPath.evaluate("error.reason"), containsString("is not accessible on master node")); + } + + public void testNonexistentClient() throws Exception { + testNonexistentClient(null); + } + + public void testNonexistentClientReadonlyTrue() throws Exception { + testNonexistentClient(Boolean.TRUE); + } + + public void testNonexistentClientReadonlyFalse() throws Exception { + testNonexistentClient(Boolean.FALSE); + } + + private void testNonexistentClient(Boolean readonly) throws Exception { + final var repository = new TestRepository( + randomIdentifier(), + randomValueOtherThanMany(c -> c.equals(getClientName()) || c.equals("default"), ESTestCase::randomIdentifier), + getBucketName(), + getBasePath() + ); + final var registerRequest = repository.getRegisterRequest(readonlyOperator(readonly)); + + final var responseException = expectThrows(ResponseException.class, () -> client().performRequest(registerRequest)); + assertEquals(RestStatus.INTERNAL_SERVER_ERROR.getStatus(), responseException.getResponse().getStatusLine().getStatusCode()); + final var responseObjectPath = ObjectPath.createFromResponse(responseException.getResponse()); + assertThat(responseObjectPath.evaluate("error.type"), equalTo("repository_verification_exception")); + assertThat(responseObjectPath.evaluate("error.reason"), containsString("is not accessible on master node")); + assertThat(responseObjectPath.evaluate("error.caused_by.type"), equalTo("illegal_argument_exception")); + assertThat(responseObjectPath.evaluate("error.caused_by.reason"), containsString("Unknown s3 client name")); + } + + public void testNonexistentSnapshot() throws Exception { + testNonexistentSnapshot(null); + } + + public void testNonexistentSnapshotReadonlyTrue() throws Exception { + testNonexistentSnapshot(Boolean.TRUE); + } + + public void testNonexistentSnapshotReadonlyFalse() throws Exception { + testNonexistentSnapshot(Boolean.FALSE); + } + + private void testNonexistentSnapshot(Boolean readonly) throws Exception { + final var repository = newTestRepository(); + try (var ignored = repository.register(readonlyOperator(readonly))) { + final var repositoryName = repository.repositoryName(); + + final var getSnapshotRequest = new Request("GET", "/_snapshot/" + repositoryName + "/" + randomIdentifier()); + final var getSnapshotException = expectThrows(ResponseException.class, () -> client().performRequest(getSnapshotRequest)); + assertEquals(RestStatus.NOT_FOUND.getStatus(), getSnapshotException.getResponse().getStatusLine().getStatusCode()); + final var getResponseObjectPath = ObjectPath.createFromResponse(getSnapshotException.getResponse()); + assertThat(getResponseObjectPath.evaluate("error.type"), equalTo("snapshot_missing_exception")); + + final var restoreRequest = new Request("POST", "/_snapshot/" + repositoryName + "/" + randomIdentifier() + "/_restore"); + if (randomBoolean()) { + restoreRequest.addParameter("wait_for_completion", Boolean.toString(randomBoolean())); + } + final var restoreException = expectThrows(ResponseException.class, () -> client().performRequest(restoreRequest)); + assertEquals(RestStatus.INTERNAL_SERVER_ERROR.getStatus(), restoreException.getResponse().getStatusLine().getStatusCode()); + final var restoreResponseObjectPath = ObjectPath.createFromResponse(restoreException.getResponse()); + assertThat(restoreResponseObjectPath.evaluate("error.type"), equalTo("snapshot_restore_exception")); + + if (readonly != Boolean.TRUE) { + final var deleteRequest = new Request("DELETE", "/_snapshot/" + repositoryName + "/" + randomIdentifier()); + final var deleteException = expectThrows(ResponseException.class, () -> client().performRequest(deleteRequest)); + assertEquals(RestStatus.NOT_FOUND.getStatus(), deleteException.getResponse().getStatusLine().getStatusCode()); + final var deleteResponseObjectPath = ObjectPath.createFromResponse(deleteException.getResponse()); + assertThat(deleteResponseObjectPath.evaluate("error.type"), equalTo("snapshot_missing_exception")); + } + } + } + + public void testUsageStats() throws Exception { + testUsageStats(null); + } + + public void testUsageStatsReadonlyTrue() throws Exception { + testUsageStats(Boolean.TRUE); + } + + public void testUsageStatsReadonlyFalse() throws Exception { + testUsageStats(Boolean.FALSE); + } + + private void testUsageStats(Boolean readonly) throws Exception { + final var repository = newTestRepository(); + try (var ignored = repository.register(readonlyOperator(readonly))) { + final var responseObjectPath = assertOKAndCreateObjectPath(client().performRequest(new Request("GET", "/_cluster/stats"))); + assertThat(responseObjectPath.evaluate("repositories.s3.count"), equalTo(1)); + + if (readonly == Boolean.TRUE) { + assertThat(responseObjectPath.evaluate("repositories.s3.read_only"), equalTo(1)); + assertNull(responseObjectPath.evaluate("repositories.s3.read_write")); + } else { + assertNull(responseObjectPath.evaluate("repositories.s3.read_only")); + assertThat(responseObjectPath.evaluate("repositories.s3.read_write"), equalTo(1)); + } + } + } + + public void testSnapshotAndRestore() throws Exception { + final var repository = newTestRepository(); + try (var ignored = repository.register()) { + final var repositoryName = repository.repositoryName(); + final var indexName = randomIdentifier(); + final var snapshotsToDelete = new ArrayList(2); + + try { + indexDocuments(indexName, """ + {"index":{"_id":"1"}} + {"snapshot":"one"} + {"index":{"_id":"2"}} + {"snapshot":"one"} + {"index":{"_id":"3"}} + {"snapshot":"one"} + """, 3); + + // create the first snapshot + final var snapshot1Name = randomIdentifier(); + createSnapshot(repositoryName, snapshotsToDelete, snapshot1Name); + + // check the first snapshot's status + { + final var snapshotStatusResponse = assertOKAndCreateObjectPath( + client().performRequest(new Request("GET", "/_snapshot/" + repositoryName + "/" + snapshot1Name + "/_status")) + ); + assertEquals(snapshot1Name, snapshotStatusResponse.evaluate("snapshots.0.snapshot")); + assertEquals("SUCCESS", snapshotStatusResponse.evaluate("snapshots.0.state")); + } + + // add more documents to the index + indexDocuments(indexName, """ + {"index":{"_id":"4"}} + {"snapshot":"one"} + {"index":{"_id":"5"}} + {"snapshot":"one"} + {"index":{"_id":"6"}} + {"snapshot":"one"} + {"index":{"_id":"7"}} + {"snapshot":"one"} + """, 7); + + // create the second snapshot + final var snapshot2Name = randomValueOtherThan(snapshot1Name, ESTestCase::randomIdentifier); + createSnapshot(repositoryName, snapshotsToDelete, snapshot2Name); + + // list the snapshots + { + final var listSnapshotsResponse = assertOKAndCreateObjectPath( + client().performRequest( + new Request("GET", "/_snapshot/" + repositoryName + "/" + snapshot1Name + "," + snapshot2Name) + ) + ); + assertEquals(2, listSnapshotsResponse.evaluateArraySize("snapshots")); + assertEquals( + Set.of(snapshot1Name, snapshot2Name), + Set.of( + listSnapshotsResponse.evaluate("snapshots.0.snapshot"), + listSnapshotsResponse.evaluate("snapshots.1.snapshot") + ) + ); + assertEquals("SUCCESS", listSnapshotsResponse.evaluate("snapshots.0.state")); + assertEquals("SUCCESS", listSnapshotsResponse.evaluate("snapshots.1.state")); + } + + // delete and restore the index from snapshot 2 + deleteAndRestoreIndex(indexName, repositoryName, snapshot2Name, 7); + + // delete and restore the index from snapshot 1 + deleteAndRestoreIndex(indexName, repositoryName, snapshot1Name, 3); + } finally { + if (snapshotsToDelete.isEmpty() == false) { + assertOK( + client().performRequest( + new Request( + "DELETE", + "/_snapshot/" + repositoryName + "/" + snapshotsToDelete.stream().collect(Collectors.joining(",")) + ) + ) + ); + } + } + } + } + + private static void deleteAndRestoreIndex(String indexName, String repositoryName, String snapshot2Name, int expectedDocCount) + throws IOException { + assertOK(client().performRequest(new Request("DELETE", "/" + indexName))); + final var restoreRequest = new Request("POST", "/_snapshot/" + repositoryName + "/" + snapshot2Name + "/_restore"); + restoreRequest.addParameter("wait_for_completion", "true"); + assertOK(client().performRequest(restoreRequest)); + assertIndexDocCount(indexName, expectedDocCount); + } + + private static void indexDocuments(String indexName, String body, int expectedDocCount) throws IOException { + // create and populate an index + final var indexDocsRequest = new Request("POST", "/" + indexName + "/_bulk"); + indexDocsRequest.addParameter("refresh", "true"); + indexDocsRequest.setJsonEntity(body); + assertFalse(assertOKAndCreateObjectPath(client().performRequest(indexDocsRequest)).evaluate("errors")); + + // check the index contents + assertIndexDocCount(indexName, expectedDocCount); + } + + private static void createSnapshot(String repositoryName, ArrayList snapshotsToDelete, String snapshotName) throws IOException { + final var createSnapshotRequest = new Request("POST", "/_snapshot/" + repositoryName + "/" + snapshotName); + createSnapshotRequest.addParameter("wait_for_completion", "true"); + final var createSnapshotResponse = assertOKAndCreateObjectPath(client().performRequest(createSnapshotRequest)); + snapshotsToDelete.add(snapshotName); + assertEquals(snapshotName, createSnapshotResponse.evaluate("snapshot.snapshot")); + assertEquals("SUCCESS", createSnapshotResponse.evaluate("snapshot.state")); + assertThat(createSnapshotResponse.evaluate("snapshot.shards.failed"), equalTo(0)); + } + + private static void assertIndexDocCount(String indexName, int expectedCount) throws IOException { + assertThat( + assertOKAndCreateObjectPath(client().performRequest(new Request("GET", "/" + indexName + "/_count"))).evaluate("count"), + equalTo(expectedCount) + ); + } +} diff --git a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3BasicCredentialsRestIT.java b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3BasicCredentialsRestIT.java new file mode 100644 index 0000000000000..10901424512a3 --- /dev/null +++ b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3BasicCredentialsRestIT.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.repositories.s3; + +import fixture.s3.S3HttpFixture; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +import static fixture.aws.AwsCredentialsUtils.fixedAccessKey; + +@ThreadLeakFilters(filters = { TestContainersThreadFilter.class }) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482 +public class RepositoryS3BasicCredentialsRestIT extends AbstractRepositoryS3RestTestCase { + + private static final String PREFIX = getIdentifierPrefix("RepositoryS3BasicCredentialsRestIT"); + private static final String BUCKET = PREFIX + "bucket"; + private static final String BASE_PATH = PREFIX + "base_path"; + private static final String ACCESS_KEY = PREFIX + "access-key"; + private static final String SECRET_KEY = PREFIX + "secret-key"; + private static final String CLIENT = "basic_credentials_client"; + + private static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, fixedAccessKey(ACCESS_KEY)); + + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .module("repository-s3") + .keystore("s3.client." + CLIENT + ".access_key", ACCESS_KEY) + .keystore("s3.client." + CLIENT + ".secret_key", SECRET_KEY) + .setting("s3.client." + CLIENT + ".endpoint", s3Fixture::getAddress) + .build(); + + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(s3Fixture).around(cluster); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @Override + protected String getBucketName() { + return BUCKET; + } + + @Override + protected String getBasePath() { + return BASE_PATH; + } + + @Override + protected String getClientName() { + return CLIENT; + } +} diff --git a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsCredentialsRestIT.java b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsCredentialsRestIT.java new file mode 100644 index 0000000000000..c525c2ea42f58 --- /dev/null +++ b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsCredentialsRestIT.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.repositories.s3; + +import fixture.aws.DynamicAwsCredentials; +import fixture.aws.imds.Ec2ImdsHttpFixture; +import fixture.aws.imds.Ec2ImdsServiceBuilder; +import fixture.aws.imds.Ec2ImdsVersion; +import fixture.s3.S3HttpFixture; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +import java.util.Set; + +@ThreadLeakFilters(filters = { TestContainersThreadFilter.class }) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482 +public class RepositoryS3EcsCredentialsRestIT extends AbstractRepositoryS3RestTestCase { + + private static final String PREFIX = getIdentifierPrefix("RepositoryS3EcsCredentialsRestIT"); + private static final String BUCKET = PREFIX + "bucket"; + private static final String BASE_PATH = PREFIX + "base_path"; + private static final String CLIENT = "ecs_credentials_client"; + + private static final DynamicAwsCredentials dynamicCredentials = new DynamicAwsCredentials(); + + private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture( + new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V1).newCredentialsConsumer(dynamicCredentials::addValidCredentials) + .alternativeCredentialsEndpoints(Set.of("/ecs_credentials_endpoint")) + ); + + private static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, dynamicCredentials::isAuthorized); + + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .module("repository-s3") + .setting("s3.client." + CLIENT + ".endpoint", s3Fixture::getAddress) + .environment("AWS_CONTAINER_CREDENTIALS_FULL_URI", () -> ec2ImdsHttpFixture.getAddress() + "/ecs_credentials_endpoint") + .build(); + + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(s3Fixture).around(ec2ImdsHttpFixture).around(cluster); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @Override + protected String getBucketName() { + return BUCKET; + } + + @Override + protected String getBasePath() { + return BASE_PATH; + } + + @Override + protected String getClientName() { + return CLIENT; + } +} diff --git a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ImdsV1CredentialsRestIT.java b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ImdsV1CredentialsRestIT.java new file mode 100644 index 0000000000000..bc689ea52ca32 --- /dev/null +++ b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ImdsV1CredentialsRestIT.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.repositories.s3; + +import fixture.aws.DynamicAwsCredentials; +import fixture.aws.imds.Ec2ImdsHttpFixture; +import fixture.aws.imds.Ec2ImdsServiceBuilder; +import fixture.aws.imds.Ec2ImdsVersion; +import fixture.s3.S3HttpFixture; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +@ThreadLeakFilters(filters = { TestContainersThreadFilter.class }) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482 +public class RepositoryS3ImdsV1CredentialsRestIT extends AbstractRepositoryS3RestTestCase { + + private static final String PREFIX = getIdentifierPrefix("RepositoryS3ImdsV1CredentialsRestIT"); + private static final String BUCKET = PREFIX + "bucket"; + private static final String BASE_PATH = PREFIX + "base_path"; + private static final String CLIENT = "imdsv1_credentials_client"; + + private static final DynamicAwsCredentials dynamicCredentials = new DynamicAwsCredentials(); + + private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture( + new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V1).newCredentialsConsumer(dynamicCredentials::addValidCredentials) + ); + + private static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, dynamicCredentials::isAuthorized); + + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .module("repository-s3") + .setting("s3.client." + CLIENT + ".endpoint", s3Fixture::getAddress) + .systemProperty(Ec2ImdsHttpFixture.ENDPOINT_OVERRIDE_SYSPROP_NAME, ec2ImdsHttpFixture::getAddress) + .build(); + + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(ec2ImdsHttpFixture).around(s3Fixture).around(cluster); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @Override + protected String getBucketName() { + return BUCKET; + } + + @Override + protected String getBasePath() { + return BASE_PATH; + } + + @Override + protected String getClientName() { + return CLIENT; + } +} diff --git a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ImdsV2CredentialsRestIT.java b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ImdsV2CredentialsRestIT.java new file mode 100644 index 0000000000000..dedf205d3def2 --- /dev/null +++ b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ImdsV2CredentialsRestIT.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.repositories.s3; + +import fixture.aws.DynamicAwsCredentials; +import fixture.aws.imds.Ec2ImdsHttpFixture; +import fixture.aws.imds.Ec2ImdsServiceBuilder; +import fixture.aws.imds.Ec2ImdsVersion; +import fixture.s3.S3HttpFixture; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +@ThreadLeakFilters(filters = { TestContainersThreadFilter.class }) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482 +public class RepositoryS3ImdsV2CredentialsRestIT extends AbstractRepositoryS3RestTestCase { + + private static final String PREFIX = getIdentifierPrefix("RepositoryS3ImdsV2CredentialsRestIT"); + private static final String BUCKET = PREFIX + "bucket"; + private static final String BASE_PATH = PREFIX + "base_path"; + private static final String CLIENT = "imdsv2_credentials_client"; + + private static final DynamicAwsCredentials dynamicCredentials = new DynamicAwsCredentials(); + + private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture( + new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V2).newCredentialsConsumer(dynamicCredentials::addValidCredentials) + ); + + private static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, dynamicCredentials::isAuthorized); + + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .module("repository-s3") + .setting("s3.client." + CLIENT + ".endpoint", s3Fixture::getAddress) + .systemProperty(Ec2ImdsHttpFixture.ENDPOINT_OVERRIDE_SYSPROP_NAME, ec2ImdsHttpFixture::getAddress) + .build(); + + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(ec2ImdsHttpFixture).around(s3Fixture).around(cluster); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @Override + protected String getBucketName() { + return BUCKET; + } + + @Override + protected String getBasePath() { + return BASE_PATH; + } + + @Override + protected String getClientName() { + return CLIENT; + } +} diff --git a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3MinioBasicCredentialsRestIT.java b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3MinioBasicCredentialsRestIT.java new file mode 100644 index 0000000000000..93915e8491d5b --- /dev/null +++ b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3MinioBasicCredentialsRestIT.java @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.repositories.s3; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.fixtures.minio.MinioTestContainer; +import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +@ThreadLeakFilters(filters = { TestContainersThreadFilter.class }) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482 +public class RepositoryS3MinioBasicCredentialsRestIT extends AbstractRepositoryS3RestTestCase { + + private static final String PREFIX = getIdentifierPrefix("RepositoryS3MinioBasicCredentialsRestIT"); + private static final String BUCKET = PREFIX + "bucket"; + private static final String BASE_PATH = PREFIX + "base_path"; + private static final String ACCESS_KEY = PREFIX + "access-key"; + private static final String SECRET_KEY = PREFIX + "secret-key"; + private static final String CLIENT = "minio_client"; + + private static final MinioTestContainer minioFixture = new MinioTestContainer(true, ACCESS_KEY, SECRET_KEY, BUCKET); + + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .module("repository-s3") + .keystore("s3.client." + CLIENT + ".access_key", ACCESS_KEY) + .keystore("s3.client." + CLIENT + ".secret_key", SECRET_KEY) + .setting("s3.client." + CLIENT + ".endpoint", minioFixture::getAddress) + .build(); + + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(minioFixture).around(cluster); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @Override + protected String getBucketName() { + return BUCKET; + } + + @Override + protected String getBasePath() { + return BASE_PATH; + } + + @Override + protected String getClientName() { + return CLIENT; + } +} diff --git a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3RestReloadCredentialsIT.java b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3RestReloadCredentialsIT.java new file mode 100644 index 0000000000000..065d1c6c9ea27 --- /dev/null +++ b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3RestReloadCredentialsIT.java @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.repositories.s3; + +import fixture.s3.S3HttpFixture; +import io.netty.handler.codec.http.HttpMethod; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.cluster.routing.Murmur3HashFunction; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.MutableSettingsProvider; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +import java.io.IOException; + +import static fixture.aws.AwsCredentialsUtils.mutableAccessKey; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.equalTo; + +public class RepositoryS3RestReloadCredentialsIT extends ESRestTestCase { + + private static final String HASHED_SEED = Integer.toString(Murmur3HashFunction.hash(System.getProperty("tests.seed"))); + private static final String BUCKET = "RepositoryS3RestReloadCredentialsIT-bucket-" + HASHED_SEED; + private static final String BASE_PATH = "RepositoryS3RestReloadCredentialsIT-base-path-" + HASHED_SEED; + + private static volatile String repositoryAccessKey; + + public static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, mutableAccessKey(() -> repositoryAccessKey)); + + private static final MutableSettingsProvider keystoreSettings = new MutableSettingsProvider(); + + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .module("repository-s3") + .keystore(keystoreSettings) + .setting("s3.client.default.endpoint", s3Fixture::getAddress) + .build(); + + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(s3Fixture).around(cluster); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + public void testReloadCredentialsFromKeystore() throws IOException { + // Register repository (?verify=false because we don't have access to the blob store yet) + final var repositoryName = randomIdentifier(); + registerRepository( + repositoryName, + S3Repository.TYPE, + false, + Settings.builder().put("bucket", BUCKET).put("base_path", BASE_PATH).build() + ); + final var verifyRequest = new Request("POST", "/_snapshot/" + repositoryName + "/_verify"); + + // Set up initial credentials + final var accessKey1 = randomIdentifier(); + repositoryAccessKey = accessKey1; + keystoreSettings.put("s3.client.default.access_key", accessKey1); + keystoreSettings.put("s3.client.default.secret_key", randomSecretKey()); + cluster.updateStoredSecureSettings(); + + assertOK(client().performRequest(createReloadSecureSettingsRequest())); + + // Check access using initial credentials + assertOK(client().performRequest(verifyRequest)); + + // Rotate credentials in blob store + final var accessKey2 = randomValueOtherThan(accessKey1, ESTestCase::randomSecretKey); + repositoryAccessKey = accessKey2; + + // Ensure that initial credentials now invalid + final var accessDeniedException2 = expectThrows(ResponseException.class, () -> client().performRequest(verifyRequest)); + assertThat(accessDeniedException2.getResponse().getStatusLine().getStatusCode(), equalTo(500)); + assertThat( + accessDeniedException2.getMessage(), + allOf(containsString("Access denied"), containsString("Status Code: 403"), containsString("Error Code: AccessDenied")) + ); + + // Set up refreshed credentials + keystoreSettings.put("s3.client.default.access_key", accessKey2); + cluster.updateStoredSecureSettings(); + assertOK(client().performRequest(createReloadSecureSettingsRequest())); + + // Check access using refreshed credentials + assertOK(client().performRequest(verifyRequest)); + } + + private Request createReloadSecureSettingsRequest() throws IOException { + return newXContentRequest( + HttpMethod.POST, + "/_nodes/reload_secure_settings", + (b, p) -> inFipsJvm() ? b.field("secure_settings_password", "keystore-password") : b + ); + } +} diff --git a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3SessionCredentialsRestIT.java b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3SessionCredentialsRestIT.java new file mode 100644 index 0000000000000..a8009d594926f --- /dev/null +++ b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3SessionCredentialsRestIT.java @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.repositories.s3; + +import fixture.s3.S3HttpFixture; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +import static fixture.aws.AwsCredentialsUtils.fixedAccessKeyAndToken; + +@ThreadLeakFilters(filters = { TestContainersThreadFilter.class }) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482 +public class RepositoryS3SessionCredentialsRestIT extends AbstractRepositoryS3RestTestCase { + + private static final String PREFIX = getIdentifierPrefix("RepositoryS3SessionCredentialsRestIT"); + private static final String BUCKET = PREFIX + "bucket"; + private static final String BASE_PATH = PREFIX + "base_path"; + private static final String ACCESS_KEY = PREFIX + "access-key"; + private static final String SECRET_KEY = PREFIX + "secret-key"; + private static final String SESSION_TOKEN = PREFIX + "session-token"; + private static final String CLIENT = "session_credentials_client"; + + private static final S3HttpFixture s3Fixture = new S3HttpFixture( + true, + BUCKET, + BASE_PATH, + fixedAccessKeyAndToken(ACCESS_KEY, SESSION_TOKEN) + ); + + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .module("repository-s3") + .keystore("s3.client." + CLIENT + ".access_key", ACCESS_KEY) + .keystore("s3.client." + CLIENT + ".secret_key", SECRET_KEY) + .keystore("s3.client." + CLIENT + ".session_token", SESSION_TOKEN) + .setting("s3.client." + CLIENT + ".endpoint", s3Fixture::getAddress) + .build(); + + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(s3Fixture).around(cluster); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @Override + protected String getBucketName() { + return BUCKET; + } + + @Override + protected String getBasePath() { + return BASE_PATH; + } + + @Override + protected String getClientName() { + return CLIENT; + } +} diff --git a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3StsCredentialsRestIT.java b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3StsCredentialsRestIT.java new file mode 100644 index 0000000000000..61f0acdb16154 --- /dev/null +++ b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3StsCredentialsRestIT.java @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.repositories.s3; + +import fixture.aws.DynamicAwsCredentials; +import fixture.aws.sts.AwsStsHttpFixture; +import fixture.s3.S3HttpFixture; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.util.resource.Resource; +import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +@ThreadLeakFilters(filters = { TestContainersThreadFilter.class }) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482 +public class RepositoryS3StsCredentialsRestIT extends AbstractRepositoryS3RestTestCase { + + private static final String PREFIX = getIdentifierPrefix("RepositoryS3StsCredentialsRestIT"); + private static final String BUCKET = PREFIX + "bucket"; + private static final String BASE_PATH = PREFIX + "base_path"; + private static final String CLIENT = "sts_credentials_client"; + + private static final DynamicAwsCredentials dynamicCredentials = new DynamicAwsCredentials(); + + private static final S3HttpFixture s3HttpFixture = new S3HttpFixture(true, BUCKET, BASE_PATH, dynamicCredentials::isAuthorized); + + private static final String WEB_IDENTITY_TOKEN_FILE_CONTENTS = """ + Atza|IQEBLjAsAhRFiXuWpUXuRvQ9PZL3GMFcYevydwIUFAHZwXZXXXXXXXXJnrulxKDHwy87oGKPznh0D6bEQZTSCzyoCtL_8S07pLpr0zMbn6w1lfVZKNTBdDans\ + FBmtGnIsIapjI6xKR02Yc_2bQ8LZbUXSGm6Ry6_BG7PrtLZtj_dfCTj92xNGed-CrKqjG7nPBjNIL016GGvuS5gSvPRUxWES3VYfm1wl7WTI7jn-Pcb6M-buCgHhFO\ + zTQxod27L9CqnOLio7N3gZAGpsp6n1-AJBOCJckcyXe2c6uD0srOJeZlKUm2eTDVMf8IehDVI0r1QOnTV6KzzAI3OY87Vd_cVMQ"""; + + private static final AwsStsHttpFixture stsHttpFixture = new AwsStsHttpFixture( + dynamicCredentials::addValidCredentials, + WEB_IDENTITY_TOKEN_FILE_CONTENTS + ); + + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .module("repository-s3") + .setting("s3.client." + CLIENT + ".endpoint", s3HttpFixture::getAddress) + .systemProperty( + "com.amazonaws.sdk.stsMetadataServiceEndpointOverride", + () -> stsHttpFixture.getAddress() + "/assume-role-with-web-identity" + ) + .configFile( + S3Service.CustomWebIdentityTokenCredentialsProvider.WEB_IDENTITY_TOKEN_FILE_LOCATION, + Resource.fromString(WEB_IDENTITY_TOKEN_FILE_CONTENTS) + ) + .environment("AWS_WEB_IDENTITY_TOKEN_FILE", S3Service.CustomWebIdentityTokenCredentialsProvider.WEB_IDENTITY_TOKEN_FILE_LOCATION) + // The AWS STS SDK requires the role and session names to be set. We can verify that they are sent to S3S in the + // S3HttpFixtureWithSTS fixture + .environment("AWS_ROLE_ARN", "arn:aws:iam::123456789012:role/FederatedWebIdentityRole") + .environment("AWS_ROLE_SESSION_NAME", "sts-fixture-test") + .build(); + + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(s3HttpFixture).around(stsHttpFixture).around(cluster); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @Override + protected String getBucketName() { + return BUCKET; + } + + @Override + protected String getBasePath() { + return BASE_PATH; + } + + @Override + protected String getClientName() { + return CLIENT; + } +} diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/AmazonS3Reference.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/AmazonS3Reference.java index 684ff58c315e8..e098422ab8b98 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/AmazonS3Reference.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/AmazonS3Reference.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.s3; diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BasicCredentials.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BasicCredentials.java index f3f2af79d6c5e..ab9ba64d1fa69 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BasicCredentials.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BasicCredentials.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.s3; diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BasicSessionCredentials.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BasicSessionCredentials.java index 2123fdc7168c0..0dee56938c408 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BasicSessionCredentials.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BasicSessionCredentials.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.s3; diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java index cf3e73df2aee2..13eb2d4dec13c 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.s3; @@ -328,10 +329,10 @@ public DeleteResult delete(OperationPurpose purpose) throws IOException { return summary.getKey(); }); if (list.isTruncated()) { - blobStore.deleteBlobsIgnoringIfNotExists(purpose, blobNameIterator); + blobStore.deleteBlobs(purpose, blobNameIterator); prevListing = list; } else { - blobStore.deleteBlobsIgnoringIfNotExists(purpose, Iterators.concat(blobNameIterator, Iterators.single(keyPath))); + blobStore.deleteBlobs(purpose, Iterators.concat(blobNameIterator, Iterators.single(keyPath))); break; } } @@ -343,7 +344,7 @@ public DeleteResult delete(OperationPurpose purpose) throws IOException { @Override public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobNames) throws IOException { - blobStore.deleteBlobsIgnoringIfNotExists(purpose, Iterators.map(blobNames, this::buildKey)); + blobStore.deleteBlobs(purpose, Iterators.map(blobNames, this::buildKey)); } @Override @@ -881,8 +882,13 @@ public void compareAndExchangeRegister( final var clientReference = blobStore.clientReference(); ActionListener.run(ActionListener.releaseAfter(listener.delegateResponse((delegate, e) -> { logger.trace(() -> Strings.format("[%s]: compareAndExchangeRegister failed", key), e); - if (e instanceof AmazonS3Exception amazonS3Exception && amazonS3Exception.getStatusCode() == 404) { - // an uncaught 404 means that our multipart upload was aborted by a concurrent operation before we could complete it + if (e instanceof AmazonS3Exception amazonS3Exception + && (amazonS3Exception.getStatusCode() == 404 + || amazonS3Exception.getStatusCode() == 0 && "NoSuchUpload".equals(amazonS3Exception.getErrorCode()))) { + // An uncaught 404 means that our multipart upload was aborted by a concurrent operation before we could complete it. + // Also (rarely) S3 can start processing the request during a concurrent abort and this can result in a 200 OK with an + // NoSuchUpload... in the response, which the SDK translates to status code 0. Either way, this means + // that our write encountered contention: delegate.onResponse(OptionalBytesReference.MISSING); } else { delegate.onFailure(e); @@ -981,7 +987,7 @@ public void onResponse(Void unused) { // should be no other processes interacting with the repository. logger.warn( Strings.format( - "failed to clean up multipart upload [{}] of blob [{}][{}][{}]", + "failed to clean up multipart upload [%s] of blob [%s][%s][%s]", abortMultipartUploadRequest.getUploadId(), blobStore.getRepositoryMetadata().name(), abortMultipartUploadRequest.getBucketName(), diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java index 03605d50750f0..5237031fcdd6c 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.s3; @@ -331,8 +332,7 @@ public BlobContainer blobContainer(BlobPath path) { return new S3BlobContainer(path, this); } - @Override - public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobNames) throws IOException { + void deleteBlobs(OperationPurpose purpose, Iterator blobNames) throws IOException { if (blobNames.hasNext() == false) { return; } @@ -395,7 +395,7 @@ private static DeleteObjectsRequest bulkDelete(OperationPurpose purpose, S3BlobS @Override public void close() throws IOException { - this.service.close(); + service.onBlobStoreClose(); } @Override diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java index 3d36a08473b57..6eb253bf5074c 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.s3; diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoriesMetrics.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoriesMetrics.java index e025214998d5b..74682ca190a0c 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoriesMetrics.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoriesMetrics.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.s3; diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index d75a3e8ad433e..e7687b2d6774f 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.s3; @@ -140,6 +141,11 @@ class S3Repository extends MeteredBlobStoreRepository { MAX_FILE_SIZE_USING_MULTIPART ); + /** + * Maximum parts number for multipart upload. (see https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html) + */ + static final Setting MAX_MULTIPART_PARTS = Setting.intSetting("max_multipart_parts", 10_000, 1, 10_000); + /** * Sets the S3 storage class type for the backup files. Values may be standard, reduced_redundancy, * standard_ia, onezone_ia and intelligent_tiering. Defaults to standard. @@ -253,7 +259,9 @@ class S3Repository extends MeteredBlobStoreRepository { } this.bufferSize = BUFFER_SIZE_SETTING.get(metadata.settings()); - this.chunkSize = CHUNK_SIZE_SETTING.get(metadata.settings()); + var maxChunkSize = CHUNK_SIZE_SETTING.get(metadata.settings()); + var maxPartsNum = MAX_MULTIPART_PARTS.get(metadata.settings()); + this.chunkSize = objectSizeLimit(maxChunkSize, bufferSize, maxPartsNum); // We make sure that chunkSize is bigger or equal than/to bufferSize if (this.chunkSize.getBytes() < bufferSize.getBytes()) { @@ -280,8 +288,7 @@ class S3Repository extends MeteredBlobStoreRepository { deprecationLogger.critical( DeprecationCategory.SECURITY, "s3_repository_secret_settings", - "Using s3 access/secret key from repository settings. Instead " - + "store these in named clients and the elasticsearch keystore for secure settings." + INSECURE_CREDENTIALS_DEPRECATION_WARNING ); } @@ -298,10 +305,29 @@ class S3Repository extends MeteredBlobStoreRepository { ); } + static final String INSECURE_CREDENTIALS_DEPRECATION_WARNING = Strings.format(""" + This repository's settings include a S3 access key and secret key, but repository settings are stored in plaintext and must not be \ + used for security-sensitive information. Instead, store all secure settings in the keystore. See [%s] for more information.\ + """, ReferenceDocs.SECURE_SETTINGS); + private static Map buildLocation(RepositoryMetadata metadata) { return Map.of("base_path", BASE_PATH_SETTING.get(metadata.settings()), "bucket", BUCKET_SETTING.get(metadata.settings())); } + /** + * Calculates S3 object size limit based on 2 constraints: maximum object(chunk) size + * and maximum number of parts for multipart upload. + * https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html + * + * @param chunkSize s3 object size + * @param bufferSize s3 multipart upload part size + * @param maxPartsNum s3 multipart upload max parts number + */ + private static ByteSizeValue objectSizeLimit(ByteSizeValue chunkSize, ByteSizeValue bufferSize, int maxPartsNum) { + var bytes = Math.min(chunkSize.getBytes(), bufferSize.getBytes() * maxPartsNum); + return ByteSizeValue.ofBytes(bytes); + } + /** * Holds a reference to delayed repository operation {@link Scheduler.Cancellable} so it can be cancelled should the repository be * closed concurrently. diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java index 1092b76001cb3..37b960b33eb79 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.s3; diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RetryingInputStream.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RetryingInputStream.java index df5f3a2c93d28..da357dc09ab95 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RetryingInputStream.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RetryingInputStream.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.s3; diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java index 4a1308fa59419..7a2fd5c506c08 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.s3; @@ -291,6 +292,10 @@ private synchronized void releaseCachedClients() { IdleConnectionReaper.shutdown(); } + public void onBlobStoreClose() { + releaseCachedClients(); + } + @Override public void close() throws IOException { releaseCachedClients(); @@ -333,6 +338,8 @@ static class CustomWebIdentityTokenCredentialsProvider implements AWSCredentials private static final String STS_HOSTNAME = "https://sts.amazonaws.com"; + static final String WEB_IDENTITY_TOKEN_FILE_LOCATION = "repository-s3/aws-web-identity-token-file"; + private STSAssumeRoleWithWebIdentitySessionCredentialsProvider credentialsProvider; private AWSSecurityTokenService stsClient; private String stsRegion; @@ -351,7 +358,7 @@ static class CustomWebIdentityTokenCredentialsProvider implements AWSCredentials } // Make sure that a readable symlink to the token file exists in the plugin config directory // AWS_WEB_IDENTITY_TOKEN_FILE exists but we only use Web Identity Tokens if a corresponding symlink exists and is readable - Path webIdentityTokenFileSymlink = environment.configFile().resolve("repository-s3/aws-web-identity-token-file"); + Path webIdentityTokenFileSymlink = environment.configFile().resolve(WEB_IDENTITY_TOKEN_FILE_LOCATION); if (Files.exists(webIdentityTokenFileSymlink) == false) { LOGGER.warn( "Cannot use AWS Web Identity Tokens: AWS_WEB_IDENTITY_TOKEN_FILE is defined but no corresponding symlink exists " diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/SocketAccess.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/SocketAccess.java index 6b2ee323477c3..e097ed2df02a8 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/SocketAccess.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/SocketAccess.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.s3; diff --git a/modules/repository-s3/src/main/plugin-metadata/plugin-security.policy b/modules/repository-s3/src/main/plugin-metadata/plugin-security.policy index c071b62b2164e..9c8495aa9423d 100644 --- a/modules/repository-s3/src/main/plugin-metadata/plugin-security.policy +++ b/modules/repository-s3/src/main/plugin-metadata/plugin-security.policy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ grant { diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Wrapper.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Wrapper.java index 2e029ac3eab66..17b56131938d8 100644 --- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Wrapper.java +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Wrapper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.s3; diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java index ac97346f717a7..0aac0ba898f97 100644 --- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.s3; diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/CustomWebIdentityTokenCredentialsProviderTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/CustomWebIdentityTokenCredentialsProviderTests.java index fb775ab31c04d..69fd0c0f5d6a7 100644 --- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/CustomWebIdentityTokenCredentialsProviderTests.java +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/CustomWebIdentityTokenCredentialsProviderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.s3; diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java index e59b3e8f90620..8e5f6634372db 100644 --- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.s3; @@ -106,10 +107,9 @@ public void testRepositoryCredentialsOverrideSecureCredentials() { assertThat(credentials.getAWSSecretKey(), is("insecure_aws_secret")); assertCriticalWarnings( + "[access_key] setting was deprecated in Elasticsearch and will be removed in a future release.", "[secret_key] setting was deprecated in Elasticsearch and will be removed in a future release.", - "Using s3 access/secret key from repository settings. Instead store these in named clients and" - + " the elasticsearch keystore for secure settings.", - "[access_key] setting was deprecated in Elasticsearch and will be removed in a future release." + S3Repository.INSECURE_CREDENTIALS_DEPRECATION_WARNING ); } @@ -193,10 +193,9 @@ public void testReinitSecureCredentials() { if (hasInsecureSettings) { assertCriticalWarnings( + "[access_key] setting was deprecated in Elasticsearch and will be removed in a future release.", "[secret_key] setting was deprecated in Elasticsearch and will be removed in a future release.", - "Using s3 access/secret key from repository settings. Instead store these in named clients and" - + " the elasticsearch keystore for secure settings.", - "[access_key] setting was deprecated in Elasticsearch and will be removed in a future release." + S3Repository.INSECURE_CREDENTIALS_DEPRECATION_WARNING ); } } @@ -237,10 +236,7 @@ public void sendResponse(RestResponse response) { throw error.get(); } - assertWarnings( - "Using s3 access/secret key from repository settings. Instead store these in named clients and" - + " the elasticsearch keystore for secure settings." - ); + assertWarnings(S3Repository.INSECURE_CREDENTIALS_DEPRECATION_WARNING); } private void createRepository(final String name, final Settings repositorySettings) { diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java index 52cc48ddac589..35359a116bccc 100644 --- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.s3; @@ -572,16 +573,16 @@ public void handle(HttpExchange exchange) throws IOException { ), -1 ); + exchange.getResponseBody().flush(); } else if (randomBoolean()) { final var bytesSent = sendIncompleteContent(exchange, bytes); if (bytesSent < meaningfulProgressBytes) { failuresWithoutProgress += 1; - } else { - exchange.getResponseBody().flush(); } } else { failuresWithoutProgress += 1; } + exchange.getResponseBody().flush(); exchange.close(); } } @@ -626,6 +627,7 @@ public void handle(HttpExchange exchange) throws IOException { failureCount += 1; Streams.readFully(exchange.getRequestBody()); sendIncompleteContent(exchange, bytes); + exchange.getResponseBody().flush(); exchange.close(); } } diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreContainerTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreContainerTests.java index fbbcfa475da44..f52b3f4b53a62 100644 --- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreContainerTests.java +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreContainerTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.s3; diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java index c8aaa2a5e0a4b..ddc7a1851c663 100644 --- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.s3; diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java index 4bbc791e5fe21..3817af4def888 100644 --- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.s3; @@ -174,4 +175,37 @@ public void testAnalysisFailureDetail() { } } + // ensures that chunkSize is limited to chunk_size setting, when buffer_size * parts_num is bigger + public void testChunkSizeLimit() { + var meta = new RepositoryMetadata( + "dummy-repo", + "mock", + Settings.builder() + .put(S3Repository.BUCKET_SETTING.getKey(), "bucket") + .put(S3Repository.CHUNK_SIZE_SETTING.getKey(), "1GB") + .put(S3Repository.BUFFER_SIZE_SETTING.getKey(), "100MB") + .put(S3Repository.MAX_MULTIPART_PARTS.getKey(), 10_000) // ~1TB + .build() + ); + try (var repo = createS3Repo(meta)) { + assertEquals(ByteSizeValue.ofGb(1), repo.chunkSize()); + } + } + + // ensures that chunkSize is limited to buffer_size * parts_num, when chunk_size setting is bigger + public void testPartsNumLimit() { + var meta = new RepositoryMetadata( + "dummy-repo", + "mock", + Settings.builder() + .put(S3Repository.BUCKET_SETTING.getKey(), "bucket") + .put(S3Repository.CHUNK_SIZE_SETTING.getKey(), "5TB") + .put(S3Repository.BUFFER_SIZE_SETTING.getKey(), "100MB") + .put(S3Repository.MAX_MULTIPART_PARTS.getKey(), 10_000) + .build() + ); + try (var repo = createS3Repo(meta)) { + assertEquals(ByteSizeValue.ofMb(1_000_000), repo.chunkSize()); + } + } } diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RetryingInputStreamTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RetryingInputStreamTests.java index fbfdea8109485..c49d6e2a689bf 100644 --- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RetryingInputStreamTests.java +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RetryingInputStreamTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.s3; diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ServiceTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ServiceTests.java index 33e56bcf2180b..7bfaf56127fc7 100644 --- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ServiceTests.java +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ServiceTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.s3; diff --git a/modules/repository-s3/src/test/resources/aws-web-identity-token-file b/modules/repository-s3/src/test/resources/aws-web-identity-token-file deleted file mode 100644 index 15cb29eac2ff6..0000000000000 --- a/modules/repository-s3/src/test/resources/aws-web-identity-token-file +++ /dev/null @@ -1 +0,0 @@ -Atza|IQEBLjAsAhRFiXuWpUXuRvQ9PZL3GMFcYevydwIUFAHZwXZXXXXXXXXJnrulxKDHwy87oGKPznh0D6bEQZTSCzyoCtL_8S07pLpr0zMbn6w1lfVZKNTBdDansFBmtGnIsIapjI6xKR02Yc_2bQ8LZbUXSGm6Ry6_BG7PrtLZtj_dfCTj92xNGed-CrKqjG7nPBjNIL016GGvuS5gSvPRUxWES3VYfm1wl7WTI7jn-Pcb6M-buCgHhFOzTQxod27L9CqnOLio7N3gZAGpsp6n1-AJBOCJckcyXe2c6uD0srOJeZlKUm2eTDVMf8IehDVI0r1QOnTV6KzzAI3OY87Vd_cVMQ diff --git a/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/AbstractRepositoryS3ClientYamlTestSuiteIT.java b/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/AbstractRepositoryS3ClientYamlTestSuiteIT.java index ecf6709a2fcef..f7d58d9445fe3 100644 --- a/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/AbstractRepositoryS3ClientYamlTestSuiteIT.java +++ b/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/AbstractRepositoryS3ClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.s3; diff --git a/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java b/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java index 2f2f42974f131..89919f0f3ddf1 100644 --- a/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java +++ b/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java @@ -1,16 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.s3; import fixture.s3.S3HttpFixture; -import fixture.s3.S3HttpFixtureWithEC2; -import fixture.s3.S3HttpFixtureWithSessionToken; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; @@ -24,41 +23,35 @@ import org.junit.rules.RuleChain; import org.junit.rules.TestRule; +import static fixture.aws.AwsCredentialsUtils.fixedAccessKey; + @ThreadLeakFilters(filters = { TestContainersThreadFilter.class }) @ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482 public class RepositoryS3ClientYamlTestSuiteIT extends AbstractRepositoryS3ClientYamlTestSuiteIT { - public static final S3HttpFixture s3Fixture = new S3HttpFixture(); - public static final S3HttpFixtureWithSessionToken s3HttpFixtureWithSessionToken = new S3HttpFixtureWithSessionToken(); - public static final S3HttpFixtureWithEC2 s3Ec2 = new S3HttpFixtureWithEC2(); + private static final String ACCESS_KEY = "RepositoryS3ClientYamlTestSuiteIT-access-key"; + private static final String SECRET_KEY = "RepositoryS3ClientYamlTestSuiteIT-secret-key"; - private static final String s3TemporarySessionToken = "session_token"; + private static final S3HttpFixture s3Fixture = new S3HttpFixture( + true, + "bucket", + "base_path_integration_tests", + fixedAccessKey(ACCESS_KEY) + ); public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .module("repository-s3") - .keystore("s3.client.integration_test_permanent.access_key", System.getProperty("s3PermanentAccessKey")) - .keystore("s3.client.integration_test_permanent.secret_key", System.getProperty("s3PermanentSecretKey")) - .keystore("s3.client.integration_test_temporary.access_key", System.getProperty("s3TemporaryAccessKey")) - .keystore("s3.client.integration_test_temporary.secret_key", System.getProperty("s3TemporarySecretKey")) - .keystore("s3.client.integration_test_temporary.session_token", s3TemporarySessionToken) + .keystore("s3.client.integration_test_permanent.access_key", ACCESS_KEY) + .keystore("s3.client.integration_test_permanent.secret_key", SECRET_KEY) .setting("s3.client.integration_test_permanent.endpoint", s3Fixture::getAddress) - .setting("s3.client.integration_test_temporary.endpoint", s3HttpFixtureWithSessionToken::getAddress) - .setting("s3.client.integration_test_ec2.endpoint", s3Ec2::getAddress) - .systemProperty("com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", s3Ec2::getAddress) .build(); @ClassRule - public static TestRule ruleChain = RuleChain.outerRule(s3Fixture).around(s3Ec2).around(s3HttpFixtureWithSessionToken).around(cluster); + public static TestRule ruleChain = RuleChain.outerRule(s3Fixture).around(cluster); @ParametersFactory public static Iterable parameters() throws Exception { - return createParameters( - new String[] { - "repository_s3/10_basic", - "repository_s3/20_repository_permanent_credentials", - "repository_s3/30_repository_temporary_credentials", - "repository_s3/40_repository_ec2_credentials" } - ); + return createParameters(new String[] { "repository_s3/10_basic", "repository_s3/20_repository_permanent_credentials" }); } public RepositoryS3ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { diff --git a/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsClientYamlTestSuiteIT.java b/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsClientYamlTestSuiteIT.java deleted file mode 100644 index e9bc9d0537cbb..0000000000000 --- a/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsClientYamlTestSuiteIT.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.repositories.s3; - -import fixture.s3.S3HttpFixtureWithECS; - -import com.carrotsearch.randomizedtesting.annotations.Name; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - -import org.elasticsearch.test.cluster.ElasticsearchCluster; -import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; -import org.junit.ClassRule; -import org.junit.rules.RuleChain; -import org.junit.rules.TestRule; - -public class RepositoryS3EcsClientYamlTestSuiteIT extends AbstractRepositoryS3ClientYamlTestSuiteIT { - private static final S3HttpFixtureWithECS s3Ecs = new S3HttpFixtureWithECS(); - - public static ElasticsearchCluster cluster = ElasticsearchCluster.local() - .module("repository-s3") - .setting("s3.client.integration_test_ecs.endpoint", s3Ecs::getAddress) - .environment("AWS_CONTAINER_CREDENTIALS_FULL_URI", () -> (s3Ecs.getAddress() + "/ecs_credentials_endpoint")) - .build(); - - @ClassRule - public static TestRule ruleChain = RuleChain.outerRule(s3Ecs).around(cluster); - - @ParametersFactory - public static Iterable parameters() throws Exception { - return createParameters(new String[] { "repository_s3/50_repository_ecs_credentials" }); - } - - public RepositoryS3EcsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { - super(testCandidate); - } - - @Override - protected String getTestRestCluster() { - return cluster.getHttpAddresses(); - } -} diff --git a/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3MinioClientYamlTestSuiteIT.java b/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3MinioClientYamlTestSuiteIT.java deleted file mode 100644 index 41f9983ef26e6..0000000000000 --- a/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3MinioClientYamlTestSuiteIT.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.repositories.s3; - -import com.carrotsearch.randomizedtesting.annotations.Name; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; - -import org.elasticsearch.test.cluster.ElasticsearchCluster; -import org.elasticsearch.test.fixtures.minio.MinioTestContainer; -import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter; -import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; -import org.junit.ClassRule; -import org.junit.rules.RuleChain; -import org.junit.rules.TestRule; - -@ThreadLeakFilters(filters = { TestContainersThreadFilter.class }) -public class RepositoryS3MinioClientYamlTestSuiteIT extends AbstractRepositoryS3ClientYamlTestSuiteIT { - - public static MinioTestContainer minio = new MinioTestContainer(); - - public static ElasticsearchCluster cluster = ElasticsearchCluster.local() - .module("repository-s3") - .keystore("s3.client.integration_test_permanent.access_key", System.getProperty("s3PermanentAccessKey")) - .keystore("s3.client.integration_test_permanent.secret_key", System.getProperty("s3PermanentSecretKey")) - .setting("s3.client.integration_test_permanent.endpoint", () -> minio.getAddress()) - .build(); - - @ClassRule - public static TestRule ruleChain = RuleChain.outerRule(minio).around(cluster); - - @ParametersFactory - public static Iterable parameters() throws Exception { - return createParameters(new String[] { "repository_s3/10_basic", "repository_s3/20_repository_permanent_credentials" }); - } - - public RepositoryS3MinioClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { - super(testCandidate); - } - - @Override - protected String getTestRestCluster() { - return cluster.getHttpAddresses(); - } -} diff --git a/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3RegionalStsClientYamlTestSuiteIT.java b/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3RegionalStsClientYamlTestSuiteIT.java index b0a7f84c03c85..ac356083983eb 100644 --- a/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3RegionalStsClientYamlTestSuiteIT.java +++ b/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3RegionalStsClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.s3; @@ -20,10 +21,11 @@ public class RepositoryS3RegionalStsClientYamlTestSuiteIT extends AbstractReposi @ClassRule public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .module("repository-s3") - .configFile("repository-s3/aws-web-identity-token-file", Resource.fromClasspath("aws-web-identity-token-file")) - .environment("AWS_WEB_IDENTITY_TOKEN_FILE", System.getProperty("awsWebIdentityTokenExternalLocation")) - // The AWS STS SDK requires the role and session names to be set. We can verify that they are sent to S3S in the - // S3HttpFixtureWithSTS fixture + .configFile(S3Service.CustomWebIdentityTokenCredentialsProvider.WEB_IDENTITY_TOKEN_FILE_LOCATION, Resource.fromString(""" + Atza|IQEBLjAsAhRFiXuWpUXuRvQ9PZL3GMFcYevydwIUFAHZwXZXXXXXXXXJnrulxKDHwy87oGKPznh0D6bEQZTSCzyoCtL_8S07pLpr0zMbn6w1lfVZKNTBdDans\ + FBmtGnIsIapjI6xKR02Yc_2bQ8LZbUXSGm6Ry6_BG7PrtLZtj_dfCTj92xNGed-CrKqjG7nPBjNIL016GGvuS5gSvPRUxWES3VYfm1wl7WTI7jn-Pcb6M-buCgHhFO\ + zTQxod27L9CqnOLio7N3gZAGpsp6n1-AJBOCJckcyXe2c6uD0srOJeZlKUm2eTDVMf8IehDVI0r1QOnTV6KzzAI3OY87Vd_cVMQ""")) + .environment("AWS_WEB_IDENTITY_TOKEN_FILE", S3Service.CustomWebIdentityTokenCredentialsProvider.WEB_IDENTITY_TOKEN_FILE_LOCATION) .environment("AWS_ROLE_ARN", "arn:aws:iam::123456789012:role/FederatedWebIdentityRole") .environment("AWS_ROLE_SESSION_NAME", "sts-fixture-test") .environment("AWS_STS_REGIONAL_ENDPOINTS", "regional") @@ -32,6 +34,9 @@ public class RepositoryS3RegionalStsClientYamlTestSuiteIT extends AbstractReposi @ParametersFactory public static Iterable parameters() throws Exception { + // Run just the basic sanity test to make sure ES starts up and loads the S3 repository with a regional endpoint without an error. + // It would be great to make actual requests against a test fixture, but setting the region means using a production endpoint. + // See #102230 for more details. return createParameters(new String[] { "repository_s3/10_basic" }); } diff --git a/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3StsClientYamlTestSuiteIT.java b/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3StsClientYamlTestSuiteIT.java deleted file mode 100644 index eb105e02353b6..0000000000000 --- a/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3StsClientYamlTestSuiteIT.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.repositories.s3; - -import fixture.s3.S3HttpFixture; -import fixture.s3.S3HttpFixtureWithSTS; - -import com.carrotsearch.randomizedtesting.annotations.Name; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - -import org.elasticsearch.test.cluster.ElasticsearchCluster; -import org.elasticsearch.test.cluster.util.resource.Resource; -import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; -import org.junit.ClassRule; -import org.junit.rules.RuleChain; -import org.junit.rules.TestRule; - -public class RepositoryS3StsClientYamlTestSuiteIT extends AbstractRepositoryS3ClientYamlTestSuiteIT { - - public static final S3HttpFixture s3Fixture = new S3HttpFixture(); - private static final S3HttpFixtureWithSTS s3Sts = new S3HttpFixtureWithSTS(); - - public static ElasticsearchCluster cluster = ElasticsearchCluster.local() - .module("repository-s3") - .setting("s3.client.integration_test_sts.endpoint", s3Sts::getAddress) - .systemProperty("com.amazonaws.sdk.stsMetadataServiceEndpointOverride", () -> s3Sts.getAddress() + "/assume-role-with-web-identity") - .configFile("repository-s3/aws-web-identity-token-file", Resource.fromClasspath("aws-web-identity-token-file")) - .environment("AWS_WEB_IDENTITY_TOKEN_FILE", System.getProperty("awsWebIdentityTokenExternalLocation")) - // // The AWS STS SDK requires the role and session names to be set. We can verify that they are sent to S3S in the - // // S3HttpFixtureWithSTS fixture - .environment("AWS_ROLE_ARN", "arn:aws:iam::123456789012:role/FederatedWebIdentityRole") - .environment("AWS_ROLE_SESSION_NAME", "sts-fixture-test") - .build(); - - @ClassRule - public static TestRule ruleChain = RuleChain.outerRule(s3Fixture).around(s3Sts).around(cluster); - - @ParametersFactory - public static Iterable parameters() throws Exception { - return createParameters(new String[] { "repository_s3/60_repository_sts_credentials" }); - } - - public RepositoryS3StsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { - super(testCandidate); - } - - @Override - protected String getTestRestCluster() { - return cluster.getHttpAddresses(); - } -} diff --git a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml b/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml index e88a0861ec01c..6f6fdaed8c666 100644 --- a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml +++ b/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml @@ -10,12 +10,11 @@ setup: body: type: s3 settings: - bucket: @permanent_bucket@ + bucket: bucket client: integration_test_permanent - base_path: "@permanent_base_path@" + base_path: base_path_integration_tests canned_acl: private storage_class: standard - disable_chunked_encoding: @disable_chunked_encoding@ # Remove the snapshots, if a previous test failed to delete them. This is # useful for third party tests that runs the test against a real external service. @@ -40,9 +39,9 @@ setup: body: type: s3 settings: - bucket: @permanent_bucket@ + bucket: bucket client: integration_test_permanent - base_path: "@permanent_base_path@" + base_path: base_path_integration_tests endpoint: 127.0.0.1:5 canned_acl: private storage_class: standard @@ -55,9 +54,9 @@ setup: body: type: s3 settings: - bucket: @permanent_bucket@ + bucket: bucket client: integration_test_permanent - base_path: "@permanent_base_path@" + base_path: base_path_integration_tests endpoint: 127.0.0.1:5 canned_acl: private storage_class: standard @@ -106,258 +105,6 @@ setup: - match: { snapshot.include_global_state: true } - match: { snapshot.shards.failed: 0 } ---- -"Snapshot and Restore with repository-s3 using permanent credentials": - - # Get repository - - do: - snapshot.get_repository: - repository: repository_permanent - - - match: { repository_permanent.settings.bucket : @permanent_bucket@ } - - match: { repository_permanent.settings.client : "integration_test_permanent" } - - match: { repository_permanent.settings.base_path : "@permanent_base_path@" } - - match: { repository_permanent.settings.canned_acl : "private" } - - match: { repository_permanent.settings.storage_class : "standard" } - - is_false: repository_permanent.settings.access_key - - is_false: repository_permanent.settings.secret_key - - is_false: repository_permanent.settings.session_token - - # Index documents - - do: - bulk: - refresh: true - body: - - index: - _index: docs - _id: "1" - - snapshot: one - - index: - _index: docs - _id: "2" - - snapshot: one - - index: - _index: docs - _id: "3" - - snapshot: one - - - do: - count: - index: docs - - - match: {count: 3} - - # Create a first snapshot - - do: - snapshot.create: - repository: repository_permanent - snapshot: snapshot-one - wait_for_completion: true - - - match: { snapshot.snapshot: snapshot-one } - - match: { snapshot.state : SUCCESS } - - match: { snapshot.include_global_state: true } - - match: { snapshot.shards.failed : 0 } - - - do: - snapshot.status: - repository: repository_permanent - snapshot: snapshot-one - - - is_true: snapshots - - match: { snapshots.0.snapshot: snapshot-one } - - match: { snapshots.0.state : SUCCESS } - - # Index more documents - - do: - bulk: - refresh: true - body: - - index: - _index: docs - _id: "4" - - snapshot: two - - index: - _index: docs - _id: "5" - - snapshot: two - - index: - _index: docs - _id: "6" - - snapshot: two - - index: - _index: docs - _id: "7" - - snapshot: two - - - do: - count: - index: docs - - - match: {count: 7} - - # Create a second snapshot - - do: - snapshot.create: - repository: repository_permanent - snapshot: snapshot-two - wait_for_completion: true - - - match: { snapshot.snapshot: snapshot-two } - - match: { snapshot.state : SUCCESS } - - match: { snapshot.shards.failed : 0 } - - - do: - snapshot.get: - repository: repository_permanent - snapshot: snapshot-one,snapshot-two - - - is_true: snapshots - - match: { snapshots.0.state : SUCCESS } - - match: { snapshots.1.state : SUCCESS } - - # Delete the index - - do: - indices.delete: - index: docs - - # Restore the second snapshot - - do: - snapshot.restore: - repository: repository_permanent - snapshot: snapshot-two - wait_for_completion: true - - - do: - count: - index: docs - - - match: {count: 7} - - # Delete the index again - - do: - indices.delete: - index: docs - - # Restore the first snapshot - - do: - snapshot.restore: - repository: repository_permanent - snapshot: snapshot-one - wait_for_completion: true - - - do: - count: - index: docs - - - match: {count: 3} - - # Remove the snapshots - - do: - snapshot.delete: - repository: repository_permanent - snapshot: snapshot-two - - - do: - snapshot.delete: - repository: repository_permanent - snapshot: snapshot-one - ---- -"Register a repository with a non existing bucket": - - - do: - catch: /repository_verification_exception/ - snapshot.create_repository: - repository: repository_permanent - body: - type: s3 - settings: - bucket: zHHkfSqlbnBsbpSgvCYtxrEfFLqghXtyPvvvKPNBnRCicNHQLE - client: integration_test_permanent - ---- -"Register a repository with a non existing client": - - - do: - catch: /illegal_argument_exception/ - snapshot.create_repository: - repository: repository_permanent - body: - type: s3 - settings: - bucket: repository_permanent - client: unknown - ---- -"Register a read-only repository with a non existing bucket": - -- do: - catch: /repository_verification_exception/ - snapshot.create_repository: - repository: repository_permanent - body: - type: s3 - settings: - readonly: true - bucket: zHHkfSqlbnBsbpSgvCYtxrEfFLqghXtyPvvvKPNBnRCicNHQLE - client: integration_test_permanent - ---- -"Register a read-only repository with a non existing client": - -- do: - catch: /illegal_argument_exception/ - snapshot.create_repository: - repository: repository_permanent - body: - type: s3 - settings: - readonly: true - bucket: repository_permanent - client: unknown - ---- -"Get a non existing snapshot": - - - do: - catch: /snapshot_missing_exception/ - snapshot.get: - repository: repository_permanent - snapshot: missing - ---- -"Delete a non existing snapshot": - - - do: - catch: /snapshot_missing_exception/ - snapshot.delete: - repository: repository_permanent - snapshot: missing - ---- -"Restore a non existing snapshot": - - - do: - catch: /snapshot_restore_exception/ - snapshot.restore: - repository: repository_permanent - snapshot: missing - wait_for_completion: true - ---- -"Usage stats": - - requires: - cluster_features: - - repositories.supports_usage_stats - reason: requires this feature - - - do: - cluster.stats: {} - - - gte: { repositories.s3.count: 1 } - - gte: { repositories.s3.read_write: 1 } - --- teardown: diff --git a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml b/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml deleted file mode 100644 index 501af980e17e3..0000000000000 --- a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml +++ /dev/null @@ -1,278 +0,0 @@ -# Integration tests for repository-s3 - ---- -setup: - - # Register repository with temporary credentials - - do: - snapshot.create_repository: - repository: repository_temporary - body: - type: s3 - settings: - bucket: @temporary_bucket@ - client: integration_test_temporary - base_path: "@temporary_base_path@" - canned_acl: private - storage_class: standard - disable_chunked_encoding: @disable_chunked_encoding@ - ---- -"Snapshot and Restore with repository-s3 using temporary credentials": - - # Get repository - - do: - snapshot.get_repository: - repository: repository_temporary - - - match: { repository_temporary.settings.bucket : @temporary_bucket@ } - - match: { repository_temporary.settings.client : "integration_test_temporary" } - - match: { repository_temporary.settings.base_path : "@temporary_base_path@" } - - match: { repository_temporary.settings.canned_acl : "private" } - - match: { repository_temporary.settings.storage_class : "standard" } - - is_false: repository_temporary.settings.access_key - - is_false: repository_temporary.settings.secret_key - - is_false: repository_temporary.settings.session_token - - # Index documents - - do: - bulk: - refresh: true - body: - - index: - _index: docs - _id: "1" - - snapshot: one - - index: - _index: docs - _id: "2" - - snapshot: one - - index: - _index: docs - _id: "3" - - snapshot: one - - - do: - count: - index: docs - - - match: {count: 3} - - # Create a first snapshot - - do: - snapshot.create: - repository: repository_temporary - snapshot: snapshot-one - wait_for_completion: true - - - match: { snapshot.snapshot: snapshot-one } - - match: { snapshot.state : SUCCESS } - - match: { snapshot.include_global_state: true } - - match: { snapshot.shards.failed : 0 } - - - do: - snapshot.status: - repository: repository_temporary - snapshot: snapshot-one - - - is_true: snapshots - - match: { snapshots.0.snapshot: snapshot-one } - - match: { snapshots.0.state : SUCCESS } - - # Index more documents - - do: - bulk: - refresh: true - body: - - index: - _index: docs - _id: "4" - - snapshot: two - - index: - _index: docs - _id: "5" - - snapshot: two - - index: - _index: docs - _id: "6" - - snapshot: two - - index: - _index: docs - _id: "7" - - snapshot: two - - - do: - count: - index: docs - - - match: {count: 7} - - # Create a second snapshot - - do: - snapshot.create: - repository: repository_temporary - snapshot: snapshot-two - wait_for_completion: true - - - match: { snapshot.snapshot: snapshot-two } - - match: { snapshot.state : SUCCESS } - - match: { snapshot.shards.failed : 0 } - - - do: - snapshot.get: - repository: repository_temporary - snapshot: snapshot-one,snapshot-two - - - is_true: snapshots - - match: { snapshots.0.state : SUCCESS } - - match: { snapshots.1.state : SUCCESS } - - # Delete the index - - do: - indices.delete: - index: docs - - # Restore the second snapshot - - do: - snapshot.restore: - repository: repository_temporary - snapshot: snapshot-two - wait_for_completion: true - - - do: - count: - index: docs - - - match: {count: 7} - - # Delete the index again - - do: - indices.delete: - index: docs - - # Restore the first snapshot - - do: - snapshot.restore: - repository: repository_temporary - snapshot: snapshot-one - wait_for_completion: true - - - do: - count: - index: docs - - - match: {count: 3} - - # Remove the snapshots - - do: - snapshot.delete: - repository: repository_temporary - snapshot: snapshot-two - - - do: - snapshot.delete: - repository: repository_temporary - snapshot: snapshot-one - ---- -"Register a repository with a non existing bucket": - - - do: - catch: /repository_verification_exception/ - snapshot.create_repository: - repository: repository_temporary - body: - type: s3 - settings: - bucket: zHHkfSqlbnBsbpSgvCYtxrEfFLqghXtyPvvvKPNBnRCicNHQLE - client: integration_test_temporary - ---- -"Register a repository with a non existing client": - - - do: - catch: /illegal_argument_exception/ - snapshot.create_repository: - repository: repository_temporary - body: - type: s3 - settings: - bucket: repository_temporary - client: unknown - ---- -"Register a read-only repository with a non existing bucket": - -- do: - catch: /repository_verification_exception/ - snapshot.create_repository: - repository: repository_temporary - body: - type: s3 - settings: - readonly: true - bucket: zHHkfSqlbnBsbpSgvCYtxrEfFLqghXtyPvvvKPNBnRCicNHQLE - client: integration_test_temporary - ---- -"Register a read-only repository with a non existing client": - -- do: - catch: /illegal_argument_exception/ - snapshot.create_repository: - repository: repository_temporary - body: - type: s3 - settings: - readonly: true - bucket: repository_temporary - client: unknown - ---- -"Get a non existing snapshot": - - - do: - catch: /snapshot_missing_exception/ - snapshot.get: - repository: repository_temporary - snapshot: missing - ---- -"Delete a non existing snapshot": - - - do: - catch: /snapshot_missing_exception/ - snapshot.delete: - repository: repository_temporary - snapshot: missing - ---- -"Restore a non existing snapshot": - - - do: - catch: /snapshot_restore_exception/ - snapshot.restore: - repository: repository_temporary - snapshot: missing - wait_for_completion: true - ---- -"Usage stats": - - requires: - cluster_features: - - repositories.supports_usage_stats - reason: requires this feature - - - do: - cluster.stats: {} - - - gte: { repositories.s3.count: 1 } - - gte: { repositories.s3.read_write: 1 } - ---- -teardown: - - # Remove our repository - - do: - snapshot.delete_repository: - repository: repository_temporary diff --git a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml b/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml deleted file mode 100644 index 129f0ba5d7588..0000000000000 --- a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml +++ /dev/null @@ -1,278 +0,0 @@ -# Integration tests for repository-s3 - ---- -setup: - - # Register repository with ec2 credentials - - do: - snapshot.create_repository: - repository: repository_ec2 - body: - type: s3 - settings: - bucket: @ec2_bucket@ - client: integration_test_ec2 - base_path: "@ec2_base_path@" - canned_acl: private - storage_class: standard - disable_chunked_encoding: @disable_chunked_encoding@ - ---- -"Snapshot and Restore with repository-s3 using ec2 credentials": - - # Get repository - - do: - snapshot.get_repository: - repository: repository_ec2 - - - match: { repository_ec2.settings.bucket : @ec2_bucket@ } - - match: { repository_ec2.settings.client : "integration_test_ec2" } - - match: { repository_ec2.settings.base_path : "@ec2_base_path@" } - - match: { repository_ec2.settings.canned_acl : "private" } - - match: { repository_ec2.settings.storage_class : "standard" } - - is_false: repository_ec2.settings.access_key - - is_false: repository_ec2.settings.secret_key - - is_false: repository_ec2.settings.session_token - - # Index documents - - do: - bulk: - refresh: true - body: - - index: - _index: docs - _id: "1" - - snapshot: one - - index: - _index: docs - _id: "2" - - snapshot: one - - index: - _index: docs - _id: "3" - - snapshot: one - - - do: - count: - index: docs - - - match: {count: 3} - - # Create a first snapshot - - do: - snapshot.create: - repository: repository_ec2 - snapshot: snapshot-one - wait_for_completion: true - - - match: { snapshot.snapshot: snapshot-one } - - match: { snapshot.state : SUCCESS } - - match: { snapshot.include_global_state: true } - - match: { snapshot.shards.failed : 0 } - - - do: - snapshot.status: - repository: repository_ec2 - snapshot: snapshot-one - - - is_true: snapshots - - match: { snapshots.0.snapshot: snapshot-one } - - match: { snapshots.0.state : SUCCESS } - - # Index more documents - - do: - bulk: - refresh: true - body: - - index: - _index: docs - _id: "4" - - snapshot: two - - index: - _index: docs - _id: "5" - - snapshot: two - - index: - _index: docs - _id: "6" - - snapshot: two - - index: - _index: docs - _id: "7" - - snapshot: two - - - do: - count: - index: docs - - - match: {count: 7} - - # Create a second snapshot - - do: - snapshot.create: - repository: repository_ec2 - snapshot: snapshot-two - wait_for_completion: true - - - match: { snapshot.snapshot: snapshot-two } - - match: { snapshot.state : SUCCESS } - - match: { snapshot.shards.failed : 0 } - - - do: - snapshot.get: - repository: repository_ec2 - snapshot: snapshot-one,snapshot-two - - - is_true: snapshots - - match: { snapshots.0.state : SUCCESS } - - match: { snapshots.1.state : SUCCESS } - - # Delete the index - - do: - indices.delete: - index: docs - - # Restore the second snapshot - - do: - snapshot.restore: - repository: repository_ec2 - snapshot: snapshot-two - wait_for_completion: true - - - do: - count: - index: docs - - - match: {count: 7} - - # Delete the index again - - do: - indices.delete: - index: docs - - # Restore the first snapshot - - do: - snapshot.restore: - repository: repository_ec2 - snapshot: snapshot-one - wait_for_completion: true - - - do: - count: - index: docs - - - match: {count: 3} - - # Remove the snapshots - - do: - snapshot.delete: - repository: repository_ec2 - snapshot: snapshot-two - - - do: - snapshot.delete: - repository: repository_ec2 - snapshot: snapshot-one - ---- -"Register a repository with a non existing bucket": - - - do: - catch: /repository_verification_exception/ - snapshot.create_repository: - repository: repository_ec2 - body: - type: s3 - settings: - bucket: zHHkfSqlbnBsbpSgvCYtxrEfFLqghXtyPvvvKPNBnRCicNHQLE - client: integration_test_temporary - ---- -"Register a repository with a non existing client": - - - do: - catch: /illegal_argument_exception/ - snapshot.create_repository: - repository: repository_ec2 - body: - type: s3 - settings: - bucket: repository_ec2 - client: unknown - ---- -"Register a read-only repository with a non existing bucket": - -- do: - catch: /repository_verification_exception/ - snapshot.create_repository: - repository: repository_ec2 - body: - type: s3 - settings: - readonly: true - bucket: zHHkfSqlbnBsbpSgvCYtxrEfFLqghXtyPvvvKPNBnRCicNHQLE - client: integration_test_temporary - ---- -"Register a read-only repository with a non existing client": - -- do: - catch: /illegal_argument_exception/ - snapshot.create_repository: - repository: repository_ec2 - body: - type: s3 - settings: - readonly: true - bucket: repository_ec2 - client: unknown - ---- -"Get a non existing snapshot": - - - do: - catch: /snapshot_missing_exception/ - snapshot.get: - repository: repository_ec2 - snapshot: missing - ---- -"Delete a non existing snapshot": - - - do: - catch: /snapshot_missing_exception/ - snapshot.delete: - repository: repository_ec2 - snapshot: missing - ---- -"Restore a non existing snapshot": - - - do: - catch: /snapshot_restore_exception/ - snapshot.restore: - repository: repository_ec2 - snapshot: missing - wait_for_completion: true - ---- -"Usage stats": - - requires: - cluster_features: - - repositories.supports_usage_stats - reason: requires this feature - - - do: - cluster.stats: {} - - - gte: { repositories.s3.count: 1 } - - gte: { repositories.s3.read_write: 1 } - ---- -teardown: - - # Remove our repository - - do: - snapshot.delete_repository: - repository: repository_ec2 diff --git a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml b/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml deleted file mode 100644 index de334b4b3df96..0000000000000 --- a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml +++ /dev/null @@ -1,278 +0,0 @@ -# Integration tests for repository-s3 - ---- -setup: - - # Register repository with ecs credentials - - do: - snapshot.create_repository: - repository: repository_ecs - body: - type: s3 - settings: - bucket: @ecs_bucket@ - client: integration_test_ecs - base_path: "@ecs_base_path@" - canned_acl: private - storage_class: standard - disable_chunked_encoding: @disable_chunked_encoding@ - ---- -"Snapshot and Restore with repository-s3 using ecs credentials": - - # Get repository - - do: - snapshot.get_repository: - repository: repository_ecs - - - match: { repository_ecs.settings.bucket : @ecs_bucket@ } - - match: { repository_ecs.settings.client : "integration_test_ecs" } - - match: { repository_ecs.settings.base_path : "@ecs_base_path@" } - - match: { repository_ecs.settings.canned_acl : "private" } - - match: { repository_ecs.settings.storage_class : "standard" } - - is_false: repository_ecs.settings.access_key - - is_false: repository_ecs.settings.secret_key - - is_false: repository_ecs.settings.session_token - - # Index documents - - do: - bulk: - refresh: true - body: - - index: - _index: docs - _id: "1" - - snapshot: one - - index: - _index: docs - _id: "2" - - snapshot: one - - index: - _index: docs - _id: "3" - - snapshot: one - - - do: - count: - index: docs - - - match: {count: 3} - - # Create a first snapshot - - do: - snapshot.create: - repository: repository_ecs - snapshot: snapshot-one - wait_for_completion: true - - - match: { snapshot.snapshot: snapshot-one } - - match: { snapshot.state : SUCCESS } - - match: { snapshot.include_global_state: true } - - match: { snapshot.shards.failed : 0 } - - - do: - snapshot.status: - repository: repository_ecs - snapshot: snapshot-one - - - is_true: snapshots - - match: { snapshots.0.snapshot: snapshot-one } - - match: { snapshots.0.state : SUCCESS } - - # Index more documents - - do: - bulk: - refresh: true - body: - - index: - _index: docs - _id: "4" - - snapshot: two - - index: - _index: docs - _id: "5" - - snapshot: two - - index: - _index: docs - _id: "6" - - snapshot: two - - index: - _index: docs - _id: "7" - - snapshot: two - - - do: - count: - index: docs - - - match: {count: 7} - - # Create a second snapshot - - do: - snapshot.create: - repository: repository_ecs - snapshot: snapshot-two - wait_for_completion: true - - - match: { snapshot.snapshot: snapshot-two } - - match: { snapshot.state : SUCCESS } - - match: { snapshot.shards.failed : 0 } - - - do: - snapshot.get: - repository: repository_ecs - snapshot: snapshot-one,snapshot-two - - - is_true: snapshots - - match: { snapshots.0.state : SUCCESS } - - match: { snapshots.1.state : SUCCESS } - - # Delete the index - - do: - indices.delete: - index: docs - - # Restore the second snapshot - - do: - snapshot.restore: - repository: repository_ecs - snapshot: snapshot-two - wait_for_completion: true - - - do: - count: - index: docs - - - match: {count: 7} - - # Delete the index again - - do: - indices.delete: - index: docs - - # Restore the first snapshot - - do: - snapshot.restore: - repository: repository_ecs - snapshot: snapshot-one - wait_for_completion: true - - - do: - count: - index: docs - - - match: {count: 3} - - # Remove the snapshots - - do: - snapshot.delete: - repository: repository_ecs - snapshot: snapshot-two - - - do: - snapshot.delete: - repository: repository_ecs - snapshot: snapshot-one - ---- -"Register a repository with a non existing bucket": - - - do: - catch: /repository_verification_exception/ - snapshot.create_repository: - repository: repository_ecs - body: - type: s3 - settings: - bucket: zHHkfSqlbnBsbpSgvCYtxrEfFLqghXtyPvvvKPNBnRCicNHQLE - client: integration_test_ecs - ---- -"Register a repository with a non existing client": - - - do: - catch: /illegal_argument_exception/ - snapshot.create_repository: - repository: repository_ecs - body: - type: s3 - settings: - bucket: repository_ecs - client: unknown - ---- -"Register a read-only repository with a non existing bucket": - -- do: - catch: /repository_verification_exception/ - snapshot.create_repository: - repository: repository_ecs - body: - type: s3 - settings: - readonly: true - bucket: zHHkfSqlbnBsbpSgvCYtxrEfFLqghXtyPvvvKPNBnRCicNHQLE - client: integration_test_ecs - ---- -"Register a read-only repository with a non existing client": - -- do: - catch: /illegal_argument_exception/ - snapshot.create_repository: - repository: repository_ecs - body: - type: s3 - settings: - readonly: true - bucket: repository_ecs - client: unknown - ---- -"Get a non existing snapshot": - - - do: - catch: /snapshot_missing_exception/ - snapshot.get: - repository: repository_ecs - snapshot: missing - ---- -"Delete a non existing snapshot": - - - do: - catch: /snapshot_missing_exception/ - snapshot.delete: - repository: repository_ecs - snapshot: missing - ---- -"Restore a non existing snapshot": - - - do: - catch: /snapshot_restore_exception/ - snapshot.restore: - repository: repository_ecs - snapshot: missing - wait_for_completion: true - ---- -"Usage stats": - - requires: - cluster_features: - - repositories.supports_usage_stats - reason: requires this feature - - - do: - cluster.stats: {} - - - gte: { repositories.s3.count: 1 } - - gte: { repositories.s3.read_write: 1 } - ---- -teardown: - - # Remove our repository - - do: - snapshot.delete_repository: - repository: repository_ecs diff --git a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/60_repository_sts_credentials.yml b/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/60_repository_sts_credentials.yml deleted file mode 100644 index 09a8526017960..0000000000000 --- a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/60_repository_sts_credentials.yml +++ /dev/null @@ -1,279 +0,0 @@ -# Integration tests for repository-s3 - ---- -setup: - - # Register repository with sts credentials - - do: - snapshot.create_repository: - repository: repository_sts - body: - type: s3 - settings: - bucket: @sts_bucket@ - client: integration_test_sts - base_path: "@sts_base_path@" - canned_acl: private - storage_class: standard - disable_chunked_encoding: @disable_chunked_encoding@ - ---- -"Snapshot and Restore repository-s3 using sts credentials": - - # Get repository - - do: - snapshot.get_repository: - repository: repository_sts - - - match: { repository_sts.settings.bucket: @sts_bucket@ } - - match: { repository_sts.settings.client: "integration_test_sts" } - - match: { repository_sts.settings.base_path: "@sts_base_path@" } - - match: { repository_sts.settings.canned_acl: "private" } - - match: { repository_sts.settings.storage_class: "standard" } - - is_false: repository_sts.settings.access_key - - is_false: repository_sts.settings.secret_key - - is_false: repository_sts.settings.session_token - - # Index documents - - do: - bulk: - refresh: true - body: - - index: - _index: docs - _id: 1 - - snapshot: one - - index: - _index: docs - _id: 2 - - snapshot: one - - index: - _index: docs - _id: 3 - - snapshot: one - - - do: - count: - index: docs - - - match: { count: 3 } - - # Create a first snapshot - - do: - snapshot.create: - repository: repository_sts - snapshot: snapshot-one - wait_for_completion: true - - - match: { snapshot.snapshot: snapshot-one } - - match: { snapshot.state: SUCCESS } - - match: { snapshot.include_global_state: true } - - match: { snapshot.shards.failed: 0 } - - - do: - snapshot.status: - repository: repository_sts - snapshot: snapshot-one - - - is_true: snapshots - - match: { snapshots.0.snapshot: snapshot-one } - - match: { snapshots.0.state: SUCCESS } - - # Index more documents - - do: - bulk: - refresh: true - body: - - index: - _index: docs - _id: 4 - - snapshot: two - - index: - _index: docs - _id: 5 - - snapshot: two - - index: - _index: docs - _id: 6 - - snapshot: two - - index: - _index: docs - _id: 7 - - snapshot: two - - - do: - count: - index: docs - - - match: { count: 7 } - - # Create a second snapshot - - do: - snapshot.create: - repository: repository_sts - snapshot: snapshot-two - wait_for_completion: true - - - match: { snapshot.snapshot: snapshot-two } - - match: { snapshot.state: SUCCESS } - - match: { snapshot.shards.failed: 0 } - - - do: - snapshot.get: - repository: repository_sts - snapshot: snapshot-one,snapshot-two - - - is_true: snapshots - - match: { snapshots.0.state: SUCCESS } - - match: { snapshots.1.state: SUCCESS } - - # Delete the index - - do: - indices.delete: - index: docs - - # Restore the second snapshot - - do: - snapshot.restore: - repository: repository_sts - snapshot: snapshot-two - wait_for_completion: true - - - do: - count: - index: docs - - - match: { count: 7 } - - # Delete the index again - - do: - indices.delete: - index: docs - - # Restore the first snapshot - - do: - snapshot.restore: - repository: repository_sts - snapshot: snapshot-one - wait_for_completion: true - - - do: - count: - index: docs - - - match: { count: 3 } - - # Remove the snapshots - - do: - snapshot.delete: - repository: repository_sts - snapshot: snapshot-two - - - do: - snapshot.delete: - repository: repository_sts - snapshot: snapshot-one - ---- - -"Register a repository with a non existing bucket": - - - do: - catch: /repository_verification_exception/ - snapshot.create_repository: - repository: repository_sts - body: - type: s3 - settings: - bucket: zHHkfSqlbnBsbpSgvCYtxrEfFLqghXtyPvvvKPNBnRCicNHQLE - client: integration_test_sts - ---- -"Register a repository with a non existing client": - - - do: - catch: /illegal_argument_exception/ - snapshot.create_repository: - repository: repository_sts - body: - type: s3 - settings: - bucket: repository_sts - client: unknown - ---- -"Register a read-only repository with a non existing bucket": - - - do: - catch: /repository_verification_exception/ - snapshot.create_repository: - repository: repository_sts - body: - type: s3 - settings: - readonly: true - bucket: zHHkfSqlbnBsbpSgvCYtxrEfFLqghXtyPvvvKPNBnRCicNHQLE - client: integration_test_sts - ---- -"Register a read-only repository with a non existing client": - - - do: - catch: /illegal_argument_exception/ - snapshot.create_repository: - repository: repository_sts - body: - type: s3 - settings: - readonly: true - bucket: repository_sts - client: unknown - ---- -"Get a non existing snapshot": - - - do: - catch: /snapshot_missing_exception/ - snapshot.get: - repository: repository_sts - snapshot: missing - ---- -"Delete a non existing snapshot": - - - do: - catch: /snapshot_missing_exception/ - snapshot.delete: - repository: repository_sts - snapshot: missing - ---- -"Restore a non existing snapshot": - - - do: - catch: /snapshot_restore_exception/ - snapshot.restore: - repository: repository_sts - snapshot: missing - wait_for_completion: true - ---- -"Usage stats": - - requires: - cluster_features: - - repositories.supports_usage_stats - reason: requires this feature - - - do: - cluster.stats: {} - - - gte: { repositories.s3.count: 1 } - - gte: { repositories.s3.read_write: 1 } - ---- -teardown: - - # Remove our repository - - do: - snapshot.delete_repository: - repository: repository_sts diff --git a/modules/repository-url/build.gradle b/modules/repository-url/build.gradle index 3fe2f9d9bae42..cbe36f9874bfe 100644 --- a/modules/repository-url/build.gradle +++ b/modules/repository-url/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.PropertyNormalization @@ -13,8 +14,8 @@ apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { - description 'Module for URL repository' - classname 'org.elasticsearch.plugin.repository.url.URLRepositoryPlugin' + description = 'Module for URL repository' + classname ='org.elasticsearch.plugin.repository.url.URLRepositoryPlugin' } restResources { diff --git a/modules/repository-url/src/internalClusterTest/java/org/elasticsearch/repositories/url/URLSnapshotRestoreIT.java b/modules/repository-url/src/internalClusterTest/java/org/elasticsearch/repositories/url/URLSnapshotRestoreIT.java index 335da9123ed5a..4d980f5f031c1 100644 --- a/modules/repository-url/src/internalClusterTest/java/org/elasticsearch/repositories/url/URLSnapshotRestoreIT.java +++ b/modules/repository-url/src/internalClusterTest/java/org/elasticsearch/repositories/url/URLSnapshotRestoreIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.url; diff --git a/modules/repository-url/src/main/java/module-info.java b/modules/repository-url/src/main/java/module-info.java index 1745720a83cdf..c299d3357e4f1 100644 --- a/modules/repository-url/src/main/java/module-info.java +++ b/modules/repository-url/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.repository.url { diff --git a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/FileURLBlobContainer.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/FileURLBlobContainer.java index 7bb732d2c17ed..dc18a577a11fe 100644 --- a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/FileURLBlobContainer.java +++ b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/FileURLBlobContainer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.blobstore.url; diff --git a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java index c33e1519e8a28..419261d12c38a 100644 --- a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java +++ b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.blobstore.url; diff --git a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java index 0a6a7c22e79e5..0e9c735b22fd6 100644 --- a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java +++ b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.blobstore.url; @@ -12,7 +13,6 @@ import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.blobstore.BlobStoreException; -import org.elasticsearch.common.blobstore.OperationPurpose; import org.elasticsearch.common.blobstore.url.http.HttpURLBlobContainer; import org.elasticsearch.common.blobstore.url.http.URLHttpClient; import org.elasticsearch.common.blobstore.url.http.URLHttpClientSettings; @@ -22,10 +22,8 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.CheckedFunction; -import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; -import java.util.Iterator; import java.util.List; /** @@ -108,11 +106,6 @@ public BlobContainer blobContainer(BlobPath blobPath) { } } - @Override - public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobNames) throws IOException { - throw new UnsupportedOperationException("Bulk deletes are not supported in URL repositories"); - } - @Override public void close() { // nothing to do here... diff --git a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/HttpResponseInputStream.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/HttpResponseInputStream.java index bc3640014bebc..457cb226af536 100644 --- a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/HttpResponseInputStream.java +++ b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/HttpResponseInputStream.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.blobstore.url.http; diff --git a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/HttpURLBlobContainer.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/HttpURLBlobContainer.java index 186c119d78230..86e35bd670fb6 100644 --- a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/HttpURLBlobContainer.java +++ b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/HttpURLBlobContainer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.blobstore.url.http; diff --git a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/RetryingHttpInputStream.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/RetryingHttpInputStream.java index ddf23bce09721..e207b6abc0f95 100644 --- a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/RetryingHttpInputStream.java +++ b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/RetryingHttpInputStream.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.blobstore.url.http; diff --git a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClient.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClient.java index 490787714ff3a..7e800b3e57d91 100644 --- a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClient.java +++ b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClient.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.blobstore.url.http; diff --git a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClientException.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClientException.java index 52800c52b4eb8..859c4df34e898 100644 --- a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClientException.java +++ b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClientException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.blobstore.url.http; diff --git a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClientIOException.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClientIOException.java index 9768ffdb47434..0f0c31fa61f58 100644 --- a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClientIOException.java +++ b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClientIOException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.blobstore.url.http; diff --git a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClientSettings.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClientSettings.java index e9c95271b9d0b..f192f2434ea2d 100644 --- a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClientSettings.java +++ b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClientSettings.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.blobstore.url.http; diff --git a/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java b/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java index ec0ca43c84b49..75da8aad02fab 100644 --- a/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java +++ b/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.repository.url; diff --git a/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java b/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java index 3390a5ce8ab43..51a223315644a 100644 --- a/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java +++ b/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.url; diff --git a/modules/repository-url/src/main/plugin-metadata/plugin-security.policy b/modules/repository-url/src/main/plugin-metadata/plugin-security.policy index 7002fba5c0c4f..bfd77bc296124 100644 --- a/modules/repository-url/src/main/plugin-metadata/plugin-security.policy +++ b/modules/repository-url/src/main/plugin-metadata/plugin-security.policy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ grant { diff --git a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/AbstractURLBlobStoreTests.java b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/AbstractURLBlobStoreTests.java index 132760c8b410b..0cb47d7f4c186 100644 --- a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/AbstractURLBlobStoreTests.java +++ b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/AbstractURLBlobStoreTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.blobstore.url; diff --git a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/FileURLBlobStoreTests.java b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/FileURLBlobStoreTests.java index 6254ec1be6332..1d5bd2692c845 100644 --- a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/FileURLBlobStoreTests.java +++ b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/FileURLBlobStoreTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.blobstore.url; diff --git a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/HttpURLBlobStoreTests.java b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/HttpURLBlobStoreTests.java index b5be5cdbbe3d9..51e91eeedb69c 100644 --- a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/HttpURLBlobStoreTests.java +++ b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/HttpURLBlobStoreTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.blobstore.url; diff --git a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobContainerRetriesTests.java b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobContainerRetriesTests.java index af91d807305fc..74b81197d4ec8 100644 --- a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobContainerRetriesTests.java +++ b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobContainerRetriesTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.blobstore.url; diff --git a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/http/RetryingHttpInputStreamTests.java b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/http/RetryingHttpInputStreamTests.java index d1425e56072ef..69386c460d8ed 100644 --- a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/http/RetryingHttpInputStreamTests.java +++ b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/http/RetryingHttpInputStreamTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.blobstore.url.http; diff --git a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/http/URLHttpClientTests.java b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/http/URLHttpClientTests.java index 7bc79a9cf8534..32451bb71fa7c 100644 --- a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/http/URLHttpClientTests.java +++ b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/http/URLHttpClientTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.blobstore.url.http; diff --git a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java index 39ddc1366f934..6243de0774685 100644 --- a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java +++ b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.url; diff --git a/modules/repository-url/src/yamlRestTest/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java b/modules/repository-url/src/yamlRestTest/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java index a5b1a48f94ac9..c502ed6b92007 100644 --- a/modules/repository-url/src/yamlRestTest/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java +++ b/modules/repository-url/src/yamlRestTest/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.url; diff --git a/modules/rest-root/build.gradle b/modules/rest-root/build.gradle index 235f1e71a0238..2cbd57ec31ef2 100644 --- a/modules/rest-root/build.gradle +++ b/modules/rest-root/build.gradle @@ -2,18 +2,19 @@ import org.elasticsearch.gradle.internal.info.BuildParams /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.internal-yaml-rest-test' apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { - description 'Adds HEAD and GET / endpoint to Elasticsearch' - classname 'org.elasticsearch.rest.root.MainRestPlugin' + description = 'Adds HEAD and GET / endpoint to Elasticsearch' + classname ='org.elasticsearch.rest.root.MainRestPlugin' } restResources { diff --git a/modules/rest-root/src/main/java/module-info.java b/modules/rest-root/src/main/java/module-info.java index ac6ce583b3643..06c2d517c2195 100644 --- a/modules/rest-root/src/main/java/module-info.java +++ b/modules/rest-root/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.rest.root { diff --git a/modules/rest-root/src/main/java/org/elasticsearch/rest/root/MainRequest.java b/modules/rest-root/src/main/java/org/elasticsearch/rest/root/MainRequest.java index 48ef81f93611f..edcf845512924 100644 --- a/modules/rest-root/src/main/java/org/elasticsearch/rest/root/MainRequest.java +++ b/modules/rest-root/src/main/java/org/elasticsearch/rest/root/MainRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.rest.root; diff --git a/modules/rest-root/src/main/java/org/elasticsearch/rest/root/MainResponse.java b/modules/rest-root/src/main/java/org/elasticsearch/rest/root/MainResponse.java index db74b61a359aa..4f764addb6c04 100644 --- a/modules/rest-root/src/main/java/org/elasticsearch/rest/root/MainResponse.java +++ b/modules/rest-root/src/main/java/org/elasticsearch/rest/root/MainResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.rest.root; diff --git a/modules/rest-root/src/main/java/org/elasticsearch/rest/root/MainRestPlugin.java b/modules/rest-root/src/main/java/org/elasticsearch/rest/root/MainRestPlugin.java index 7b234dbbe9177..760f79e9b96ef 100644 --- a/modules/rest-root/src/main/java/org/elasticsearch/rest/root/MainRestPlugin.java +++ b/modules/rest-root/src/main/java/org/elasticsearch/rest/root/MainRestPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.rest.root; diff --git a/modules/rest-root/src/main/java/org/elasticsearch/rest/root/RestMainAction.java b/modules/rest-root/src/main/java/org/elasticsearch/rest/root/RestMainAction.java index 2b14d52850a13..e58584248c6a9 100644 --- a/modules/rest-root/src/main/java/org/elasticsearch/rest/root/RestMainAction.java +++ b/modules/rest-root/src/main/java/org/elasticsearch/rest/root/RestMainAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.rest.root; diff --git a/modules/rest-root/src/main/java/org/elasticsearch/rest/root/TransportMainAction.java b/modules/rest-root/src/main/java/org/elasticsearch/rest/root/TransportMainAction.java index fa48fc8d2907c..2598f943ea27c 100644 --- a/modules/rest-root/src/main/java/org/elasticsearch/rest/root/TransportMainAction.java +++ b/modules/rest-root/src/main/java/org/elasticsearch/rest/root/TransportMainAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.rest.root; @@ -13,6 +14,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; @@ -47,7 +49,7 @@ protected void doExecute(Task task, MainRequest request, ActionListener responseRef = new AtomicReference<>(); - action.doExecute(mock(Task.class), new MainRequest(), new ActionListener<>() { - @Override - public void onResponse(MainResponse mainResponse) { - responseRef.set(mainResponse); - } - - @Override - public void onFailure(Exception e) { - logger.error("unexpected error", e); - } - }); + final AtomicBoolean listenerCalled = new AtomicBoolean(); + new TransportMainAction(settings, transportService, mock(ActionFilters.class), clusterService).doExecute( + mock(Task.class), + new MainRequest(), + ActionTestUtils.assertNoFailureListener(mainResponse -> { + assertNotNull(mainResponse); + assertEquals( + state.metadata().clusterUUIDCommitted() ? state.metadata().clusterUUID() : Metadata.UNKNOWN_CLUSTER_UUID, + mainResponse.getClusterUuid() + ); + assertFalse(listenerCalled.getAndSet(true)); + }) + ); - assertNotNull(responseRef.get()); + assertTrue(listenerCalled.get()); verify(clusterService, times(1)).state(); } } diff --git a/modules/rest-root/src/test/java/org/elasticsearch/rest/root/MainResponseTests.java b/modules/rest-root/src/test/java/org/elasticsearch/rest/root/MainResponseTests.java index e715638ee52bb..2369d43281134 100644 --- a/modules/rest-root/src/test/java/org/elasticsearch/rest/root/MainResponseTests.java +++ b/modules/rest-root/src/test/java/org/elasticsearch/rest/root/MainResponseTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.rest.root; diff --git a/modules/rest-root/src/test/java/org/elasticsearch/rest/root/RestMainActionTests.java b/modules/rest-root/src/test/java/org/elasticsearch/rest/root/RestMainActionTests.java index 81d9cbeda96e8..a537bf9ef14fe 100644 --- a/modules/rest-root/src/test/java/org/elasticsearch/rest/root/RestMainActionTests.java +++ b/modules/rest-root/src/test/java/org/elasticsearch/rest/root/RestMainActionTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.rest.root; diff --git a/modules/rest-root/src/yamlRestTest/java/org/elasticsearch/rest/root/RestMainClientYamlTestSuiteIT.java b/modules/rest-root/src/yamlRestTest/java/org/elasticsearch/rest/root/RestMainClientYamlTestSuiteIT.java index 6c03469d4f61e..dcc4ce76cbee5 100644 --- a/modules/rest-root/src/yamlRestTest/java/org/elasticsearch/rest/root/RestMainClientYamlTestSuiteIT.java +++ b/modules/rest-root/src/yamlRestTest/java/org/elasticsearch/rest/root/RestMainClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.rest.root; diff --git a/modules/runtime-fields-common/build.gradle b/modules/runtime-fields-common/build.gradle index 5a2d268cf7a4e..9e39094acd486 100644 --- a/modules/runtime-fields-common/build.gradle +++ b/modules/runtime-fields-common/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.validate-rest-spec' @@ -11,15 +12,15 @@ apply plugin: 'elasticsearch.internal-yaml-rest-test' apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { - description 'Module for runtime fields features and extensions that have large dependencies' - classname 'org.elasticsearch.runtimefields.RuntimeFieldsCommonPlugin' + description = 'Module for runtime fields features and extensions that have large dependencies' + classname = 'org.elasticsearch.runtimefields.RuntimeFieldsCommonPlugin' extendedPlugins = ['lang-painless'] } dependencies { compileOnly project(':modules:lang-painless:spi') - api project(':libs:elasticsearch-grok') - api project(':libs:elasticsearch-dissect') + api project(':libs:grok') + api project(':libs:dissect') } tasks.named("yamlRestTestV7CompatTransform").configure { task -> @@ -27,4 +28,5 @@ tasks.named("yamlRestTestV7CompatTransform").configure { task -> task.skipTest("runtime_fields/101_geo_point_from_source/fetch fields from source", "Format changed. Old format was a bug.") task.skipTest("runtime_fields/102_geo_point_source_in_query/fetch fields from source", "Format changed. Old format was a bug.") task.skipTest("runtime_fields/103_geo_point_calculated_at_index/fetch fields from source", "Format changed. Old format was a bug.") + task.skipTestsByFilePattern("**/runtime_fields/110_composite.yml", "warning does not exist for compatibility") } diff --git a/modules/runtime-fields-common/src/main/java/module-info.java b/modules/runtime-fields-common/src/main/java/module-info.java index 882dad6e00947..2971684905c83 100644 --- a/modules/runtime-fields-common/src/main/java/module-info.java +++ b/modules/runtime-fields-common/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.runtimefields { diff --git a/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/NamedGroupExtractor.java b/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/NamedGroupExtractor.java index 16d22407c25fa..78a26483002bc 100644 --- a/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/NamedGroupExtractor.java +++ b/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/NamedGroupExtractor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.runtimefields; diff --git a/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/RuntimeFieldsCommonPlugin.java b/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/RuntimeFieldsCommonPlugin.java index b0adda3b2062b..71188c2793d6c 100644 --- a/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/RuntimeFieldsCommonPlugin.java +++ b/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/RuntimeFieldsCommonPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.runtimefields; diff --git a/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/RuntimeFieldsPainlessExtension.java b/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/RuntimeFieldsPainlessExtension.java index ded0f9b8ea0bf..7317bab1193ee 100644 --- a/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/RuntimeFieldsPainlessExtension.java +++ b/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/RuntimeFieldsPainlessExtension.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.runtimefields; diff --git a/modules/runtime-fields-common/src/main/resources/org/elasticsearch/runtimefields/common_whitelist.txt b/modules/runtime-fields-common/src/main/resources/org/elasticsearch/runtimefields/common_whitelist.txt index 264f1b611098c..c5fc71a916409 100644 --- a/modules/runtime-fields-common/src/main/resources/org/elasticsearch/runtimefields/common_whitelist.txt +++ b/modules/runtime-fields-common/src/main/resources/org/elasticsearch/runtimefields/common_whitelist.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # class org.elasticsearch.runtimefields.NamedGroupExtractor @no_import { diff --git a/modules/runtime-fields-common/src/yamlRestTest/java/org/elasticsearch/painless/RuntimeFieldsClientYamlTestSuiteIT.java b/modules/runtime-fields-common/src/yamlRestTest/java/org/elasticsearch/painless/RuntimeFieldsClientYamlTestSuiteIT.java index 1304148911b4c..b1a6763febe89 100644 --- a/modules/runtime-fields-common/src/yamlRestTest/java/org/elasticsearch/painless/RuntimeFieldsClientYamlTestSuiteIT.java +++ b/modules/runtime-fields-common/src/yamlRestTest/java/org/elasticsearch/painless/RuntimeFieldsClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.painless; diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/10_keyword.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/10_keyword.yml index 11214907eb17e..8728d4ac413b7 100644 --- a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/10_keyword.yml +++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/10_keyword.yml @@ -19,7 +19,7 @@ setup: script: | Instant instant = Instant.ofEpochMilli(params._source.timestamp); ZonedDateTime dt = ZonedDateTime.ofInstant(instant, ZoneId.of("UTC")); - emit(dt.dayOfWeek.getDisplayName(TextStyle.FULL, Locale.ROOT)); + emit(dt.dayOfWeek.getDisplayName(TextStyle.FULL, Locale.ENGLISH)); # Test fetching many values day_of_week_letters: type: keyword @@ -218,7 +218,7 @@ setup: day_of_week: type: keyword script: | - emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.SHORT, Locale.ROOT)); + emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.SHORT, Locale.ENGLISH)); - do: search: index: sensor diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/110_composite.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/110_composite.yml index 13af92f9aa84b..eb6ce4ae61ae2 100644 --- a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/110_composite.yml +++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/110_composite.yml @@ -83,8 +83,8 @@ query: query: range: http.clientip: - from: 232.0.0.0 - to: 253.0.0.0 + gte: 232.0.0.0 + lte: 253.0.0.0 - match: { hits.total.value: 4 } --- diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/13_keyword_calculated_at_index.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/13_keyword_calculated_at_index.yml index 4bedfa3e923a8..c27ddab72bff4 100644 --- a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/13_keyword_calculated_at_index.yml +++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/13_keyword_calculated_at_index.yml @@ -28,7 +28,7 @@ setup: script: | Instant instant = Instant.ofEpochMilli(params._source.timestamp); ZonedDateTime dt = ZonedDateTime.ofInstant(instant, ZoneId.of("UTC")); - emit(dt.dayOfWeek.getDisplayName(TextStyle.FULL, Locale.ROOT)); + emit(dt.dayOfWeek.getDisplayName(TextStyle.FULL, Locale.ENGLISH)); # Test fetching many values day_of_week_letters: type: keyword diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/270_synthetic_source.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/270_synthetic_source.yml index 8832b3230910c..46ff988ac6e67 100644 --- a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/270_synthetic_source.yml +++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/270_synthetic_source.yml @@ -1,16 +1,17 @@ --- keywords: - requires: - cluster_features: ["gte_v7.12.0"] - reason: Runtime mappings support was added in 7.12 + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" - do: indices.create: index: index1 body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: field1: type: keyword @@ -69,16 +70,17 @@ keywords: --- doubles: - requires: - cluster_features: ["gte_v7.12.0"] - reason: Runtime mappings support was added in 7.12 + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" - do: indices.create: index: index1 body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: field1: type: double diff --git a/modules/systemd/build.gradle b/modules/systemd/build.gradle index 351211ffd3c0e..9155554797587 100644 --- a/modules/systemd/build.gradle +++ b/modules/systemd/build.gradle @@ -1,17 +1,18 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ esplugin { - description 'Integrates Elasticsearch with systemd' - classname 'org.elasticsearch.systemd.SystemdPlugin' + description = 'Integrates Elasticsearch with systemd' + classname ='org.elasticsearch.systemd.SystemdPlugin' } dependencies { - implementation project(':libs:elasticsearch-native') + implementation project(':libs:native') } diff --git a/modules/systemd/src/main/java/module-info.java b/modules/systemd/src/main/java/module-info.java index b3f5b64ff312f..415b5384601ae 100644 --- a/modules/systemd/src/main/java/module-info.java +++ b/modules/systemd/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.systemd { diff --git a/modules/systemd/src/main/java/org/elasticsearch/systemd/SystemdPlugin.java b/modules/systemd/src/main/java/org/elasticsearch/systemd/SystemdPlugin.java index 947d1fa58e963..6ae8e336ec987 100644 --- a/modules/systemd/src/main/java/org/elasticsearch/systemd/SystemdPlugin.java +++ b/modules/systemd/src/main/java/org/elasticsearch/systemd/SystemdPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.systemd; diff --git a/modules/systemd/src/main/plugin-metadata/plugin-security.policy b/modules/systemd/src/main/plugin-metadata/plugin-security.policy index ebb76b006af13..c8f6a798fdc2f 100644 --- a/modules/systemd/src/main/plugin-metadata/plugin-security.policy +++ b/modules/systemd/src/main/plugin-metadata/plugin-security.policy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ grant codeBase "${codebase.systemd}" { diff --git a/modules/systemd/src/test/java/org/elasticsearch/systemd/SystemdPluginTests.java b/modules/systemd/src/test/java/org/elasticsearch/systemd/SystemdPluginTests.java index 712483e9c603c..d45b61d73e10b 100644 --- a/modules/systemd/src/test/java/org/elasticsearch/systemd/SystemdPluginTests.java +++ b/modules/systemd/src/test/java/org/elasticsearch/systemd/SystemdPluginTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.systemd; diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle index 14b0b57da3b0c..4b64b9c56917d 100644 --- a/modules/transport-netty4/build.gradle +++ b/modules/transport-netty4/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.internal.test.RestIntegTestTask @@ -22,8 +23,8 @@ apply plugin: 'elasticsearch.publish' * fix the hack in the build framework that copies transport-netty4 into the integ test cluster */ esplugin { - description 'Netty 4 based transport implementation' - classname 'org.elasticsearch.transport.netty4.Netty4Plugin' + description = 'Netty 4 based transport implementation' + classname = 'org.elasticsearch.transport.netty4.Netty4Plugin' } // exclude transitively pulled in version via the esplugin plugin to always build from fresh sources and make jar-hell checks pass @@ -34,7 +35,7 @@ configurations { } dependencies { - api project(":libs:elasticsearch-ssl-config") + api project(":libs:ssl-config") // network stack api "io.netty:netty-buffer:${versions.netty}" @@ -176,9 +177,8 @@ tasks.named("thirdPartyAudit").configure { 'com.google.protobuf.nano.CodedOutputByteBufferNano', 'com.google.protobuf.nano.MessageNano', 'com.github.luben.zstd.Zstd', - 'com.github.luben.zstd.BaseZstdBufferDecompressingStreamNoFinalizer', - 'com.github.luben.zstd.ZstdBufferDecompressingStreamNoFinalizer', - 'com.github.luben.zstd.ZstdDirectBufferDecompressingStreamNoFinalizer', + 'com.github.luben.zstd.ZstdInputStreamNoFinalizer', + 'com.github.luben.zstd.util.Native', 'com.jcraft.jzlib.Deflater', 'com.jcraft.jzlib.Inflater', 'com.jcraft.jzlib.JZlib$WrapperType', @@ -230,8 +230,14 @@ tasks.named("thirdPartyAudit").configure { 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField', 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField', 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField', + 'io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueConsumerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueProducerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueConsumerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerLimitField', 'io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess', 'io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess', + 'io.netty.util.internal.shaded.org.jctools.util.UnsafeLongArrayAccess', 'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator', 'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$1', 'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$2', @@ -240,3 +246,7 @@ tasks.named("thirdPartyAudit").configure { 'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$5' ) } + +tasks.named('forbiddenApisMain').configure { + signaturesFiles += files('forbidden/netty-signatures.txt') +} diff --git a/modules/transport-netty4/forbidden/netty-signatures.txt b/modules/transport-netty4/forbidden/netty-signatures.txt new file mode 100644 index 0000000000000..4c47b2aee1253 --- /dev/null +++ b/modules/transport-netty4/forbidden/netty-signatures.txt @@ -0,0 +1,9 @@ + # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + # or more contributor license agreements. Licensed under the "Elastic License + # 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + # Public License v 1"; you may not use this file except in compliance with, at + # your election, the "Elastic License 2.0", the "GNU Affero General Public + # License v3.0 only", or the "Server Side Public License, v 1". + +@defaultMessage Use org.elasticsearch.transport.netty4.Netty4Utils.addListener(io.netty.channel.ChannelFuture, io.netty.channel.ChannelFutureListener) instead +io.netty.channel.ChannelFuture#addListener(io.netty.util.concurrent.GenericFutureListener) diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/ESNetty4IntegTestCase.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/ESNetty4IntegTestCase.java index 65fbde5d42005..95412d0268008 100644 --- a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/ESNetty4IntegTestCase.java +++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/ESNetty4IntegTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch; diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4ChunkedContinuationsIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4ChunkedContinuationsIT.java index 7432a510763e0..402730f063e5b 100644 --- a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4ChunkedContinuationsIT.java +++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4ChunkedContinuationsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.netty4; diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4ChunkedEncodingIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4ChunkedEncodingIT.java index e3f60ea7a48e0..d220438fc3170 100644 --- a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4ChunkedEncodingIT.java +++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4ChunkedEncodingIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.netty4; diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java index 0dadc159c41c7..fcd45e9f9f47e 100644 --- a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java +++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.netty4; @@ -13,6 +14,7 @@ import io.netty.util.ReferenceCounted; import org.elasticsearch.ESNetty4IntegTestCase; +import org.elasticsearch.action.bulk.IncrementalBulkService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.unit.ByteSizeUnit; @@ -51,6 +53,8 @@ protected boolean addMockHttpTransport() { protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { return Settings.builder() .put(super.nodeSettings(nodeOrdinal, otherSettings)) + // TODO: We do not currently support in flight circuit breaker limits for bulk. However, IndexingPressure applies + .put(IncrementalBulkService.INCREMENTAL_BULK.getKey(), false) .put(HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), LIMIT) .build(); } diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4IncrementalRequestHandlingIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4IncrementalRequestHandlingIT.java new file mode 100644 index 0000000000000..4bb27af4bd0f5 --- /dev/null +++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4IncrementalRequestHandlingIT.java @@ -0,0 +1,724 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.http.netty4; + +import io.netty.bootstrap.Bootstrap; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufInputStream; +import io.netty.buffer.Unpooled; +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; +import io.netty.handler.codec.http.DefaultFullHttpRequest; +import io.netty.handler.codec.http.DefaultHttpContent; +import io.netty.handler.codec.http.DefaultHttpRequest; +import io.netty.handler.codec.http.DefaultLastHttpContent; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpChunkedInput; +import io.netty.handler.codec.http.HttpClientCodec; +import io.netty.handler.codec.http.HttpContent; +import io.netty.handler.codec.http.HttpObjectAggregator; +import io.netty.handler.codec.http.HttpRequest; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.netty.handler.codec.http.HttpUtil; +import io.netty.handler.codec.http.LastHttpContent; +import io.netty.handler.stream.ChunkedStream; +import io.netty.handler.stream.ChunkedWriteHandler; + +import org.apache.logging.log4j.Level; +import org.elasticsearch.ESNetty4IntegTestCase; +import org.elasticsearch.action.support.SubscribableListener; +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.bytes.ReleasableBytesReference; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.http.HttpBodyTracer; +import org.elasticsearch.http.HttpHandlingSettings; +import org.elasticsearch.http.HttpServerTransport; +import org.elasticsearch.http.HttpTransportSettings; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.MockLog; +import org.elasticsearch.test.junit.annotations.TestLogging; +import org.elasticsearch.transport.Transports; +import org.elasticsearch.transport.netty4.Netty4Utils; + +import java.util.Collection; +import java.util.List; +import java.util.concurrent.BlockingDeque; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.function.Supplier; + +import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_LENGTH; +import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_TYPE; +import static io.netty.handler.codec.http.HttpHeaderValues.APPLICATION_JSON; +import static io.netty.handler.codec.http.HttpMethod.POST; +import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; + +@ESIntegTestCase.ClusterScope(numDataNodes = 1) +public class Netty4IncrementalRequestHandlingIT extends ESNetty4IntegTestCase { + + @Override + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings)); + builder.put(HttpTransportSettings.SETTING_HTTP_MAX_CONTENT_LENGTH.getKey(), new ByteSizeValue(50, ByteSizeUnit.MB)); + return builder.build(); + } + + // ensure empty http content has single 0 size chunk + public void testEmptyContent() throws Exception { + try (var ctx = setupClientCtx()) { + var totalRequests = randomIntBetween(1, 10); + for (int reqNo = 0; reqNo < totalRequests; reqNo++) { + var opaqueId = opaqueId(reqNo); + + // send request with empty content + ctx.clientChannel.writeAndFlush(fullHttpRequest(opaqueId, Unpooled.EMPTY_BUFFER)); + var handler = ctx.awaitRestChannelAccepted(opaqueId); + handler.stream.next(); + + // should receive a single empty chunk + var recvChunk = safePoll(handler.recvChunks); + assertTrue(recvChunk.isLast); + assertEquals(0, recvChunk.chunk.length()); + recvChunk.chunk.close(); + assertFalse(handler.streamClosed); + + // send response to process following request + handler.sendResponse(new RestResponse(RestStatus.OK, "")); + assertBusy(() -> assertTrue(handler.streamClosed)); + } + assertBusy(() -> assertEquals("should receive all server responses", totalRequests, ctx.clientRespQueue.size())); + } + } + + // ensures content integrity, no loses and re-order + public void testReceiveAllChunks() throws Exception { + try (var ctx = setupClientCtx()) { + var totalRequests = randomIntBetween(1, 10); + for (int reqNo = 0; reqNo < totalRequests; reqNo++) { + var opaqueId = opaqueId(reqNo); + + // this dataset will be compared with one on server side + var dataSize = randomIntBetween(1024, maxContentLength()); + var sendData = Unpooled.wrappedBuffer(randomByteArrayOfLength(dataSize)); + sendData.retain(); + ctx.clientChannel.writeAndFlush(fullHttpRequest(opaqueId, sendData)); + + var handler = ctx.awaitRestChannelAccepted(opaqueId); + + var recvData = Unpooled.buffer(dataSize); + while (true) { + handler.stream.next(); + var recvChunk = safePoll(handler.recvChunks); + try (recvChunk.chunk) { + recvData.writeBytes(Netty4Utils.toByteBuf(recvChunk.chunk)); + if (recvChunk.isLast) { + break; + } + } + } + + assertFalse(handler.streamClosed); + assertEquals("sent and received payloads are not the same", sendData, recvData); + handler.sendResponse(new RestResponse(RestStatus.OK, "")); + assertBusy(() -> assertTrue(handler.streamClosed)); + } + assertBusy(() -> assertEquals("should receive all server responses", totalRequests, ctx.clientRespQueue.size())); + } + } + + // ensures that all received chunks are released when connection closed and handler notified + public void testClientConnectionCloseMidStream() throws Exception { + try (var ctx = setupClientCtx()) { + var opaqueId = opaqueId(0); + + // write half of http request + ctx.clientChannel.write(httpRequest(opaqueId, 2 * 1024)); + ctx.clientChannel.writeAndFlush(randomContent(1024, false)); + + // await stream handler is ready and request full content + var handler = ctx.awaitRestChannelAccepted(opaqueId); + assertBusy(() -> assertNotEquals(0, handler.stream.bufSize())); + + assertFalse(handler.streamClosed); + + // terminate client connection + ctx.clientChannel.close(); + // read the first half of the request + handler.stream.next(); + // attempt to read more data and it should notice channel being closed eventually + handler.stream.next(); + + // wait for resources to be released + assertBusy(() -> { + assertEquals(0, handler.stream.bufSize()); + assertTrue(handler.streamClosed); + }); + } + } + + // ensures that all recieved chunks are released when server decides to close connection + public void testServerCloseConnectionMidStream() throws Exception { + try (var ctx = setupClientCtx()) { + var opaqueId = opaqueId(0); + + // write half of http request + ctx.clientChannel.write(httpRequest(opaqueId, 2 * 1024)); + ctx.clientChannel.writeAndFlush(randomContent(1024, false)); + + // await stream handler is ready and request full content + var handler = ctx.awaitRestChannelAccepted(opaqueId); + assertBusy(() -> assertNotEquals(0, handler.stream.bufSize())); + assertFalse(handler.streamClosed); + + // terminate connection on server and wait resources are released + handler.channel.request().getHttpChannel().close(); + assertBusy(() -> { + assertEquals(0, handler.stream.bufSize()); + assertTrue(handler.streamClosed); + }); + } + } + + public void testServerExceptionMidStream() throws Exception { + try (var ctx = setupClientCtx()) { + var opaqueId = opaqueId(0); + + // write half of http request + ctx.clientChannel.write(httpRequest(opaqueId, 2 * 1024)); + ctx.clientChannel.writeAndFlush(randomContent(1024, false)); + + // await stream handler is ready and request full content + var handler = ctx.awaitRestChannelAccepted(opaqueId); + assertBusy(() -> assertNotEquals(0, handler.stream.bufSize())); + assertFalse(handler.streamClosed); + + handler.shouldThrowInsideHandleChunk = true; + handler.stream.next(); + + assertBusy(() -> { + assertEquals(0, handler.stream.bufSize()); + assertTrue(handler.streamClosed); + }); + } + } + + // ensure that client's socket buffers data when server is not consuming data + public void testClientBackpressure() throws Exception { + try (var ctx = setupClientCtx()) { + var opaqueId = opaqueId(0); + var payloadSize = maxContentLength(); + var totalParts = 10; + var partSize = payloadSize / totalParts; + ctx.clientChannel.writeAndFlush(httpRequest(opaqueId, payloadSize)); + for (int i = 0; i < totalParts; i++) { + ctx.clientChannel.writeAndFlush(randomContent(partSize, false)); + } + assertFalse( + "should not flush last content immediately", + ctx.clientChannel.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT).isDone() + ); + + var handler = ctx.awaitRestChannelAccepted(opaqueId); + + // some data flushes from channel into OS buffer and won't be visible here, usually 4-8Mb + var osBufferOffset = MBytes(10); + + // incrementally read data on server side and ensure client side buffer drains accordingly + for (int readBytes = 0; readBytes <= payloadSize; readBytes += partSize) { + var minBufSize = Math.max(payloadSize - readBytes - osBufferOffset, 0); + var maxBufSize = Math.max(payloadSize - readBytes, 0); + // it is hard to tell that client's channel is no logger flushing data + // it might take a few busy-iterations before channel buffer flush to OS + // and bytesBeforeWritable will stop changing + assertBusy(() -> { + var bufSize = ctx.clientChannel.bytesBeforeWritable(); + assertTrue( + "client's channel buffer should be in range [" + minBufSize + "," + maxBufSize + "], got " + bufSize, + bufSize >= minBufSize && bufSize <= maxBufSize + ); + }); + handler.readBytes(partSize); + } + assertTrue(handler.stream.hasLast()); + } + } + + // ensures that server reply 100-continue on acceptable request size + public void test100Continue() throws Exception { + try (var ctx = setupClientCtx()) { + for (int reqNo = 0; reqNo < randomIntBetween(2, 10); reqNo++) { + var id = opaqueId(reqNo); + var acceptableContentLength = randomIntBetween(0, maxContentLength()); + + // send request header and await 100-continue + var req = httpRequest(id, acceptableContentLength); + HttpUtil.set100ContinueExpected(req, true); + ctx.clientChannel.writeAndFlush(req); + var resp = (FullHttpResponse) safePoll(ctx.clientRespQueue); + assertEquals(HttpResponseStatus.CONTINUE, resp.status()); + resp.release(); + + // send content + var content = randomContent(acceptableContentLength, true); + ctx.clientChannel.writeAndFlush(content); + + // consume content and reply 200 + var handler = ctx.awaitRestChannelAccepted(id); + var consumed = handler.readAllBytes(); + assertEquals(acceptableContentLength, consumed); + handler.sendResponse(new RestResponse(RestStatus.OK, "")); + + resp = (FullHttpResponse) safePoll(ctx.clientRespQueue); + assertEquals(HttpResponseStatus.OK, resp.status()); + resp.release(); + } + } + } + + // ensures that server reply 413-too-large on oversized request with expect-100-continue + public void test413TooLargeOnExpect100Continue() throws Exception { + try (var ctx = setupClientCtx()) { + for (int reqNo = 0; reqNo < randomIntBetween(2, 10); reqNo++) { + var id = opaqueId(reqNo); + var oversized = maxContentLength() + 1; + + // send request header and await 413 too large + var req = httpRequest(id, oversized); + HttpUtil.set100ContinueExpected(req, true); + ctx.clientChannel.writeAndFlush(req); + var resp = (FullHttpResponse) safePoll(ctx.clientRespQueue); + assertEquals(HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE, resp.status()); + resp.release(); + + // terminate request + ctx.clientChannel.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT); + } + } + } + + // ensures that oversized chunked encoded request has no limits at http layer + // rest handler is responsible for oversized requests + public void testOversizedChunkedEncodingNoLimits() throws Exception { + try (var ctx = setupClientCtx()) { + for (var reqNo = 0; reqNo < randomIntBetween(2, 10); reqNo++) { + var id = opaqueId(reqNo); + var contentSize = maxContentLength() + 1; + var content = randomByteArrayOfLength(contentSize); + var is = new ByteBufInputStream(Unpooled.wrappedBuffer(content)); + var chunkedIs = new ChunkedStream(is); + var httpChunkedIs = new HttpChunkedInput(chunkedIs, LastHttpContent.EMPTY_LAST_CONTENT); + var req = httpRequest(id, 0); + HttpUtil.setTransferEncodingChunked(req, true); + + ctx.clientChannel.pipeline().addLast(new ChunkedWriteHandler()); + ctx.clientChannel.writeAndFlush(req); + ctx.clientChannel.writeAndFlush(httpChunkedIs); + var handler = ctx.awaitRestChannelAccepted(id); + var consumed = handler.readAllBytes(); + assertEquals(contentSize, consumed); + handler.sendResponse(new RestResponse(RestStatus.OK, "")); + + var resp = (FullHttpResponse) safePoll(ctx.clientRespQueue); + assertEquals(HttpResponseStatus.OK, resp.status()); + resp.release(); + } + } + } + + // ensures that we dont leak buffers in stream on 400-bad-request + // some bad requests are dispatched from rest-controller before reaching rest handler + // test relies on netty's buffer leak detection + public void testBadRequestReleaseQueuedChunks() throws Exception { + try (var ctx = setupClientCtx()) { + for (var reqNo = 0; reqNo < randomIntBetween(2, 10); reqNo++) { + var id = opaqueId(reqNo); + var contentSize = randomIntBetween(0, maxContentLength()); + var req = httpRequest(id, contentSize); + var content = randomContent(contentSize, true); + + // set unacceptable content-type + req.headers().set(CONTENT_TYPE, "unknown"); + ctx.clientChannel.writeAndFlush(req); + ctx.clientChannel.writeAndFlush(content); + + var resp = (FullHttpResponse) safePoll(ctx.clientRespQueue); + assertEquals(HttpResponseStatus.BAD_REQUEST, resp.status()); + resp.release(); + } + } + } + + private static long transportStatsRequestBytesSize(Ctx ctx) { + var httpTransport = internalCluster().getInstance(HttpServerTransport.class, ctx.nodeName); + var stats = httpTransport.stats().clientStats(); + var bytes = 0L; + for (var s : stats) { + bytes += s.requestSizeBytes(); + } + return bytes; + } + + /** + * ensures that {@link org.elasticsearch.http.HttpClientStatsTracker} counts streamed content bytes + */ + public void testHttpClientStats() throws Exception { + try (var ctx = setupClientCtx()) { + // need to offset starting point, since we reuse cluster and other tests already sent some data + var totalBytesSent = transportStatsRequestBytesSize(ctx); + + for (var reqNo = 0; reqNo < randomIntBetween(2, 10); reqNo++) { + var id = opaqueId(reqNo); + var contentSize = randomIntBetween(0, maxContentLength()); + totalBytesSent += contentSize; + ctx.clientChannel.writeAndFlush(httpRequest(id, contentSize)); + ctx.clientChannel.writeAndFlush(randomContent(contentSize, true)); + var handler = ctx.awaitRestChannelAccepted(id); + handler.readAllBytes(); + handler.sendResponse(new RestResponse(RestStatus.OK, "")); + assertEquals(totalBytesSent, transportStatsRequestBytesSize(ctx)); + } + } + } + + /** + * ensures that we log parts of http body and final line + */ + @TestLogging( + reason = "testing TRACE logging", + value = "org.elasticsearch.http.HttpTracer:TRACE,org.elasticsearch.http.HttpBodyTracer:TRACE" + ) + public void testHttpBodyLogging() throws Exception { + assertHttpBodyLogging((ctx) -> () -> { + try { + var req = fullHttpRequest(opaqueId(0), randomByteBuf(8 * 1024)); + ctx.clientChannel.writeAndFlush(req); + var handler = ctx.awaitRestChannelAccepted(opaqueId(0)); + handler.readAllBytes(); + } catch (Exception e) { + fail(e); + } + }); + } + + /** + * ensures that we log some parts of body and final line when connection is closed in the middle + */ + @TestLogging( + reason = "testing TRACE logging", + value = "org.elasticsearch.http.HttpTracer:TRACE,org.elasticsearch.http.HttpBodyTracer:TRACE" + ) + public void testHttpBodyLoggingChannelClose() throws Exception { + assertHttpBodyLogging((ctx) -> () -> { + try { + var req = httpRequest(opaqueId(0), 2 * 8192); + var halfContent = randomContent(8192, false); + ctx.clientChannel.writeAndFlush(req); + ctx.clientChannel.writeAndFlush(halfContent); + var handler = ctx.awaitRestChannelAccepted(opaqueId(0)); + handler.readBytes(8192); + ctx.clientChannel.close(); + handler.stream.next(); + assertBusy(() -> assertTrue(handler.streamClosed)); + } catch (Exception e) { + fail(e); + } + }); + } + + // asserts that we emit at least one logging event for a part and last line + // http body should be large enough to split across multiple lines, > 4kb + private void assertHttpBodyLogging(Function test) throws Exception { + try (var ctx = setupClientCtx()) { + MockLog.assertThatLogger( + test.apply(ctx), + HttpBodyTracer.class, + new MockLog.SeenEventExpectation( + "request part", + HttpBodyTracer.class.getCanonicalName(), + Level.TRACE, + "* request body [part *]*" + ), + new MockLog.SeenEventExpectation( + "request end", + HttpBodyTracer.class.getCanonicalName(), + Level.TRACE, + "* request body (gzip compressed, base64-encoded, and split into * parts on preceding log lines;*)" + ) + ); + } + } + + private int maxContentLength() { + return HttpHandlingSettings.fromSettings(internalCluster().getInstance(Settings.class)).maxContentLength(); + } + + private String opaqueId(int reqNo) { + return getTestName() + "-" + reqNo; + } + + static int MBytes(int m) { + return m * 1024 * 1024; + } + + static T safePoll(BlockingDeque queue) { + try { + var t = queue.poll(SAFE_AWAIT_TIMEOUT.seconds(), TimeUnit.SECONDS); + assertNotNull("queue is empty", t); + return t; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + } + + static FullHttpRequest fullHttpRequest(String opaqueId, ByteBuf content) { + var req = new DefaultFullHttpRequest(HTTP_1_1, POST, ControlServerRequestPlugin.ROUTE, Unpooled.wrappedBuffer(content)); + req.headers().add(CONTENT_LENGTH, content.readableBytes()); + req.headers().add(CONTENT_TYPE, APPLICATION_JSON); + req.headers().add(Task.X_OPAQUE_ID_HTTP_HEADER, opaqueId); + return req; + } + + static HttpRequest httpRequest(String opaqueId, int contentLength) { + return httpRequest(ControlServerRequestPlugin.ROUTE, opaqueId, contentLength); + } + + static HttpRequest httpRequest(String uri, String opaqueId, int contentLength) { + var req = new DefaultHttpRequest(HTTP_1_1, POST, uri); + req.headers().add(CONTENT_LENGTH, contentLength); + req.headers().add(CONTENT_TYPE, APPLICATION_JSON); + req.headers().add(Task.X_OPAQUE_ID_HTTP_HEADER, opaqueId); + return req; + } + + static HttpContent randomContent(int size, boolean isLast) { + var buf = Unpooled.wrappedBuffer(randomByteArrayOfLength(size)); + if (isLast) { + return new DefaultLastHttpContent(buf); + } else { + return new DefaultHttpContent(buf); + } + } + + static ByteBuf randomByteBuf(int size) { + return Unpooled.wrappedBuffer(randomByteArrayOfLength(size)); + } + + Ctx setupClientCtx() throws Exception { + var nodeName = internalCluster().getRandomNodeName(); + var clientRespQueue = new LinkedBlockingDeque<>(16); + var bootstrap = bootstrapClient(nodeName, clientRespQueue); + var channel = bootstrap.connect().sync().channel(); + return new Ctx(getTestName(), nodeName, bootstrap, channel, clientRespQueue); + } + + Bootstrap bootstrapClient(String node, BlockingQueue queue) { + var httpServer = internalCluster().getInstance(HttpServerTransport.class, node); + var remoteAddr = randomFrom(httpServer.boundAddress().boundAddresses()); + return new Bootstrap().group(new NioEventLoopGroup(1)) + .channel(NioSocketChannel.class) + .remoteAddress(remoteAddr.getAddress(), remoteAddr.getPort()) + .handler(new ChannelInitializer() { + @Override + protected void initChannel(SocketChannel ch) { + var p = ch.pipeline(); + p.addLast(new HttpClientCodec()); + p.addLast(new HttpObjectAggregator(4096)); + p.addLast(new SimpleChannelInboundHandler() { + @Override + protected void channelRead0(ChannelHandlerContext ctx, FullHttpResponse msg) { + msg.retain(); + queue.add(msg); + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + queue.add(cause); + } + }); + } + }); + } + + @Override + protected Collection> nodePlugins() { + return CollectionUtils.concatLists(List.of(ControlServerRequestPlugin.class), super.nodePlugins()); + } + + @Override + protected boolean addMockHttpTransport() { + return false; // enable http + } + + record Ctx(String testName, String nodeName, Bootstrap clientBootstrap, Channel clientChannel, BlockingDeque clientRespQueue) + implements + AutoCloseable { + + @Override + public void close() throws Exception { + safeGet(clientChannel.close()); + safeGet(clientBootstrap.config().group().shutdownGracefully()); + clientRespQueue.forEach(o -> { if (o instanceof FullHttpResponse resp) resp.release(); }); + for (var opaqueId : ControlServerRequestPlugin.handlers.keySet()) { + if (opaqueId.startsWith(testName)) { + var handler = ControlServerRequestPlugin.handlers.get(opaqueId); + handler.recvChunks.forEach(c -> c.chunk.close()); + handler.channel.request().getHttpChannel().close(); + ControlServerRequestPlugin.handlers.remove(opaqueId); + } + } + } + + ServerRequestHandler awaitRestChannelAccepted(String opaqueId) throws Exception { + assertBusy(() -> assertTrue(ControlServerRequestPlugin.handlers.containsKey(opaqueId))); + var handler = ControlServerRequestPlugin.handlers.get(opaqueId); + safeAwait(handler.channelAccepted); + return handler; + } + } + + static class ServerRequestHandler implements BaseRestHandler.RequestBodyChunkConsumer { + final SubscribableListener channelAccepted = new SubscribableListener<>(); + final String opaqueId; + final BlockingDeque recvChunks = new LinkedBlockingDeque<>(); + final Netty4HttpRequestBodyStream stream; + RestChannel channel; + boolean recvLast = false; + volatile boolean streamClosed = false; + volatile boolean shouldThrowInsideHandleChunk = false; + + ServerRequestHandler(String opaqueId, Netty4HttpRequestBodyStream stream) { + this.opaqueId = opaqueId; + this.stream = stream; + } + + @Override + public void handleChunk(RestChannel channel, ReleasableBytesReference chunk, boolean isLast) { + Transports.assertTransportThread(); + if (shouldThrowInsideHandleChunk) { + // Must close the chunk. This is the contract of this method. + chunk.close(); + throw new RuntimeException("simulated exception inside handleChunk"); + } + recvChunks.add(new Chunk(chunk, isLast)); + } + + @Override + public void accept(RestChannel channel) throws Exception { + this.channel = channel; + channelAccepted.onResponse(null); + } + + @Override + public void streamClose() { + streamClosed = true; + } + + void sendResponse(RestResponse response) { + channel.sendResponse(response); + } + + int readBytes(int bytes) { + var consumed = 0; + if (recvLast == false) { + while (consumed < bytes) { + stream.next(); + var recvChunk = safePoll(recvChunks); + consumed += recvChunk.chunk.length(); + recvChunk.chunk.close(); + if (recvChunk.isLast) { + recvLast = true; + break; + } + } + } + return consumed; + } + + int readAllBytes() { + return readBytes(Integer.MAX_VALUE); + } + + record Chunk(ReleasableBytesReference chunk, boolean isLast) {} + } + + // takes full control of rest handler from the outside + public static class ControlServerRequestPlugin extends Plugin implements ActionPlugin { + + static final String ROUTE = "/_test/request-stream"; + + static final ConcurrentHashMap handlers = new ConcurrentHashMap<>(); + + @Override + public Collection getRestHandlers( + Settings settings, + NamedWriteableRegistry namedWriteableRegistry, + RestController restController, + ClusterSettings clusterSettings, + IndexScopedSettings indexScopedSettings, + SettingsFilter settingsFilter, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier nodesInCluster, + Predicate clusterSupportsFeature + ) { + return List.of(new BaseRestHandler() { + @Override + public String getName() { + return ROUTE; + } + + @Override + public List routes() { + return List.of(new Route(RestRequest.Method.POST, ROUTE)); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { + var stream = (Netty4HttpRequestBodyStream) request.contentStream(); + var opaqueId = request.getHeaders().get(Task.X_OPAQUE_ID_HTTP_HEADER).get(0); + var handler = new ServerRequestHandler(opaqueId, stream); + handlers.put(opaqueId, handler); + return handler; + } + }); + } + } + +} diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java index 89a76dd26e285..417faaf64f7f3 100644 --- a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java +++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.netty4; diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4TrashingAllocatorIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4TrashingAllocatorIT.java new file mode 100644 index 0000000000000..f3a10ce228117 --- /dev/null +++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4TrashingAllocatorIT.java @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.http.netty4; + +import io.netty.handler.codec.http.HttpResponseStatus; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.ESNetty4IntegTestCase; +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.http.HttpServerTransport; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.RestStatus; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.function.Predicate; +import java.util.function.Supplier; + +public class Netty4TrashingAllocatorIT extends ESNetty4IntegTestCase { + + @Override + protected Collection> nodePlugins() { + return CollectionUtils.concatLists(List.of(Handler.class), super.nodePlugins()); + } + + @Override + protected boolean addMockHttpTransport() { + return false; + } + + public void testTrashContent() throws InterruptedException { + try (var client = new Netty4HttpClient()) { + var addr = randomFrom(internalCluster().getInstance(HttpServerTransport.class).boundAddress().boundAddresses()).address(); + var content = randomAlphaOfLength(between(1024, 2048)); + var responses = client.post(addr, List.of(new Tuple<>(Handler.ROUTE, content))); + assertEquals(HttpResponseStatus.OK, responses.stream().findFirst().get().status()); + } + } + + public static class Handler extends Plugin implements ActionPlugin { + static final String ROUTE = "/_test/trashing-alloc"; + + @Override + public Collection getRestHandlers( + Settings settings, + NamedWriteableRegistry namedWriteableRegistry, + RestController restController, + ClusterSettings clusterSettings, + IndexScopedSettings indexScopedSettings, + SettingsFilter settingsFilter, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier nodesInCluster, + Predicate clusterSupportsFeature + ) { + return List.of(new BaseRestHandler() { + @Override + public String getName() { + return ROUTE; + } + + @Override + public List routes() { + return List.of(new Route(RestRequest.Method.POST, ROUTE)); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + var content = request.content(); + var iter = content.iterator(); + return (chan) -> { + request.getHttpRequest().release(); + assertFalse(content.hasReferences()); + BytesRef br; + while ((br = iter.next()) != null) { + for (int i = br.offset; i < br.offset + br.length; i++) { + if (br.bytes[i] != 0) { + fail( + new AssertionError( + "buffer is not trashed, off=" + + br.offset + + " len=" + + br.length + + " pos=" + + i + + " ind=" + + (i - br.offset) + ) + ); + } + } + } + chan.sendResponse(new RestResponse(RestStatus.OK, "")); + }; + } + }); + } + } +} diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java index b4ec0269db9a4..0ddf0e5513feb 100644 --- a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java +++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport.netty4; diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java index 104677b64351f..5fe128e3cd3e3 100644 --- a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java +++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport.netty4; diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java index bfad7889baa17..eaacb541ef977 100644 --- a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java +++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport.netty4; diff --git a/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4BadRequestIT.java b/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4BadRequestIT.java index a7bc031448087..ce30759de403d 100644 --- a/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4BadRequestIT.java +++ b/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4BadRequestIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.rest; diff --git a/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java b/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java index e705c43036870..6eea5d09590f8 100644 --- a/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java +++ b/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.rest; diff --git a/modules/transport-netty4/src/main/java/module-info.java b/modules/transport-netty4/src/main/java/module-info.java index e531615d5cc1e..04a00a6175070 100644 --- a/modules/transport-netty4/src/main/java/module-info.java +++ b/modules/transport-netty4/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.transport.netty4 { diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4ChunkedHttpContinuation.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4ChunkedHttpContinuation.java index cde0249216981..433d1996c35ad 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4ChunkedHttpContinuation.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4ChunkedHttpContinuation.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.netty4; diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4ChunkedHttpResponse.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4ChunkedHttpResponse.java index 3abab9fa2526f..d81f8986dfd7f 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4ChunkedHttpResponse.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4ChunkedHttpResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.netty4; diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4FullHttpResponse.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4FullHttpResponse.java index a350427c75ec5..33a926b9f47df 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4FullHttpResponse.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4FullHttpResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.netty4; diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpAggregator.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpAggregator.java new file mode 100644 index 0000000000000..021ce09e0ed8e --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpAggregator.java @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.http.netty4; + +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpContent; +import io.netty.handler.codec.http.HttpObject; +import io.netty.handler.codec.http.HttpObjectAggregator; +import io.netty.handler.codec.http.HttpRequest; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.netty.handler.codec.http.HttpUtil; + +import org.elasticsearch.http.HttpPreRequest; +import org.elasticsearch.http.netty4.internal.HttpHeadersAuthenticatorUtils; + +import java.util.function.Predicate; + +/** + * A wrapper around {@link HttpObjectAggregator}. Provides optional content aggregation based on + * predicate. {@link HttpObjectAggregator} also handles Expect: 100-continue and oversized content. + * Unfortunately, Netty does not provide handlers for oversized messages beyond HttpObjectAggregator. + */ +public class Netty4HttpAggregator extends HttpObjectAggregator { + private static final Predicate IGNORE_TEST = (req) -> req.uri().startsWith("/_test/request-stream") == false; + + private final Predicate decider; + private boolean aggregating = true; + private boolean ignoreContentAfterContinueResponse = false; + + public Netty4HttpAggregator(int maxContentLength, Predicate decider) { + super(maxContentLength); + this.decider = decider; + } + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + assert msg instanceof HttpObject; + if (msg instanceof HttpRequest request) { + var preReq = HttpHeadersAuthenticatorUtils.asHttpPreRequest(request); + aggregating = (decider.test(preReq) && IGNORE_TEST.test(preReq)) || request.decoderResult().isFailure(); + } + if (aggregating || msg instanceof FullHttpRequest) { + super.channelRead(ctx, msg); + } else { + handle(ctx, (HttpObject) msg); + } + } + + private void handle(ChannelHandlerContext ctx, HttpObject msg) { + if (msg instanceof HttpRequest request) { + var continueResponse = newContinueResponse(request, maxContentLength(), ctx.pipeline()); + if (continueResponse != null) { + // there are 3 responses expected: 100, 413, 417 + // on 100 we pass request further and reply to client to continue + // on 413/417 we ignore following content + ctx.writeAndFlush(continueResponse); + var resp = (FullHttpResponse) continueResponse; + if (resp.status() != HttpResponseStatus.CONTINUE) { + ignoreContentAfterContinueResponse = true; + return; + } + HttpUtil.set100ContinueExpected(request, false); + } + ignoreContentAfterContinueResponse = false; + ctx.fireChannelRead(msg); + } else { + var httpContent = (HttpContent) msg; + if (ignoreContentAfterContinueResponse) { + httpContent.release(); + } else { + ctx.fireChannelRead(msg); + } + } + } +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpChannel.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpChannel.java index 83728b8ef73c2..c5a4754e419b4 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpChannel.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpChannel.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.netty4; diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpHeaderValidator.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpHeaderValidator.java index ad322503b0d06..95a68cb52bbdb 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpHeaderValidator.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpHeaderValidator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.netty4; @@ -60,6 +61,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception pending.add(ReferenceCountUtil.retain(httpObject)); requestStart(ctx); assert state == QUEUEING_DATA; + assert ctx.channel().config().isAutoRead() == false; break; case QUEUEING_DATA: pending.add(ReferenceCountUtil.retain(httpObject)); @@ -76,14 +78,14 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception if (httpObject instanceof LastHttpContent) { state = WAITING_TO_START; } - // fall-through + ReferenceCountUtil.release(httpObject); + break; case DROPPING_DATA_PERMANENTLY: assert pending.isEmpty(); ReferenceCountUtil.release(httpObject); // consume without enqueuing + ctx.channel().config().setAutoRead(false); break; } - - setAutoReadForState(ctx, state); } private void requestStart(ChannelHandlerContext ctx) { @@ -104,6 +106,7 @@ private void requestStart(ChannelHandlerContext ctx) { } state = QUEUEING_DATA; + ctx.channel().config().setAutoRead(false); if (httpRequest == null) { // this looks like a malformed request and will forward without validation @@ -149,6 +152,7 @@ private void forwardFullRequest(ChannelHandlerContext ctx) { assert ctx.channel().config().isAutoRead() == false; assert state == QUEUEING_DATA; + ctx.channel().config().setAutoRead(true); boolean fullRequestForwarded = forwardData(ctx, pending); assert fullRequestForwarded || pending.isEmpty(); @@ -160,7 +164,6 @@ private void forwardFullRequest(ChannelHandlerContext ctx) { } assert state == WAITING_TO_START || state == QUEUEING_DATA || state == FORWARDING_DATA_UNTIL_NEXT_REQUEST; - setAutoReadForState(ctx, state); } private void forwardRequestWithDecoderExceptionAndNoContent(ChannelHandlerContext ctx, Exception e) { @@ -176,6 +179,8 @@ private void forwardRequestWithDecoderExceptionAndNoContent(ChannelHandlerContex messageToForward = toReplace.replace(Unpooled.EMPTY_BUFFER); } messageToForward.setDecoderResult(DecoderResult.failure(e)); + + ctx.channel().config().setAutoRead(true); ctx.fireChannelRead(messageToForward); assert fullRequestDropped || pending.isEmpty(); @@ -187,7 +192,6 @@ private void forwardRequestWithDecoderExceptionAndNoContent(ChannelHandlerContex } assert state == WAITING_TO_START || state == QUEUEING_DATA || state == DROPPING_DATA_UNTIL_NEXT_REQUEST; - setAutoReadForState(ctx, state); } @Override @@ -243,10 +247,6 @@ private static void maybeResizePendingDown(int largeSize, ArrayDeque } } - private static void setAutoReadForState(ChannelHandlerContext ctx, State state) { - ctx.channel().config().setAutoRead((state == QUEUEING_DATA || state == DROPPING_DATA_PERMANENTLY) == false); - } - enum State { WAITING_TO_START, QUEUEING_DATA, diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java index b915011514d9a..1a391a05add58 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.netty4; @@ -19,8 +20,11 @@ import io.netty.handler.codec.http.DefaultHttpResponse; import io.netty.handler.codec.http.DefaultLastHttpContent; import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.HttpContent; import io.netty.handler.codec.http.HttpObject; +import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.codec.http.LastHttpContent; import io.netty.handler.ssl.SslCloseCompletionEvent; import io.netty.util.ReferenceCountUtil; import io.netty.util.concurrent.Future; @@ -70,6 +74,12 @@ private record ChunkedWrite(PromiseCombiner combiner, ChannelPromise onDone, Chu @Nullable private ChunkedWrite currentChunkedWrite; + /** + * HTTP request content stream for current request, it's null if there is no current request or request is fully-aggregated + */ + @Nullable + private Netty4HttpRequestBodyStream currentRequestStream; + /* * The current read and write sequence numbers. Read sequence numbers are attached to requests in the order they are read from the * channel, and then transferred to responses. A response is not written to the channel context until its sequence number matches the @@ -109,23 +119,41 @@ public Netty4HttpPipeliningHandler( public void channelRead(final ChannelHandlerContext ctx, final Object msg) { activityTracker.startActivity(); try { - assert msg instanceof FullHttpRequest : "Should have fully aggregated message already but saw [" + msg + "]"; - final FullHttpRequest fullHttpRequest = (FullHttpRequest) msg; - final Netty4HttpRequest netty4HttpRequest; - if (fullHttpRequest.decoderResult().isFailure()) { - final Throwable cause = fullHttpRequest.decoderResult().cause(); - final Exception nonError; - if (cause instanceof Error) { - ExceptionsHelper.maybeDieOnAnotherThread(cause); - nonError = new Exception(cause); + if (msg instanceof HttpRequest request) { + final Netty4HttpRequest netty4HttpRequest; + if (request.decoderResult().isFailure()) { + final Throwable cause = request.decoderResult().cause(); + final Exception nonError; + if (cause instanceof Error) { + ExceptionsHelper.maybeDieOnAnotherThread(cause); + nonError = new Exception(cause); + } else { + nonError = (Exception) cause; + } + netty4HttpRequest = new Netty4HttpRequest(readSequence++, (FullHttpRequest) request, nonError); } else { - nonError = (Exception) cause; + assert currentRequestStream == null : "current stream must be null for new request"; + if (request instanceof FullHttpRequest fullHttpRequest) { + netty4HttpRequest = new Netty4HttpRequest(readSequence++, fullHttpRequest); + currentRequestStream = null; + } else { + var contentStream = new Netty4HttpRequestBodyStream( + ctx.channel(), + serverTransport.getThreadPool().getThreadContext() + ); + currentRequestStream = contentStream; + netty4HttpRequest = new Netty4HttpRequest(readSequence++, request, contentStream); + } } - netty4HttpRequest = new Netty4HttpRequest(readSequence++, fullHttpRequest, nonError); + handlePipelinedRequest(ctx, netty4HttpRequest); } else { - netty4HttpRequest = new Netty4HttpRequest(readSequence++, fullHttpRequest); + assert msg instanceof HttpContent : "expect HttpContent got " + msg; + assert currentRequestStream != null : "current stream must exists before handling http content"; + currentRequestStream.handleNettyContent((HttpContent) msg); + if (msg instanceof LastHttpContent) { + currentRequestStream = null; + } } - handlePipelinedRequest(ctx, netty4HttpRequest); } finally { activityTracker.stopActivity(); } @@ -301,8 +329,8 @@ public void onFailure(Exception e) { Strings.format("failed to get continuation of HTTP response body for [%s], closing connection", channel), e ); - channel.close().addListener(ignored -> { - finishingWrite.combiner().add(channel.newFailedFuture(e)); + Netty4Utils.addListener(channel.close(), f -> { + finishingWrite.combiner().add(f.channel().newFailedFuture(e)); finishingWrite.combiner().finish(finishingWrite.onDone()); }); checkShutdown(); @@ -416,7 +444,7 @@ private boolean writeChunk(ChannelHandlerContext ctx, ChunkedWrite chunkedWrite) final boolean isPartComplete = bodyPart.isPartComplete(); final boolean isBodyComplete = isPartComplete && bodyPart.isLastPart(); final ChannelFuture f = ctx.write(isBodyComplete ? new DefaultLastHttpContent(content) : new DefaultHttpContent(content)); - f.addListener(ignored -> bytes.close()); + Netty4Utils.addListener(f, ignored -> bytes.close()); combiner.add(f); return isPartComplete; } diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java index 1e35f084c87ec..2662ddf7e1440 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java @@ -1,25 +1,28 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.netty4; -import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.handler.codec.http.DefaultFullHttpRequest; +import io.netty.handler.codec.http.EmptyHttpHeaders; import io.netty.handler.codec.http.FullHttpRequest; import io.netty.handler.codec.http.HttpHeaderNames; import io.netty.handler.codec.http.HttpHeaders; import io.netty.handler.codec.http.HttpMethod; +import io.netty.handler.codec.http.QueryStringDecoder; import io.netty.handler.codec.http.cookie.Cookie; import io.netty.handler.codec.http.cookie.ServerCookieDecoder; import io.netty.handler.codec.http.cookie.ServerCookieEncoder; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.http.HttpBody; import org.elasticsearch.http.HttpRequest; import org.elasticsearch.http.HttpResponse; import org.elasticsearch.rest.ChunkedRestResponseBodyPart; @@ -39,22 +42,41 @@ public class Netty4HttpRequest implements HttpRequest { private final FullHttpRequest request; - private final BytesReference content; + private final HttpBody content; private final Map> headers; private final AtomicBoolean released; private final Exception inboundException; private final boolean pooled; private final int sequence; + private final QueryStringDecoder queryStringDecoder; + + Netty4HttpRequest(int sequence, io.netty.handler.codec.http.HttpRequest request, Netty4HttpRequestBodyStream contentStream) { + this( + sequence, + new DefaultFullHttpRequest( + request.protocolVersion(), + request.method(), + request.uri(), + Unpooled.EMPTY_BUFFER, + request.headers(), + EmptyHttpHeaders.INSTANCE + ), + new AtomicBoolean(false), + true, + contentStream, + null + ); + } Netty4HttpRequest(int sequence, FullHttpRequest request) { - this(sequence, request, new AtomicBoolean(false), true, Netty4Utils.toBytesReference(request.content())); + this(sequence, request, new AtomicBoolean(false), true, Netty4Utils.fullHttpBodyFrom(request.content())); } Netty4HttpRequest(int sequence, FullHttpRequest request, Exception inboundException) { - this(sequence, request, new AtomicBoolean(false), true, Netty4Utils.toBytesReference(request.content()), inboundException); + this(sequence, request, new AtomicBoolean(false), true, Netty4Utils.fullHttpBodyFrom(request.content()), inboundException); } - private Netty4HttpRequest(int sequence, FullHttpRequest request, AtomicBoolean released, boolean pooled, BytesReference content) { + private Netty4HttpRequest(int sequence, FullHttpRequest request, AtomicBoolean released, boolean pooled, HttpBody content) { this(sequence, request, released, pooled, content, null); } @@ -63,7 +85,7 @@ private Netty4HttpRequest( FullHttpRequest request, AtomicBoolean released, boolean pooled, - BytesReference content, + HttpBody content, Exception inboundException ) { this.sequence = sequence; @@ -73,6 +95,7 @@ private Netty4HttpRequest( this.pooled = pooled; this.released = released; this.inboundException = inboundException; + this.queryStringDecoder = new QueryStringDecoder(request.uri()); } @Override @@ -86,7 +109,12 @@ public String uri() { } @Override - public BytesReference content() { + public String rawPath() { + return queryStringDecoder.rawPath(); + } + + @Override + public HttpBody body() { assert released.get() == false; return content; } @@ -95,33 +123,7 @@ public BytesReference content() { public void release() { if (pooled && released.compareAndSet(false, true)) { request.release(); - } - } - - @Override - public HttpRequest releaseAndCopy() { - assert released.get() == false; - if (pooled == false) { - return this; - } - try { - final ByteBuf copiedContent = Unpooled.copiedBuffer(request.content()); - return new Netty4HttpRequest( - sequence, - new DefaultFullHttpRequest( - request.protocolVersion(), - request.method(), - request.uri(), - copiedContent, - request.headers(), - request.trailingHeaders() - ), - new AtomicBoolean(false), - false, - Netty4Utils.toBytesReference(copiedContent) - ); - } finally { - release(); + content.close(); } } diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStream.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStream.java new file mode 100644 index 0000000000000..ac3e3aecf97b9 --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStream.java @@ -0,0 +1,176 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.http.netty4; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.CompositeByteBuf; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFutureListener; +import io.netty.handler.codec.http.HttpContent; +import io.netty.handler.codec.http.LastHttpContent; + +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.http.HttpBody; +import org.elasticsearch.transport.netty4.Netty4Utils; + +import java.util.ArrayList; +import java.util.List; + +/** + * Netty based implementation of {@link HttpBody.Stream}. + * This implementation utilize {@link io.netty.channel.ChannelConfig#setAutoRead(boolean)} + * to prevent entire payload buffering. But sometimes upstream can send few chunks of data despite + * autoRead=off. In this case chunks will be buffered until downstream calls {@link Stream#next()} + */ +public class Netty4HttpRequestBodyStream implements HttpBody.Stream { + + private final Channel channel; + private final ChannelFutureListener closeListener = future -> doClose(); + private final List tracingHandlers = new ArrayList<>(4); + private final ThreadContext threadContext; + private ByteBuf buf; + private boolean requested = false; + private boolean closing = false; + private HttpBody.ChunkHandler handler; + private ThreadContext.StoredContext requestContext; + + // used in tests + private volatile int bufSize = 0; + private volatile boolean hasLast = false; + + public Netty4HttpRequestBodyStream(Channel channel, ThreadContext threadContext) { + this.channel = channel; + this.threadContext = threadContext; + this.requestContext = threadContext.newStoredContext(); + Netty4Utils.addListener(channel.closeFuture(), closeListener); + channel.config().setAutoRead(false); + } + + @Override + public ChunkHandler handler() { + return handler; + } + + @Override + public void setHandler(ChunkHandler chunkHandler) { + this.handler = chunkHandler; + } + + @Override + public void addTracingHandler(ChunkHandler chunkHandler) { + assert tracingHandlers.contains(chunkHandler) == false; + tracingHandlers.add(chunkHandler); + } + + @Override + public void next() { + assert closing == false : "cannot request next chunk on closing stream"; + assert handler != null : "handler must be set before requesting next chunk"; + requestContext = threadContext.newStoredContext(); + channel.eventLoop().submit(() -> { + requested = true; + if (buf == null) { + channel.read(); + } else { + try { + send(); + } catch (Exception e) { + channel.pipeline().fireExceptionCaught(e); + } + } + }); + } + + public void handleNettyContent(HttpContent httpContent) { + assert hasLast == false : "receive http content on completed stream"; + hasLast = httpContent instanceof LastHttpContent; + if (closing) { + httpContent.release(); + } else { + addChunk(httpContent.content()); + if (requested) { + send(); + } + } + } + + // adds chunk to current buffer, will allocate composite buffer when need to hold more than 1 chunk + private void addChunk(ByteBuf chunk) { + assert chunk != null; + if (buf == null) { + buf = chunk; + } else if (buf instanceof CompositeByteBuf comp) { + comp.addComponent(true, chunk); + } else { + var comp = channel.alloc().compositeBuffer(); + comp.addComponent(true, buf); + comp.addComponent(true, chunk); + buf = comp; + } + bufSize = buf.readableBytes(); + } + + // visible for test + int bufSize() { + return bufSize; + } + + // visible for test + boolean hasLast() { + return hasLast; + } + + private void send() { + assert requested; + assert handler != null : "must set handler before receiving next chunk"; + var bytesRef = Netty4Utils.toReleasableBytesReference(buf); + requested = false; + buf = null; + bufSize = 0; + try (var ignored = threadContext.restoreExistingContext(requestContext)) { + for (var tracer : tracingHandlers) { + tracer.onNext(bytesRef, hasLast); + } + handler.onNext(bytesRef, hasLast); + } + if (hasLast) { + channel.config().setAutoRead(true); + channel.closeFuture().removeListener(closeListener); + } + } + + @Override + public void close() { + if (channel.eventLoop().inEventLoop()) { + doClose(); + } else { + channel.eventLoop().submit(this::doClose); + } + } + + private void doClose() { + closing = true; + try (var ignored = threadContext.restoreExistingContext(requestContext)) { + for (var tracer : tracingHandlers) { + Releasables.closeExpectNoException(tracer); + } + if (handler != null) { + handler.close(); + } + } + if (buf != null) { + buf.release(); + buf = null; + bufSize = 0; + } + channel.config().setAutoRead(true); + } +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpResponse.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpResponse.java index 80cf3469c00ca..7452f01cbd5f4 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpResponse.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.netty4; diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerChannel.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerChannel.java index 58dc675b76bb1..eefa455d4137f 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerChannel.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerChannel.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.netty4; diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java index a309877e9aa83..36c860f1fb90b 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.netty4; @@ -32,10 +33,12 @@ import io.netty.handler.timeout.ReadTimeoutException; import io.netty.handler.timeout.ReadTimeoutHandler; import io.netty.util.AttributeKey; +import io.netty.util.ResourceLeakDetector; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.bulk.IncrementalBulkService; import org.elasticsearch.common.network.CloseableChannel; import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.network.ThreadWatchdog; @@ -96,6 +99,7 @@ public class Netty4HttpServerTransport extends AbstractHttpServerTransport { private final TLSConfig tlsConfig; private final AcceptChannelHandler.AcceptPredicate acceptChannelPredicate; private final HttpValidator httpValidator; + private final IncrementalBulkService.Enabled enabled; private final ThreadWatchdog threadWatchdog; private final int readTimeoutMillis; @@ -134,6 +138,7 @@ public Netty4HttpServerTransport( this.acceptChannelPredicate = acceptChannelPredicate; this.httpValidator = httpValidator; this.threadWatchdog = networkService.getThreadWatchdog(); + this.enabled = new IncrementalBulkService.Enabled(clusterSettings); this.pipeliningMaxEvents = SETTING_PIPELINING_MAX_EVENTS.get(settings); @@ -279,7 +284,7 @@ public void onException(HttpChannel channel, Exception cause) { } public ChannelHandler configureServerChannelHandler() { - return new HttpChannelHandler(this, handlingSettings, tlsConfig, acceptChannelPredicate, httpValidator); + return new HttpChannelHandler(this, handlingSettings, tlsConfig, acceptChannelPredicate, httpValidator, enabled); } static final AttributeKey HTTP_CHANNEL_KEY = AttributeKey.newInstance("es-http-channel"); @@ -292,19 +297,22 @@ protected static class HttpChannelHandler extends ChannelInitializer { private final TLSConfig tlsConfig; private final BiPredicate acceptChannelPredicate; private final HttpValidator httpValidator; + private final IncrementalBulkService.Enabled enabled; protected HttpChannelHandler( final Netty4HttpServerTransport transport, final HttpHandlingSettings handlingSettings, final TLSConfig tlsConfig, @Nullable final BiPredicate acceptChannelPredicate, - @Nullable final HttpValidator httpValidator + @Nullable final HttpValidator httpValidator, + IncrementalBulkService.Enabled enabled ) { this.transport = transport; this.handlingSettings = handlingSettings; this.tlsConfig = tlsConfig; this.acceptChannelPredicate = acceptChannelPredicate; this.httpValidator = httpValidator; + this.enabled = enabled; } @Override @@ -325,8 +333,12 @@ protected void initChannel(Channel ch) throws Exception { if (tlsConfig.isTLSEnabled()) { ch.pipeline().addLast("ssl", new SslHandler(tlsConfig.createServerSSLEngine())); } + final var threadWatchdogActivityTracker = transport.threadWatchdog.getActivityTrackerForCurrentThread(); ch.pipeline() - .addLast("chunked_writer", new Netty4WriteThrottlingHandler(transport.getThreadPool().getThreadContext())) + .addLast( + "chunked_writer", + new Netty4WriteThrottlingHandler(transport.getThreadPool().getThreadContext(), threadWatchdogActivityTracker) + ) .addLast("byte_buf_sizer", NettyByteBufSizer.INSTANCE); if (transport.readTimeoutMillis > 0) { ch.pipeline().addLast("read_timeout", new ReadTimeoutHandler(transport.readTimeoutMillis, TimeUnit.MILLISECONDS)); @@ -365,7 +377,12 @@ protected HttpMessage createMessage(String[] initialLine) throws Exception { ); } // combines the HTTP message pieces into a single full HTTP request (with headers and body) - final HttpObjectAggregator aggregator = new HttpObjectAggregator(handlingSettings.maxContentLength()); + final HttpObjectAggregator aggregator = new Netty4HttpAggregator( + handlingSettings.maxContentLength(), + httpPreRequest -> enabled.get() == false + || ((httpPreRequest.rawPath().endsWith("/_bulk") == false) + || httpPreRequest.rawPath().startsWith("/_xpack/monitoring/_bulk")) + ); aggregator.setMaxCumulationBufferComponents(transport.maxCompositeBufferComponents); ch.pipeline() .addLast("decoder_compress", new HttpContentDecompressor()) // this handles request body decompression @@ -394,14 +411,13 @@ protected Result beginEncode(HttpResponse httpResponse, String acceptEncoding) t } }); } + if (ResourceLeakDetector.isEnabled()) { + ch.pipeline().addLast(new Netty4LeakDetectionHandler()); + } ch.pipeline() .addLast( "pipelining", - new Netty4HttpPipeliningHandler( - transport.pipeliningMaxEvents, - transport, - transport.threadWatchdog.getActivityTrackerForCurrentThread() - ) + new Netty4HttpPipeliningHandler(transport.pipeliningMaxEvents, transport, threadWatchdogActivityTracker) ); transport.serverAcceptedChannel(nettyHttpChannel); } diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4LeakDetectionHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4LeakDetectionHandler.java new file mode 100644 index 0000000000000..8a0274872e493 --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4LeakDetectionHandler.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.http.netty4; + +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.handler.codec.http.HttpContent; +import io.netty.handler.codec.http.HttpRequest; + +import org.elasticsearch.tasks.Task; + +/** + * Inbound channel handler that enrich leaking buffers information from HTTP request. + * It helps to detect which handler is leaking buffers. Especially integration tests that run with + * paranoid leak detector that samples all buffers for leaking. Supplying informative opaque-id in + * integ test helps to narrow down problem (for example test name). + */ +public class Netty4LeakDetectionHandler extends ChannelInboundHandlerAdapter { + + private String info; + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) { + if (msg instanceof HttpRequest request) { + var opaqueId = request.headers().get(Task.X_OPAQUE_ID_HTTP_HEADER); + info = "method: " + request.method() + "; uri: " + request.uri() + "; x-opaque-id: " + opaqueId; + } + if (msg instanceof HttpContent content) { + content.touch(info); + } + ctx.fireChannelRead(msg); + } +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/internal/HttpHeadersAuthenticatorUtils.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/internal/HttpHeadersAuthenticatorUtils.java index a289d4b42e62d..b32116da1e78e 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/internal/HttpHeadersAuthenticatorUtils.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/internal/HttpHeadersAuthenticatorUtils.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.netty4.internal; diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/internal/HttpHeadersWithAuthenticationContext.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/internal/HttpHeadersWithAuthenticationContext.java index 357647860cc42..a75015b30b18b 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/internal/HttpHeadersWithAuthenticationContext.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/internal/HttpHeadersWithAuthenticationContext.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.netty4.internal; diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/internal/HttpValidator.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/internal/HttpValidator.java index 54b025487b2a6..04fe5e5f3dab5 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/internal/HttpValidator.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/internal/HttpValidator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.netty4.internal; diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/AcceptChannelHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/AcceptChannelHandler.java index e31c76b9ab7c2..970cc691345ca 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/AcceptChannelHandler.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/AcceptChannelHandler.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport.netty4; diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/CopyBytesServerSocketChannel.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/CopyBytesServerSocketChannel.java index ed526aca57346..7aaee0d0d1687 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/CopyBytesServerSocketChannel.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/CopyBytesServerSocketChannel.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". * * ============================================================================ * diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/CopyBytesSocketChannel.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/CopyBytesSocketChannel.java index 63032e12d37d4..72705db2b3094 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/CopyBytesSocketChannel.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/CopyBytesSocketChannel.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". * * ============================================================================= * diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ESLoggingHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ESLoggingHandler.java index 302876f1ae915..7a9dea4344ec6 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ESLoggingHandler.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ESLoggingHandler.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport.netty4; diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/NetUtils.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/NetUtils.java index 6c93b6036578d..312e3983c6348 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/NetUtils.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/NetUtils.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport.netty4; diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4MessageInboundHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4MessageInboundHandler.java index e39a60e0efd58..8fdb7051e2be6 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4MessageInboundHandler.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4MessageInboundHandler.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport.netty4; @@ -13,10 +14,8 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.ReleasableBytesReference; import org.elasticsearch.common.network.ThreadWatchdog; -import org.elasticsearch.core.RefCounted; import org.elasticsearch.core.Releasables; import org.elasticsearch.transport.InboundPipeline; import org.elasticsearch.transport.Transports; @@ -51,9 +50,8 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception final ByteBuf buffer = (ByteBuf) msg; Netty4TcpChannel channel = ctx.channel().attr(Netty4Transport.CHANNEL_KEY).get(); - final BytesReference wrapped = Netty4Utils.toBytesReference(buffer); activityTracker.startActivity(); - try (ReleasableBytesReference reference = new ReleasableBytesReference(wrapped, new ByteBufRefCounted(buffer))) { + try (ReleasableBytesReference reference = Netty4Utils.toReleasableBytesReference(buffer)) { pipeline.handleBytes(channel, reference); } finally { activityTracker.stopActivity(); @@ -80,35 +78,4 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { super.channelInactive(ctx); } - private record ByteBufRefCounted(ByteBuf buffer) implements RefCounted { - - @Override - public void incRef() { - buffer.retain(); - } - - @Override - public boolean tryIncRef() { - if (hasReferences() == false) { - return false; - } - try { - buffer.retain(); - } catch (RuntimeException e) { - assert hasReferences() == false; - return false; - } - return true; - } - - @Override - public boolean decRef() { - return buffer.release(); - } - - @Override - public boolean hasReferences() { - return buffer.refCnt() > 0; - } - } } diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4NioSocketChannel.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4NioSocketChannel.java index 1711c0d4ba9cf..4139e76c547c9 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4NioSocketChannel.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4NioSocketChannel.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport.netty4; diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Plugin.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Plugin.java index 5fd69f8d9e537..449fd72669dad 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Plugin.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Plugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport.netty4; diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4TcpChannel.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4TcpChannel.java index 726f1293c5cf3..420fe16dcb689 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4TcpChannel.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4TcpChannel.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport.netty4; diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4TcpServerChannel.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4TcpServerChannel.java index 13f691e6e0e5e..85a109070df8d 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4TcpServerChannel.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4TcpServerChannel.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport.netty4; diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java index 8080170bf1ee7..d8b02a0e9a0df 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport.netty4; @@ -358,7 +359,10 @@ private void setupPipeline(Channel ch, boolean isRemoteClusterServerChannel) { if (NetworkTraceFlag.TRACE_ENABLED) { pipeline.addLast("logging", ESLoggingHandler.INSTANCE); } - pipeline.addLast("chunked_writer", new Netty4WriteThrottlingHandler(getThreadPool().getThreadContext())); + pipeline.addLast( + "chunked_writer", + new Netty4WriteThrottlingHandler(getThreadPool().getThreadContext(), threadWatchdog.getActivityTrackerForCurrentThread()) + ); pipeline.addLast( "dispatcher", new Netty4MessageInboundHandler( @@ -380,9 +384,9 @@ protected InboundPipeline getInboundPipeline(Channel ch, boolean isRemoteCluster } private static void addClosedExceptionLogger(Channel channel) { - channel.closeFuture().addListener(f -> { - if (f.isSuccess() == false) { - logger.debug(() -> format("exception while closing channel: %s", channel), f.cause()); + Netty4Utils.addListener(channel.closeFuture(), channelFuture -> { + if (channelFuture.isSuccess() == false && logger.isDebugEnabled()) { + logger.debug(format("exception while closing channel: %s", channelFuture.channel()), channelFuture.cause()); } }); } diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java index 1025ad11ba05e..81b4fd3fbb9ee 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport.netty4; @@ -12,6 +13,9 @@ import io.netty.buffer.CompositeByteBuf; import io.netty.buffer.Unpooled; import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelPromise; import io.netty.channel.DefaultChannelPromise; import io.netty.util.NettyRuntime; import io.netty.util.concurrent.Future; @@ -23,10 +27,14 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.bytes.ReleasableBytesReference; import org.elasticsearch.common.recycler.Recycler; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Booleans; +import org.elasticsearch.core.RefCounted; +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.http.HttpBody; import org.elasticsearch.transport.TransportException; import java.io.IOException; @@ -123,6 +131,57 @@ public static BytesReference toBytesReference(final ByteBuf buffer) { } } + /** + * Wrap Netty's {@link ByteBuf} into {@link ReleasableBytesReference} and delegating reference count to ByteBuf. + */ + public static ReleasableBytesReference toReleasableBytesReference(final ByteBuf buffer) { + return new ReleasableBytesReference(toBytesReference(buffer), toRefCounted(buffer)); + } + + static ByteBufRefCounted toRefCounted(final ByteBuf buf) { + return new ByteBufRefCounted(buf); + } + + record ByteBufRefCounted(ByteBuf buffer) implements RefCounted { + + public int refCnt() { + return buffer.refCnt(); + } + + @Override + public void incRef() { + buffer.retain(); + } + + @Override + public boolean tryIncRef() { + if (hasReferences() == false) { + return false; + } + try { + buffer.retain(); + } catch (RuntimeException e) { + assert hasReferences() == false; + return false; + } + return true; + } + + @Override + public boolean decRef() { + return buffer.release(); + } + + @Override + public boolean hasReferences() { + return buffer.refCnt() > 0; + } + } + + public static HttpBody.Full fullHttpBodyFrom(final ByteBuf buf) { + return new HttpBody.ByteRefHttpBody(toReleasableBytesReference(buf)); + } + public static Recycler createRecycler(Settings settings) { // If this method is called by super ctor the processors will not be set. Accessing NettyAllocator initializes netty's internals // setting the processors. We must do it ourselves first just in case. @@ -140,7 +199,7 @@ public static void safeWriteAndFlush(Channel channel, Object message, ActionList // can only be completed by some network event from this point on. However... final var promise = new DefaultChannelPromise(channel, ImmediateEventExecutor.INSTANCE); addListener(promise, listener); - assert assertCorrectPromiseListenerThreading(channel, promise); + assert assertCorrectPromiseListenerThreading(promise); channel.writeAndFlush(message, promise); if (channel.eventLoop().isShuttingDown()) { // ... if we get here then the event loop may already have terminated, and https://github.com/netty/netty/issues/8007 means that @@ -155,10 +214,10 @@ public static void safeWriteAndFlush(Channel channel, Object message, ActionList } } - private static boolean assertCorrectPromiseListenerThreading(Channel channel, Future promise) { - final var eventLoop = channel.eventLoop(); - promise.addListener(future -> { - assert eventLoop.inEventLoop() || future.cause() instanceof RejectedExecutionException || channel.eventLoop().isTerminated() + private static boolean assertCorrectPromiseListenerThreading(ChannelPromise promise) { + addListener(promise, future -> { + var eventLoop = future.channel().eventLoop(); + assert eventLoop.inEventLoop() || future.cause() instanceof RejectedExecutionException || eventLoop.isTerminated() : future.cause(); }); return true; @@ -182,4 +241,9 @@ public static void addListener(Future future, ActionListener listene } }); } + + @SuppressForbidden(reason = "single point for adding listeners that enforces use of ChannelFutureListener") + public static void addListener(ChannelFuture channelFuture, ChannelFutureListener listener) { + channelFuture.addListener(listener); + } } diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4WriteThrottlingHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4WriteThrottlingHandler.java index ced2d7d65fa16..738da83817cb8 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4WriteThrottlingHandler.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4WriteThrottlingHandler.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport.netty4; @@ -13,15 +14,16 @@ import io.netty.channel.Channel; import io.netty.channel.ChannelDuplexHandler; import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelPromise; import io.netty.util.concurrent.Future; -import io.netty.util.concurrent.GenericFutureListener; import io.netty.util.concurrent.PromiseCombiner; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.network.ThreadWatchdog; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.transport.Transports; @@ -41,31 +43,44 @@ public final class Netty4WriteThrottlingHandler extends ChannelDuplexHandler { private final Queue queuedWrites = new LinkedList<>(); private final ThreadContext threadContext; + private final ThreadWatchdog.ActivityTracker threadWatchdogActivityTracker; private WriteOperation currentWrite; - public Netty4WriteThrottlingHandler(ThreadContext threadContext) { + public Netty4WriteThrottlingHandler(ThreadContext threadContext, ThreadWatchdog.ActivityTracker threadWatchdogActivityTracker) { this.threadContext = threadContext; + this.threadWatchdogActivityTracker = threadWatchdogActivityTracker; } @Override public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws IOException { - if (msg instanceof BytesReference reference) { - if (reference.hasArray()) { - writeSingleByteBuf(ctx, Unpooled.wrappedBuffer(reference.array(), reference.arrayOffset(), reference.length()), promise); - } else { - BytesRefIterator iter = reference.iterator(); - final PromiseCombiner combiner = new PromiseCombiner(ctx.executor()); - BytesRef next; - while ((next = iter.next()) != null) { - final ChannelPromise chunkPromise = ctx.newPromise(); - combiner.add((Future) chunkPromise); - writeSingleByteBuf(ctx, Unpooled.wrappedBuffer(next.bytes, next.offset, next.length), chunkPromise); + final boolean startedActivity = threadWatchdogActivityTracker.maybeStartActivity(); + try { + if (msg instanceof BytesReference reference) { + if (reference.hasArray()) { + writeSingleByteBuf( + ctx, + Unpooled.wrappedBuffer(reference.array(), reference.arrayOffset(), reference.length()), + promise + ); + } else { + BytesRefIterator iter = reference.iterator(); + final PromiseCombiner combiner = new PromiseCombiner(ctx.executor()); + BytesRef next; + while ((next = iter.next()) != null) { + final ChannelPromise chunkPromise = ctx.newPromise(); + combiner.add((Future) chunkPromise); + writeSingleByteBuf(ctx, Unpooled.wrappedBuffer(next.bytes, next.offset, next.length), chunkPromise); + } + combiner.finish(promise); } - combiner.finish(promise); + } else { + assert msg instanceof ByteBuf; + writeSingleByteBuf(ctx, (ByteBuf) msg, promise); + } + } finally { + if (startedActivity) { + threadWatchdogActivityTracker.stopActivity(); } - } else { - assert msg instanceof ByteBuf; - writeSingleByteBuf(ctx, (ByteBuf) msg, promise); } } @@ -93,13 +108,13 @@ private void writeInSlices(ChannelHandlerContext ctx, ChannelPromise promise, By final int bufferSize = Math.min(readableBytes, MAX_BYTES_PER_WRITE); if (readableBytes == bufferSize) { // last write for this chunk we're done - ctx.write(buf).addListener(forwardResultListener(ctx, promise)); + Netty4Utils.addListener(ctx.write(buf), forwardResultListener(promise)); return; } final int readerIndex = buf.readerIndex(); final ByteBuf writeBuffer = buf.retainedSlice(readerIndex, bufferSize); buf.readerIndex(readerIndex + bufferSize); - ctx.write(writeBuffer).addListener(forwardFailureListener(ctx, promise)); + Netty4Utils.addListener(ctx.write(writeBuffer), forwardFailureListener(promise)); if (ctx.channel().isWritable() == false) { // channel isn't writable any longer -> move to queuing queueWrite(buf, promise); @@ -115,22 +130,45 @@ private void queueWrite(ByteBuf buf, ChannelPromise promise) { @Override public void channelWritabilityChanged(ChannelHandlerContext ctx) { - if (ctx.channel().isWritable()) { - doFlush(ctx); + final boolean startedActivity = threadWatchdogActivityTracker.maybeStartActivity(); + try { + if (ctx.channel().isWritable()) { + doFlush(ctx); + } + ctx.fireChannelWritabilityChanged(); + } finally { + if (startedActivity) { + threadWatchdogActivityTracker.stopActivity(); + } } - ctx.fireChannelWritabilityChanged(); } @Override public void flush(ChannelHandlerContext ctx) { - if (doFlush(ctx) == false) { - ctx.flush(); + final boolean startedActivity = threadWatchdogActivityTracker.maybeStartActivity(); + try { + if (doFlush(ctx) == false) { + ctx.flush(); + } + } finally { + if (startedActivity) { + threadWatchdogActivityTracker.stopActivity(); + } } } @Override public void channelInactive(ChannelHandlerContext ctx) throws Exception { - doFlush(ctx); + final boolean startedActivity = threadWatchdogActivityTracker.maybeStartActivity(); + try { + doFlush(ctx); + } finally { + if (startedActivity) { + threadWatchdogActivityTracker.stopActivity(); + } + } + + // super.channelInactive() can trigger reads which are tracked separately (and are not re-entrant) so no activity tracking here super.channelInactive(ctx); } @@ -163,9 +201,9 @@ private boolean doFlush(ChannelHandlerContext ctx) { final ChannelFuture writeFuture = ctx.write(writeBuffer); if (sliced == false) { currentWrite = null; - writeFuture.addListener(forwardResultListener(ctx, write.promise)); + Netty4Utils.addListener(writeFuture, forwardResultListener(write.promise)); } else { - writeFuture.addListener(forwardFailureListener(ctx, write.promise)); + Netty4Utils.addListener(writeFuture, forwardFailureListener(write.promise)); } } ctx.flush(); @@ -175,18 +213,18 @@ private boolean doFlush(ChannelHandlerContext ctx) { return true; } - private static GenericFutureListener> forwardFailureListener(ChannelHandlerContext ctx, ChannelPromise promise) { + private static ChannelFutureListener forwardFailureListener(ChannelPromise promise) { return future -> { - assert ctx.executor().inEventLoop(); + assert future.channel().eventLoop().inEventLoop(); if (future.isSuccess() == false) { promise.tryFailure(future.cause()); } }; } - private static GenericFutureListener> forwardResultListener(ChannelHandlerContext ctx, ChannelPromise promise) { + private static ChannelFutureListener forwardResultListener(ChannelPromise promise) { return future -> { - assert ctx.executor().inEventLoop(); + assert future.channel().eventLoop().inEventLoop(); if (future.isSuccess()) { promise.trySuccess(); } else { diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/NettyAllocator.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/NettyAllocator.java index 863334af85144..e8bd5514947d6 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/NettyAllocator.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/NettyAllocator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport.netty4; @@ -23,9 +24,11 @@ import org.elasticsearch.common.recycler.Recycler; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.PageCacheRecycler; +import org.elasticsearch.core.Assertions; import org.elasticsearch.core.Booleans; import org.elasticsearch.monitor.jvm.JvmInfo; +import java.util.Arrays; import java.util.concurrent.atomic.AtomicBoolean; public class NettyAllocator { @@ -43,8 +46,9 @@ public class NettyAllocator { private static final String USE_NETTY_DEFAULT_CHUNK = "es.unsafe.use_netty_default_chunk_and_page_size"; static { + ByteBufAllocator allocator; if (Booleans.parseBoolean(System.getProperty(USE_NETTY_DEFAULT), false)) { - ALLOCATOR = ByteBufAllocator.DEFAULT; + allocator = ByteBufAllocator.DEFAULT; SUGGESTED_MAX_ALLOCATION_SIZE = 1024 * 1024; DESCRIPTION = "[name=netty_default, suggested_max_allocation_size=" + ByteSizeValue.ofBytes(SUGGESTED_MAX_ALLOCATION_SIZE) @@ -126,7 +130,12 @@ public class NettyAllocator { + g1gcRegionSize + "}]"; } - ALLOCATOR = new NoDirectBuffers(delegate); + allocator = new NoDirectBuffers(delegate); + } + if (Assertions.ENABLED) { + ALLOCATOR = new TrashingByteBufAllocator(allocator); + } else { + ALLOCATOR = allocator; } RECYCLER = new Recycler<>() { @@ -352,4 +361,62 @@ public ByteBufAllocator getDelegate() { return delegate; } } + + static class TrashingCompositeByteBuf extends CompositeByteBuf { + + TrashingCompositeByteBuf(ByteBufAllocator alloc, boolean direct, int maxNumComponents) { + super(alloc, direct, maxNumComponents); + } + + @Override + protected void deallocate() { + TrashingByteBufAllocator.trashBuffer(this); + super.deallocate(); + } + } + + static class TrashingByteBufAllocator extends NoDirectBuffers { + + static int DEFAULT_MAX_COMPONENTS = 16; + + static void trashBuffer(ByteBuf buf) { + for (var nioBuf : buf.nioBuffers()) { + if (nioBuf.hasArray()) { + var from = nioBuf.arrayOffset() + nioBuf.position(); + var to = from + nioBuf.remaining(); + Arrays.fill(nioBuf.array(), from, to, (byte) 0); + } + } + } + + TrashingByteBufAllocator(ByteBufAllocator delegate) { + super(delegate); + } + + @Override + public ByteBuf heapBuffer() { + return new TrashingByteBuf(super.heapBuffer()); + } + + @Override + public ByteBuf heapBuffer(int initialCapacity) { + return new TrashingByteBuf(super.heapBuffer(initialCapacity)); + } + + @Override + public ByteBuf heapBuffer(int initialCapacity, int maxCapacity) { + return new TrashingByteBuf(super.heapBuffer(initialCapacity, maxCapacity)); + } + + @Override + public CompositeByteBuf compositeHeapBuffer() { + return new TrashingCompositeByteBuf(this, false, DEFAULT_MAX_COMPONENTS); + } + + @Override + public CompositeByteBuf compositeHeapBuffer(int maxNumComponents) { + return new TrashingCompositeByteBuf(this, false, maxNumComponents); + } + + } } diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/NettyByteBufSizer.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/NettyByteBufSizer.java index f979bd8521920..2d62f8eb19e0b 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/NettyByteBufSizer.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/NettyByteBufSizer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport.netty4; diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/SSLExceptionHelper.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/SSLExceptionHelper.java index 19294c1936051..3a10656e8f79e 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/SSLExceptionHelper.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/SSLExceptionHelper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport.netty4; diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/SharedGroupFactory.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/SharedGroupFactory.java index 849597b1d9915..83a699cce8233 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/SharedGroupFactory.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/SharedGroupFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport.netty4; diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/TLSConfig.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/TLSConfig.java index b78fcd1a5a9b8..9e8d452ba2f46 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/TLSConfig.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/TLSConfig.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport.netty4; diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/TrashingByteBuf.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/TrashingByteBuf.java new file mode 100644 index 0000000000000..ead0d595f0105 --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/TrashingByteBuf.java @@ -0,0 +1,536 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.transport.netty4; + +import io.netty.buffer.ByteBuf; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +class TrashingByteBuf extends WrappedByteBuf { + + private boolean trashed = false; + + protected TrashingByteBuf(ByteBuf buf) { + super(buf); + } + + static TrashingByteBuf newBuf(ByteBuf buf) { + return new TrashingByteBuf(buf); + } + + @Override + public boolean release() { + if (refCnt() == 1) { + // see [NOTE on racy trashContent() calls] + trashContent(); + } + return super.release(); + } + + @Override + public boolean release(int decrement) { + if (refCnt() == decrement && refCnt() > 0) { + // see [NOTE on racy trashContent() calls] + trashContent(); + } + return super.release(decrement); + } + + // [NOTE on racy trashContent() calls]: We trash the buffer content _before_ reducing the ref + // count to zero, which looks racy because in principle a concurrent caller could come along + // and successfully retain() this buffer to keep it alive after it's been trashed. Such a + // caller would sometimes get an IllegalReferenceCountException ofc but that's something it + // could handle - see for instance org.elasticsearch.transport.netty4.Netty4Utils.ByteBufRefCounted.tryIncRef. + // Yet in practice this should never happen, we only ever retain() these buffers while we + // know them to be alive (i.e. via RefCounted#mustIncRef or its moral equivalents) so it'd + // be a bug for a caller to retain() a buffer whose ref count is heading to zero and whose + // contents we've already decided to trash. + private void trashContent() { + if (trashed == false) { + trashed = true; + NettyAllocator.TrashingByteBufAllocator.trashBuffer(buf); + } + } + + @Override + public ByteBuf capacity(int newCapacity) { + super.capacity(newCapacity); + return this; + } + + @Override + public ByteBuf order(ByteOrder endianness) { + return newBuf(super.order(endianness)); + } + + @Override + public ByteBuf asReadOnly() { + return newBuf(super.asReadOnly()); + } + + @Override + public ByteBuf setIndex(int readerIndex, int writerIndex) { + super.setIndex(readerIndex, writerIndex); + return this; + } + + @Override + public ByteBuf discardReadBytes() { + super.discardReadBytes(); + return this; + } + + @Override + public ByteBuf discardSomeReadBytes() { + super.discardSomeReadBytes(); + return this; + } + + @Override + public ByteBuf ensureWritable(int minWritableBytes) { + super.ensureWritable(minWritableBytes); + return this; + } + + @Override + public ByteBuf getBytes(int index, ByteBuf dst) { + super.getBytes(index, dst); + return this; + } + + @Override + public ByteBuf getBytes(int index, ByteBuf dst, int length) { + super.getBytes(index, dst, length); + return this; + } + + @Override + public ByteBuf getBytes(int index, ByteBuf dst, int dstIndex, int length) { + super.getBytes(index, dst, dstIndex, length); + return this; + } + + @Override + public ByteBuf getBytes(int index, byte[] dst) { + super.getBytes(index, dst); + return this; + } + + @Override + public ByteBuf getBytes(int index, byte[] dst, int dstIndex, int length) { + super.getBytes(index, dst, dstIndex, length); + return this; + } + + @Override + public ByteBuf getBytes(int index, ByteBuffer dst) { + super.getBytes(index, dst); + return this; + } + + @Override + public ByteBuf getBytes(int index, OutputStream out, int length) throws IOException { + super.getBytes(index, out, length); + return this; + } + + @Override + public ByteBuf setBoolean(int index, boolean value) { + super.setBoolean(index, value); + return this; + } + + @Override + public ByteBuf setByte(int index, int value) { + super.setByte(index, value); + return this; + } + + @Override + public ByteBuf setShort(int index, int value) { + super.setShort(index, value); + return this; + } + + @Override + public ByteBuf setShortLE(int index, int value) { + super.setShortLE(index, value); + return this; + } + + @Override + public ByteBuf setMedium(int index, int value) { + super.setMedium(index, value); + return this; + } + + @Override + public ByteBuf setMediumLE(int index, int value) { + super.setMediumLE(index, value); + return this; + } + + @Override + public ByteBuf setInt(int index, int value) { + super.setInt(index, value); + return this; + } + + @Override + public ByteBuf setIntLE(int index, int value) { + super.setIntLE(index, value); + return this; + } + + @Override + public ByteBuf setLong(int index, long value) { + super.setLong(index, value); + return this; + } + + @Override + public ByteBuf setLongLE(int index, long value) { + super.setLongLE(index, value); + return this; + } + + @Override + public ByteBuf setChar(int index, int value) { + super.setChar(index, value); + return this; + } + + @Override + public ByteBuf setFloat(int index, float value) { + super.setFloat(index, value); + return this; + } + + @Override + public ByteBuf setDouble(int index, double value) { + super.setDouble(index, value); + return this; + } + + @Override + public ByteBuf setBytes(int index, ByteBuf src) { + super.setBytes(index, src); + return this; + } + + @Override + public ByteBuf setBytes(int index, ByteBuf src, int length) { + super.setBytes(index, src, length); + return this; + } + + @Override + public ByteBuf setBytes(int index, ByteBuf src, int srcIndex, int length) { + super.setBytes(index, src, srcIndex, length); + return this; + } + + @Override + public ByteBuf setBytes(int index, byte[] src) { + super.setBytes(index, src); + return this; + } + + @Override + public ByteBuf setBytes(int index, byte[] src, int srcIndex, int length) { + super.setBytes(index, src, srcIndex, length); + return this; + } + + @Override + public ByteBuf setBytes(int index, ByteBuffer src) { + super.setBytes(index, src); + return this; + } + + @Override + public ByteBuf readBytes(int length) { + return newBuf(super.readBytes(length)); + } + + @Override + public ByteBuf readSlice(int length) { + return newBuf(super.readSlice(length)); + } + + @Override + public ByteBuf readRetainedSlice(int length) { + return newBuf(super.readRetainedSlice(length)); + } + + @Override + public ByteBuf readBytes(ByteBuf dst) { + super.readBytes(dst); + return this; + } + + @Override + public ByteBuf readBytes(ByteBuf dst, int length) { + super.readBytes(dst, length); + return this; + } + + @Override + public ByteBuf readBytes(ByteBuf dst, int dstIndex, int length) { + super.readBytes(dst, dstIndex, length); + return this; + } + + @Override + public ByteBuf readBytes(byte[] dst) { + super.readBytes(dst); + return this; + } + + @Override + public ByteBuf readBytes(ByteBuffer dst) { + super.readBytes(dst); + return this; + } + + @Override + public ByteBuf readBytes(byte[] dst, int dstIndex, int length) { + super.readBytes(dst, dstIndex, length); + return this; + } + + @Override + public ByteBuf readBytes(OutputStream out, int length) throws IOException { + super.readBytes(out, length); + return this; + } + + @Override + public ByteBuf skipBytes(int length) { + super.skipBytes(length); + return this; + } + + @Override + public ByteBuf writeBoolean(boolean value) { + super.writeBoolean(value); + return this; + } + + @Override + public ByteBuf writeByte(int value) { + super.writeByte(value); + return this; + } + + @Override + public ByteBuf writeShort(int value) { + super.writeShort(value); + return this; + } + + @Override + public ByteBuf writeShortLE(int value) { + super.writeShortLE(value); + return this; + } + + @Override + public ByteBuf writeMedium(int value) { + super.writeMedium(value); + return this; + } + + @Override + public ByteBuf writeMediumLE(int value) { + super.writeMediumLE(value); + return this; + } + + @Override + public ByteBuf writeInt(int value) { + super.writeInt(value); + return this; + + } + + @Override + public ByteBuf writeIntLE(int value) { + super.writeIntLE(value); + return this; + } + + @Override + public ByteBuf writeLong(long value) { + super.writeLong(value); + return this; + } + + @Override + public ByteBuf writeLongLE(long value) { + super.writeLongLE(value); + return this; + } + + @Override + public ByteBuf writeChar(int value) { + super.writeChar(value); + return this; + } + + @Override + public ByteBuf writeFloat(float value) { + super.writeFloat(value); + return this; + } + + @Override + public ByteBuf writeDouble(double value) { + super.writeDouble(value); + return this; + } + + @Override + public ByteBuf writeBytes(ByteBuf src) { + super.writeBytes(src); + return this; + } + + @Override + public ByteBuf writeBytes(ByteBuf src, int length) { + super.writeBytes(src, length); + return this; + } + + @Override + public ByteBuf writeBytes(ByteBuf src, int srcIndex, int length) { + super.writeBytes(src, srcIndex, length); + return this; + } + + @Override + public ByteBuf writeBytes(byte[] src) { + super.writeBytes(src); + return this; + } + + @Override + public ByteBuf writeBytes(byte[] src, int srcIndex, int length) { + super.writeBytes(src, srcIndex, length); + return this; + } + + @Override + public ByteBuf writeBytes(ByteBuffer src) { + super.writeBytes(src); + return this; + } + + @Override + public ByteBuf writeZero(int length) { + super.writeZero(length); + return this; + } + + @Override + public ByteBuf copy() { + return newBuf(super.copy()); + } + + @Override + public ByteBuf copy(int index, int length) { + return newBuf(super.copy(index, length)); + } + + @Override + public ByteBuf slice() { + return newBuf(super.slice()); + } + + @Override + public ByteBuf retainedSlice() { + return newBuf(super.retainedSlice()); + } + + @Override + public ByteBuf slice(int index, int length) { + return newBuf(super.slice(index, length)); + } + + @Override + public ByteBuf retainedSlice(int index, int length) { + return newBuf(super.retainedSlice(index, length)); + } + + @Override + public ByteBuf duplicate() { + return newBuf(super.duplicate()); + } + + @Override + public ByteBuf retainedDuplicate() { + return newBuf(super.retainedDuplicate()); + } + + @Override + public ByteBuf retain(int increment) { + super.retain(increment); + return this; + } + + @Override + public ByteBuf touch(Object hint) { + super.touch(hint); + return this; + } + + @Override + public ByteBuf retain() { + super.retain(); + return this; + } + + @Override + public ByteBuf touch() { + super.touch(); + return this; + } + + @Override + public ByteBuf setFloatLE(int index, float value) { + return super.setFloatLE(index, value); + } + + @Override + public ByteBuf setDoubleLE(int index, double value) { + super.setDoubleLE(index, value); + return this; + } + + @Override + public ByteBuf writeFloatLE(float value) { + super.writeFloatLE(value); + return this; + } + + @Override + public ByteBuf writeDoubleLE(double value) { + super.writeDoubleLE(value); + return this; + } + + @Override + public ByteBuf asByteBuf() { + return this; + } +} diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/WrappedByteBuf.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/WrappedByteBuf.java new file mode 100644 index 0000000000000..50841cec000f1 --- /dev/null +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/WrappedByteBuf.java @@ -0,0 +1,1036 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.transport.netty4; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufAllocator; +import io.netty.util.ByteProcessor; +import io.netty.util.internal.ObjectUtil; +import io.netty.util.internal.StringUtil; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.FileChannel; +import java.nio.channels.GatheringByteChannel; +import java.nio.channels.ScatteringByteChannel; +import java.nio.charset.Charset; + +/** + * A copy of Netty's WrappedByteBuf. + */ +class WrappedByteBuf extends ByteBuf { + + protected final ByteBuf buf; + + protected WrappedByteBuf(ByteBuf buf) { + this.buf = ObjectUtil.checkNotNull(buf, "buf"); + } + + @Override + public final boolean hasMemoryAddress() { + return buf.hasMemoryAddress(); + } + + @Override + public boolean isContiguous() { + return buf.isContiguous(); + } + + @Override + public final long memoryAddress() { + return buf.memoryAddress(); + } + + @Override + public final int capacity() { + return buf.capacity(); + } + + @Override + public ByteBuf capacity(int newCapacity) { + buf.capacity(newCapacity); + return this; + } + + @Override + public final int maxCapacity() { + return buf.maxCapacity(); + } + + @Override + public final ByteBufAllocator alloc() { + return buf.alloc(); + } + + @Override + public final ByteOrder order() { + return buf.order(); + } + + @Override + public ByteBuf order(ByteOrder endianness) { + return buf.order(endianness); + } + + @Override + public final ByteBuf unwrap() { + return buf; + } + + @Override + public ByteBuf asReadOnly() { + return buf.asReadOnly(); + } + + @Override + public boolean isReadOnly() { + return buf.isReadOnly(); + } + + @Override + public final boolean isDirect() { + return buf.isDirect(); + } + + @Override + public final int readerIndex() { + return buf.readerIndex(); + } + + @Override + public final ByteBuf readerIndex(int readerIndex) { + buf.readerIndex(readerIndex); + return this; + } + + @Override + public final int writerIndex() { + return buf.writerIndex(); + } + + @Override + public final ByteBuf writerIndex(int writerIndex) { + buf.writerIndex(writerIndex); + return this; + } + + @Override + public ByteBuf setIndex(int readerIndex, int writerIndex) { + buf.setIndex(readerIndex, writerIndex); + return this; + } + + @Override + public final int readableBytes() { + return buf.readableBytes(); + } + + @Override + public final int writableBytes() { + return buf.writableBytes(); + } + + @Override + public final int maxWritableBytes() { + return buf.maxWritableBytes(); + } + + @Override + public int maxFastWritableBytes() { + return buf.maxFastWritableBytes(); + } + + @Override + public final boolean isReadable() { + return buf.isReadable(); + } + + @Override + public final boolean isWritable() { + return buf.isWritable(); + } + + @Override + public final ByteBuf clear() { + buf.clear(); + return this; + } + + @Override + public final ByteBuf markReaderIndex() { + buf.markReaderIndex(); + return this; + } + + @Override + public final ByteBuf resetReaderIndex() { + buf.resetReaderIndex(); + return this; + } + + @Override + public final ByteBuf markWriterIndex() { + buf.markWriterIndex(); + return this; + } + + @Override + public final ByteBuf resetWriterIndex() { + buf.resetWriterIndex(); + return this; + } + + @Override + public ByteBuf discardReadBytes() { + buf.discardReadBytes(); + return this; + } + + @Override + public ByteBuf discardSomeReadBytes() { + buf.discardSomeReadBytes(); + return this; + } + + @Override + public ByteBuf ensureWritable(int minWritableBytes) { + buf.ensureWritable(minWritableBytes); + return this; + } + + @Override + public int ensureWritable(int minWritableBytes, boolean force) { + return buf.ensureWritable(minWritableBytes, force); + } + + @Override + public boolean getBoolean(int index) { + return buf.getBoolean(index); + } + + @Override + public byte getByte(int index) { + return buf.getByte(index); + } + + @Override + public short getUnsignedByte(int index) { + return buf.getUnsignedByte(index); + } + + @Override + public short getShort(int index) { + return buf.getShort(index); + } + + @Override + public short getShortLE(int index) { + return buf.getShortLE(index); + } + + @Override + public int getUnsignedShort(int index) { + return buf.getUnsignedShort(index); + } + + @Override + public int getUnsignedShortLE(int index) { + return buf.getUnsignedShortLE(index); + } + + @Override + public int getMedium(int index) { + return buf.getMedium(index); + } + + @Override + public int getMediumLE(int index) { + return buf.getMediumLE(index); + } + + @Override + public int getUnsignedMedium(int index) { + return buf.getUnsignedMedium(index); + } + + @Override + public int getUnsignedMediumLE(int index) { + return buf.getUnsignedMediumLE(index); + } + + @Override + public int getInt(int index) { + return buf.getInt(index); + } + + @Override + public int getIntLE(int index) { + return buf.getIntLE(index); + } + + @Override + public long getUnsignedInt(int index) { + return buf.getUnsignedInt(index); + } + + @Override + public long getUnsignedIntLE(int index) { + return buf.getUnsignedIntLE(index); + } + + @Override + public long getLong(int index) { + return buf.getLong(index); + } + + @Override + public long getLongLE(int index) { + return buf.getLongLE(index); + } + + @Override + public char getChar(int index) { + return buf.getChar(index); + } + + @Override + public float getFloat(int index) { + return buf.getFloat(index); + } + + @Override + public double getDouble(int index) { + return buf.getDouble(index); + } + + @Override + public ByteBuf getBytes(int index, ByteBuf dst) { + buf.getBytes(index, dst); + return this; + } + + @Override + public ByteBuf getBytes(int index, ByteBuf dst, int length) { + buf.getBytes(index, dst, length); + return this; + } + + @Override + public ByteBuf getBytes(int index, ByteBuf dst, int dstIndex, int length) { + buf.getBytes(index, dst, dstIndex, length); + return this; + } + + @Override + public ByteBuf getBytes(int index, byte[] dst) { + buf.getBytes(index, dst); + return this; + } + + @Override + public ByteBuf getBytes(int index, byte[] dst, int dstIndex, int length) { + buf.getBytes(index, dst, dstIndex, length); + return this; + } + + @Override + public ByteBuf getBytes(int index, ByteBuffer dst) { + buf.getBytes(index, dst); + return this; + } + + @Override + public ByteBuf getBytes(int index, OutputStream out, int length) throws IOException { + buf.getBytes(index, out, length); + return this; + } + + @Override + public int getBytes(int index, GatheringByteChannel out, int length) throws IOException { + return buf.getBytes(index, out, length); + } + + @Override + public int getBytes(int index, FileChannel out, long position, int length) throws IOException { + return buf.getBytes(index, out, position, length); + } + + @Override + public CharSequence getCharSequence(int index, int length, Charset charset) { + return buf.getCharSequence(index, length, charset); + } + + @Override + public ByteBuf setBoolean(int index, boolean value) { + buf.setBoolean(index, value); + return this; + } + + @Override + public ByteBuf setByte(int index, int value) { + buf.setByte(index, value); + return this; + } + + @Override + public ByteBuf setShort(int index, int value) { + buf.setShort(index, value); + return this; + } + + @Override + public ByteBuf setShortLE(int index, int value) { + buf.setShortLE(index, value); + return this; + } + + @Override + public ByteBuf setMedium(int index, int value) { + buf.setMedium(index, value); + return this; + } + + @Override + public ByteBuf setMediumLE(int index, int value) { + buf.setMediumLE(index, value); + return this; + } + + @Override + public ByteBuf setInt(int index, int value) { + buf.setInt(index, value); + return this; + } + + @Override + public ByteBuf setIntLE(int index, int value) { + buf.setIntLE(index, value); + return this; + } + + @Override + public ByteBuf setLong(int index, long value) { + buf.setLong(index, value); + return this; + } + + @Override + public ByteBuf setLongLE(int index, long value) { + buf.setLongLE(index, value); + return this; + } + + @Override + public ByteBuf setChar(int index, int value) { + buf.setChar(index, value); + return this; + } + + @Override + public ByteBuf setFloat(int index, float value) { + buf.setFloat(index, value); + return this; + } + + @Override + public ByteBuf setDouble(int index, double value) { + buf.setDouble(index, value); + return this; + } + + @Override + public ByteBuf setBytes(int index, ByteBuf src) { + buf.setBytes(index, src); + return this; + } + + @Override + public ByteBuf setBytes(int index, ByteBuf src, int length) { + buf.setBytes(index, src, length); + return this; + } + + @Override + public ByteBuf setBytes(int index, ByteBuf src, int srcIndex, int length) { + buf.setBytes(index, src, srcIndex, length); + return this; + } + + @Override + public ByteBuf setBytes(int index, byte[] src) { + buf.setBytes(index, src); + return this; + } + + @Override + public ByteBuf setBytes(int index, byte[] src, int srcIndex, int length) { + buf.setBytes(index, src, srcIndex, length); + return this; + } + + @Override + public ByteBuf setBytes(int index, ByteBuffer src) { + buf.setBytes(index, src); + return this; + } + + @Override + public int setBytes(int index, InputStream in, int length) throws IOException { + return buf.setBytes(index, in, length); + } + + @Override + public int setBytes(int index, ScatteringByteChannel in, int length) throws IOException { + return buf.setBytes(index, in, length); + } + + @Override + public int setBytes(int index, FileChannel in, long position, int length) throws IOException { + return buf.setBytes(index, in, position, length); + } + + @Override + public ByteBuf setZero(int index, int length) { + buf.setZero(index, length); + return this; + } + + @Override + public int setCharSequence(int index, CharSequence sequence, Charset charset) { + return buf.setCharSequence(index, sequence, charset); + } + + @Override + public boolean readBoolean() { + return buf.readBoolean(); + } + + @Override + public byte readByte() { + return buf.readByte(); + } + + @Override + public short readUnsignedByte() { + return buf.readUnsignedByte(); + } + + @Override + public short readShort() { + return buf.readShort(); + } + + @Override + public short readShortLE() { + return buf.readShortLE(); + } + + @Override + public int readUnsignedShort() { + return buf.readUnsignedShort(); + } + + @Override + public int readUnsignedShortLE() { + return buf.readUnsignedShortLE(); + } + + @Override + public int readMedium() { + return buf.readMedium(); + } + + @Override + public int readMediumLE() { + return buf.readMediumLE(); + } + + @Override + public int readUnsignedMedium() { + return buf.readUnsignedMedium(); + } + + @Override + public int readUnsignedMediumLE() { + return buf.readUnsignedMediumLE(); + } + + @Override + public int readInt() { + return buf.readInt(); + } + + @Override + public int readIntLE() { + return buf.readIntLE(); + } + + @Override + public long readUnsignedInt() { + return buf.readUnsignedInt(); + } + + @Override + public long readUnsignedIntLE() { + return buf.readUnsignedIntLE(); + } + + @Override + public long readLong() { + return buf.readLong(); + } + + @Override + public long readLongLE() { + return buf.readLongLE(); + } + + @Override + public char readChar() { + return buf.readChar(); + } + + @Override + public float readFloat() { + return buf.readFloat(); + } + + @Override + public double readDouble() { + return buf.readDouble(); + } + + @Override + public ByteBuf readBytes(int length) { + return buf.readBytes(length); + } + + @Override + public ByteBuf readSlice(int length) { + return buf.readSlice(length); + } + + @Override + public ByteBuf readRetainedSlice(int length) { + return buf.readRetainedSlice(length); + } + + @Override + public ByteBuf readBytes(ByteBuf dst) { + buf.readBytes(dst); + return this; + } + + @Override + public ByteBuf readBytes(ByteBuf dst, int length) { + buf.readBytes(dst, length); + return this; + } + + @Override + public ByteBuf readBytes(ByteBuf dst, int dstIndex, int length) { + buf.readBytes(dst, dstIndex, length); + return this; + } + + @Override + public ByteBuf readBytes(byte[] dst) { + buf.readBytes(dst); + return this; + } + + @Override + public ByteBuf readBytes(byte[] dst, int dstIndex, int length) { + buf.readBytes(dst, dstIndex, length); + return this; + } + + @Override + public ByteBuf readBytes(ByteBuffer dst) { + buf.readBytes(dst); + return this; + } + + @Override + public ByteBuf readBytes(OutputStream out, int length) throws IOException { + buf.readBytes(out, length); + return this; + } + + @Override + public int readBytes(GatheringByteChannel out, int length) throws IOException { + return buf.readBytes(out, length); + } + + @Override + public int readBytes(FileChannel out, long position, int length) throws IOException { + return buf.readBytes(out, position, length); + } + + @Override + public CharSequence readCharSequence(int length, Charset charset) { + return buf.readCharSequence(length, charset); + } + + @Override + public ByteBuf skipBytes(int length) { + buf.skipBytes(length); + return this; + } + + @Override + public ByteBuf writeBoolean(boolean value) { + buf.writeBoolean(value); + return this; + } + + @Override + public ByteBuf writeByte(int value) { + buf.writeByte(value); + return this; + } + + @Override + public ByteBuf writeShort(int value) { + buf.writeShort(value); + return this; + } + + @Override + public ByteBuf writeShortLE(int value) { + buf.writeShortLE(value); + return this; + } + + @Override + public ByteBuf writeMedium(int value) { + buf.writeMedium(value); + return this; + } + + @Override + public ByteBuf writeMediumLE(int value) { + buf.writeMediumLE(value); + return this; + } + + @Override + public ByteBuf writeInt(int value) { + buf.writeInt(value); + return this; + } + + @Override + public ByteBuf writeIntLE(int value) { + buf.writeIntLE(value); + return this; + } + + @Override + public ByteBuf writeLong(long value) { + buf.writeLong(value); + return this; + } + + @Override + public ByteBuf writeLongLE(long value) { + buf.writeLongLE(value); + return this; + } + + @Override + public ByteBuf writeChar(int value) { + buf.writeChar(value); + return this; + } + + @Override + public ByteBuf writeFloat(float value) { + buf.writeFloat(value); + return this; + } + + @Override + public ByteBuf writeDouble(double value) { + buf.writeDouble(value); + return this; + } + + @Override + public ByteBuf writeBytes(ByteBuf src) { + buf.writeBytes(src); + return this; + } + + @Override + public ByteBuf writeBytes(ByteBuf src, int length) { + buf.writeBytes(src, length); + return this; + } + + @Override + public ByteBuf writeBytes(ByteBuf src, int srcIndex, int length) { + buf.writeBytes(src, srcIndex, length); + return this; + } + + @Override + public ByteBuf writeBytes(byte[] src) { + buf.writeBytes(src); + return this; + } + + @Override + public ByteBuf writeBytes(byte[] src, int srcIndex, int length) { + buf.writeBytes(src, srcIndex, length); + return this; + } + + @Override + public ByteBuf writeBytes(ByteBuffer src) { + buf.writeBytes(src); + return this; + } + + @Override + public int writeBytes(InputStream in, int length) throws IOException { + return buf.writeBytes(in, length); + } + + @Override + public int writeBytes(ScatteringByteChannel in, int length) throws IOException { + return buf.writeBytes(in, length); + } + + @Override + public int writeBytes(FileChannel in, long position, int length) throws IOException { + return buf.writeBytes(in, position, length); + } + + @Override + public ByteBuf writeZero(int length) { + buf.writeZero(length); + return this; + } + + @Override + public int writeCharSequence(CharSequence sequence, Charset charset) { + return buf.writeCharSequence(sequence, charset); + } + + @Override + public int indexOf(int fromIndex, int toIndex, byte value) { + return buf.indexOf(fromIndex, toIndex, value); + } + + @Override + public int bytesBefore(byte value) { + return buf.bytesBefore(value); + } + + @Override + public int bytesBefore(int length, byte value) { + return buf.bytesBefore(length, value); + } + + @Override + public int bytesBefore(int index, int length, byte value) { + return buf.bytesBefore(index, length, value); + } + + @Override + public int forEachByte(ByteProcessor processor) { + return buf.forEachByte(processor); + } + + @Override + public int forEachByte(int index, int length, ByteProcessor processor) { + return buf.forEachByte(index, length, processor); + } + + @Override + public int forEachByteDesc(ByteProcessor processor) { + return buf.forEachByteDesc(processor); + } + + @Override + public int forEachByteDesc(int index, int length, ByteProcessor processor) { + return buf.forEachByteDesc(index, length, processor); + } + + @Override + public ByteBuf copy() { + return buf.copy(); + } + + @Override + public ByteBuf copy(int index, int length) { + return buf.copy(index, length); + } + + @Override + public ByteBuf slice() { + return buf.slice(); + } + + @Override + public ByteBuf retainedSlice() { + return buf.retainedSlice(); + } + + @Override + public ByteBuf slice(int index, int length) { + return buf.slice(index, length); + } + + @Override + public ByteBuf retainedSlice(int index, int length) { + return buf.retainedSlice(index, length); + } + + @Override + public ByteBuf duplicate() { + return buf.duplicate(); + } + + @Override + public ByteBuf retainedDuplicate() { + return buf.retainedDuplicate(); + } + + @Override + public int nioBufferCount() { + return buf.nioBufferCount(); + } + + @Override + public ByteBuffer nioBuffer() { + return buf.nioBuffer(); + } + + @Override + public ByteBuffer nioBuffer(int index, int length) { + return buf.nioBuffer(index, length); + } + + @Override + public ByteBuffer[] nioBuffers() { + return buf.nioBuffers(); + } + + @Override + public ByteBuffer[] nioBuffers(int index, int length) { + return buf.nioBuffers(index, length); + } + + @Override + public ByteBuffer internalNioBuffer(int index, int length) { + return buf.internalNioBuffer(index, length); + } + + @Override + public boolean hasArray() { + return buf.hasArray(); + } + + @Override + public byte[] array() { + return buf.array(); + } + + @Override + public int arrayOffset() { + return buf.arrayOffset(); + } + + @Override + public String toString(Charset charset) { + return buf.toString(charset); + } + + @Override + public String toString(int index, int length, Charset charset) { + return buf.toString(index, length, charset); + } + + @Override + public int hashCode() { + return buf.hashCode(); + } + + @Override + @SuppressWarnings("EqualsWhichDoesntCheckParameterClass") + public boolean equals(Object obj) { + return buf.equals(obj); + } + + @Override + public int compareTo(ByteBuf buffer) { + return buf.compareTo(buffer); + } + + @Override + public String toString() { + return StringUtil.simpleClassName(this) + '(' + buf.toString() + ')'; + } + + @Override + public ByteBuf retain(int increment) { + buf.retain(increment); + return this; + } + + @Override + public ByteBuf retain() { + buf.retain(); + return this; + } + + @Override + public ByteBuf touch() { + buf.touch(); + return this; + } + + @Override + public ByteBuf touch(Object hint) { + buf.touch(hint); + return this; + } + + @Override + public final boolean isReadable(int size) { + return buf.isReadable(size); + } + + @Override + public final boolean isWritable(int size) { + return buf.isWritable(size); + } + + @Override + public final int refCnt() { + return buf.refCnt(); + } + + @Override + public boolean release() { + return buf.release(); + } + + @Override + public boolean release(int decrement) { + return buf.release(decrement); + } + +} diff --git a/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy b/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy index a70fe690a53a4..ed278af96d926 100644 --- a/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy +++ b/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ grant codeBase "${codebase.netty-common}" { diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/HttpHeadersAuthenticatorUtilsTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/HttpHeadersAuthenticatorUtilsTests.java index 2151d2f1a9f41..aff4c5cccbe6d 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/HttpHeadersAuthenticatorUtilsTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/HttpHeadersAuthenticatorUtilsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.netty4; diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4BadRequestTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4BadRequestTests.java index 0b00c7d9e8ff8..5795df2e288d8 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4BadRequestTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4BadRequestTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.netty4; diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java index 3035213766584..02197995c7106 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.netty4; diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpHeaderThreadContextTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpHeaderThreadContextTests.java index aa54bb9fc965c..9a12ba75d7742 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpHeaderThreadContextTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpHeaderThreadContextTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.netty4; diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpHeaderValidatorTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpHeaderValidatorTests.java index e8622f2c95c2c..1c0b434105f28 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpHeaderValidatorTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpHeaderValidatorTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.netty4; @@ -116,6 +117,36 @@ public void testValidationPausesAndResumesData() { assertThat(netty4HttpHeaderValidator.getState(), equalTo(QUEUEING_DATA)); } + public void testValidatorDoesNotTweakAutoReadAfterValidationComplete() { + assertTrue(channel.config().isAutoRead()); + assertThat(netty4HttpHeaderValidator.getState(), equalTo(WAITING_TO_START)); + + final DefaultHttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/uri"); + DefaultHttpContent content = new DefaultHttpContent(Unpooled.buffer(4)); + channel.writeInbound(request); + channel.writeInbound(content); + + assertThat(header.get(), sameInstance(request)); + // channel is paused + assertThat(channel.readInbound(), nullValue()); + assertFalse(channel.config().isAutoRead()); + + // channel is resumed + listener.get().onResponse(null); + channel.runPendingTasks(); + + assertTrue(channel.config().isAutoRead()); + assertThat(netty4HttpHeaderValidator.getState(), equalTo(FORWARDING_DATA_UNTIL_NEXT_REQUEST)); + assertThat(channel.readInbound(), sameInstance(request)); + assertThat(channel.readInbound(), sameInstance(content)); + assertThat(channel.readInbound(), nullValue()); + assertThat(content.refCnt(), equalTo(1)); + channel.config().setAutoRead(false); + + channel.writeOutbound(new DefaultHttpContent(Unpooled.buffer(4))); + assertFalse(channel.config().isAutoRead()); + } + public void testContentForwardedAfterValidation() { assertTrue(channel.config().isAutoRead()); assertThat(netty4HttpHeaderValidator.getState(), equalTo(WAITING_TO_START)); diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandlerTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandlerTests.java index b2158384fa1cf..f5f37889054dc 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandlerTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandlerTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.netty4; diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStreamTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStreamTests.java new file mode 100644 index 0000000000000..d456bbecfbd20 --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStreamTests.java @@ -0,0 +1,201 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.http.netty4; + +import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.embedded.EmbeddedChannel; +import io.netty.handler.codec.http.DefaultHttpContent; +import io.netty.handler.codec.http.DefaultLastHttpContent; +import io.netty.handler.codec.http.HttpContent; +import io.netty.handler.flow.FlowControlHandler; + +import org.elasticsearch.common.bytes.ReleasableBytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.http.HttpBody; +import org.elasticsearch.test.ESTestCase; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.hamcrest.Matchers.hasEntry; + +public class Netty4HttpRequestBodyStreamTests extends ESTestCase { + + private final ThreadContext threadContext = new ThreadContext(Settings.EMPTY); + private EmbeddedChannel channel; + private Netty4HttpRequestBodyStream stream; + static HttpBody.ChunkHandler discardHandler = (chunk, isLast) -> chunk.close(); + + @Override + public void setUp() throws Exception { + super.setUp(); + channel = new EmbeddedChannel(); + threadContext.putHeader("header1", "value1"); + stream = new Netty4HttpRequestBodyStream(channel, threadContext); + stream.setHandler(discardHandler); // set default handler, each test might override one + channel.pipeline().addLast(new SimpleChannelInboundHandler(false) { + @Override + protected void channelRead0(ChannelHandlerContext ctx, HttpContent msg) { + stream.handleNettyContent(msg); + } + }); + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + stream.close(); + } + + // ensures that no chunks are sent downstream without request + public void testEnqueueChunksBeforeRequest() { + var totalChunks = randomIntBetween(1, 100); + for (int i = 0; i < totalChunks; i++) { + channel.writeInbound(randomContent(1024)); + } + assertEquals(totalChunks * 1024, stream.bufSize()); + } + + // ensures all received chunks can be flushed downstream + public void testFlushAllReceivedChunks() { + var chunks = new ArrayList(); + var totalBytes = new AtomicInteger(); + stream.setHandler((chunk, isLast) -> { + chunks.add(chunk); + totalBytes.addAndGet(chunk.length()); + chunk.close(); + }); + + var chunkSize = 1024; + var totalChunks = randomIntBetween(1, 100); + for (int i = 0; i < totalChunks; i++) { + channel.writeInbound(randomContent(chunkSize)); + } + stream.next(); + channel.runPendingTasks(); + assertEquals("should receive all chunks as single composite", 1, chunks.size()); + assertEquals(chunkSize * totalChunks, totalBytes.get()); + } + + // ensures that channel.setAutoRead(true) only when we flush last chunk + public void testSetAutoReadOnLastFlush() { + channel.writeInbound(randomLastContent(10)); + assertFalse("should not auto-read on last content reception", channel.config().isAutoRead()); + stream.next(); + channel.runPendingTasks(); + assertTrue("should set auto-read once last content is flushed", channel.config().isAutoRead()); + } + + // ensures that we read from channel when no current chunks available + // and pass next chunk downstream without holding + public void testReadFromChannel() { + var gotChunks = new ArrayList(); + var gotLast = new AtomicBoolean(false); + stream.setHandler((chunk, isLast) -> { + gotChunks.add(chunk); + gotLast.set(isLast); + chunk.close(); + }); + channel.pipeline().addFirst(new FlowControlHandler()); // block all incoming messages, need explicit channel.read() + var chunkSize = 1024; + var totalChunks = randomIntBetween(1, 32); + for (int i = 0; i < totalChunks - 1; i++) { + channel.writeInbound(randomContent(chunkSize)); + } + channel.writeInbound(randomLastContent(chunkSize)); + + for (int i = 0; i < totalChunks; i++) { + assertEquals("should not enqueue chunks", 0, stream.bufSize()); + stream.next(); + channel.runPendingTasks(); + assertEquals("each next() should produce single chunk", i + 1, gotChunks.size()); + } + assertTrue("should receive last content", gotLast.get()); + } + + public void testReadFromHasCorrectThreadContext() throws InterruptedException { + var gotLast = new AtomicBoolean(false); + AtomicReference> headers = new AtomicReference<>(); + stream.setHandler(new HttpBody.ChunkHandler() { + @Override + public void onNext(ReleasableBytesReference chunk, boolean isLast) { + headers.set(threadContext.getHeaders()); + gotLast.set(isLast); + chunk.close(); + } + + @Override + public void close() { + headers.set(threadContext.getHeaders()); + } + }); + channel.pipeline().addFirst(new FlowControlHandler()); // block all incoming messages, need explicit channel.read() + var chunkSize = 1024; + + channel.writeInbound(randomContent(chunkSize)); + channel.writeInbound(randomLastContent(chunkSize)); + + threadContext.putHeader("header2", "value2"); + stream.next(); + + Thread thread = new Thread(() -> channel.runPendingTasks()); + thread.start(); + thread.join(); + + assertThat(headers.get(), hasEntry("header1", "value1")); + assertThat(headers.get(), hasEntry("header2", "value2")); + + threadContext.putHeader("header3", "value3"); + stream.next(); + + thread = new Thread(() -> channel.runPendingTasks()); + thread.start(); + thread.join(); + + assertThat(headers.get(), hasEntry("header1", "value1")); + assertThat(headers.get(), hasEntry("header2", "value2")); + assertThat(headers.get(), hasEntry("header3", "value3")); + + assertTrue("should receive last content", gotLast.get()); + + headers.set(new HashMap<>()); + + stream.close(); + + assertThat(headers.get(), hasEntry("header1", "value1")); + assertThat(headers.get(), hasEntry("header2", "value2")); + assertThat(headers.get(), hasEntry("header3", "value3")); + } + + HttpContent randomContent(int size, boolean isLast) { + var buf = Unpooled.wrappedBuffer(randomByteArrayOfLength(size)); + if (isLast) { + return new DefaultLastHttpContent(buf); + } else { + return new DefaultHttpContent(buf); + } + } + + HttpContent randomContent(int size) { + return randomContent(size, false); + } + + HttpContent randomLastContent(int size) { + return randomContent(size, true); + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java index a31e2c36062f7..91e9f998ef473 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.netty4; diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java index bc6e5fef834e8..1d39b993cef92 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.netty4; @@ -39,13 +40,16 @@ import io.netty.handler.codec.http.HttpUtil; import io.netty.handler.codec.http.HttpVersion; +import org.apache.http.ConnectionClosedException; import org.apache.http.HttpHost; import org.apache.lucene.util.SetOnce; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.ElasticsearchWrapperException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.bulk.IncrementalBulkService; import org.elasticsearch.action.support.ActionTestUtils; +import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.client.Request; import org.elasticsearch.client.RestClient; @@ -98,6 +102,7 @@ import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.concurrent.CancellationException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -108,6 +113,7 @@ import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom; import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN; import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ENABLED; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_SERVER_SHUTDOWN_GRACE_PERIOD; import static org.elasticsearch.rest.RestStatus.BAD_REQUEST; import static org.elasticsearch.rest.RestStatus.OK; import static org.elasticsearch.rest.RestStatus.UNAUTHORIZED; @@ -418,7 +424,8 @@ public ChannelHandler configureServerChannelHandler() { handlingSettings, TLSConfig.noTLS(), null, - randomFrom((httpPreRequest, channel, listener) -> listener.onResponse(null), null) + randomFrom((httpPreRequest, channel, listener) -> listener.onResponse(null), null), + new IncrementalBulkService.Enabled(clusterSettings) ) { @Override protected void initChannel(Channel ch) throws Exception { @@ -904,7 +911,7 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th assertThat(channel.request().getHttpRequest().header(headerReference.get()), is(headerValueReference.get())); assertThat(channel.request().getHttpRequest().method(), is(translateRequestMethod(httpMethodReference.get()))); // assert content is dropped - assertThat(channel.request().getHttpRequest().content().utf8ToString(), is("")); + assertThat(channel.request().getHttpRequest().body().asFull().bytes().utf8ToString(), is("")); try { channel.sendResponse(new RestResponse(channel, (Exception) ((ElasticsearchWrapperException) cause).getCause())); } catch (IOException e) { @@ -1036,8 +1043,16 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th } } - public void testRespondAfterClose() throws Exception { - final String url = "/thing"; + public void testRespondAfterServiceCloseWithClientCancel() throws Exception { + runRespondAfterServiceCloseTest(true); + } + + public void testRespondAfterServiceCloseWithServerCancel() throws Exception { + runRespondAfterServiceCloseTest(false); + } + + private void runRespondAfterServiceCloseTest(boolean clientCancel) throws Exception { + final String url = "/" + randomIdentifier(); final CountDownLatch responseReleasedLatch = new CountDownLatch(1); final SubscribableListener transportClosedFuture = new SubscribableListener<>(); final CountDownLatch handlingRequestLatch = new CountDownLatch(1); @@ -1063,7 +1078,9 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th try ( Netty4HttpServerTransport transport = new Netty4HttpServerTransport( - Settings.EMPTY, + clientCancel + ? Settings.EMPTY + : Settings.builder().put(SETTING_HTTP_SERVER_SHUTDOWN_GRACE_PERIOD.getKey(), TimeValue.timeValueMillis(1)).build(), networkService, threadPool, xContentRegistry(), @@ -1079,11 +1096,24 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th transport.start(); final var address = randomFrom(transport.boundAddress().boundAddresses()).address(); try (var client = RestClient.builder(new HttpHost(address.getAddress(), address.getPort())).build()) { - client.performRequestAsync(new Request("GET", url), ActionTestUtils.wrapAsRestResponseListener(ActionListener.noop())); + final var responseExceptionFuture = new PlainActionFuture(); + final var cancellable = client.performRequestAsync( + new Request("GET", url), + ActionTestUtils.wrapAsRestResponseListener(ActionTestUtils.assertNoSuccessListener(responseExceptionFuture::onResponse)) + ); safeAwait(handlingRequestLatch); + if (clientCancel) { + threadPool.generic().execute(cancellable::cancel); + } transport.close(); transportClosedFuture.onResponse(null); safeAwait(responseReleasedLatch); + final var responseException = safeGet(responseExceptionFuture); + if (clientCancel) { + assertThat(responseException, instanceOf(CancellationException.class)); + } else { + assertThat(responseException, instanceOf(ConnectionClosedException.class)); + } } } } diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/CopyBytesSocketChannelTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/CopyBytesSocketChannelTests.java index 2f0440e9df97a..2d4bf4ab13e55 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/CopyBytesSocketChannelTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/CopyBytesSocketChannelTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport.netty4; diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NetUtilsTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NetUtilsTests.java index 1a4e7b3fc1565..2647a42308e56 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NetUtilsTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NetUtilsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport.netty4; diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java index ce7704e6e040c..03cc962a243e4 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport.netty4; diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4UtilsTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4UtilsTests.java index 5edfda5cd1bac..3844953628777 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4UtilsTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4UtilsTests.java @@ -1,15 +1,17 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport.netty4; import io.netty.buffer.ByteBuf; import io.netty.buffer.CompositeByteBuf; +import io.netty.buffer.PooledByteBufAllocator; import io.netty.buffer.Unpooled; import org.apache.lucene.util.BytesRef; @@ -67,6 +69,43 @@ public void testToChannelBuffer() throws IOException { assertArrayEquals(BytesReference.toBytes(ref), BytesReference.toBytes(bytesReference)); } + /** + * Test that wrapped reference counted object from netty reflects correct counts in ES RefCounted + */ + public void testToRefCounted() { + var buf = PooledByteBufAllocator.DEFAULT.buffer(1); + assertEquals(1, buf.refCnt()); + + var refCounted = Netty4Utils.toRefCounted(buf); + assertEquals(1, refCounted.refCnt()); + + buf.retain(); + assertEquals(2, refCounted.refCnt()); + + refCounted.incRef(); + assertEquals(3, refCounted.refCnt()); + assertEquals(buf.refCnt(), refCounted.refCnt()); + + refCounted.decRef(); + assertEquals(2, refCounted.refCnt()); + assertEquals(buf.refCnt(), refCounted.refCnt()); + assertTrue(refCounted.hasReferences()); + + refCounted.decRef(); + refCounted.decRef(); + assertFalse(refCounted.hasReferences()); + } + + /** + * Ensures that released ByteBuf cannot be accessed from ReleasableBytesReference + */ + public void testToReleasableBytesReferenceThrowOnByteBufRelease() { + var buf = PooledByteBufAllocator.DEFAULT.buffer(1); + var relBytes = Netty4Utils.toReleasableBytesReference(buf); + buf.release(); + assertThrows(AssertionError.class, () -> relBytes.get(0)); + } + private BytesReference getRandomizedBytesReference(int length) throws IOException { // we know bytes stream output always creates a paged bytes reference, we use it to create randomized content ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(length, bigarrays); diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4WriteThrottlingHandlerTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4WriteThrottlingHandlerTests.java index 8ac39d925f566..d87889c6a2416 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4WriteThrottlingHandlerTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4WriteThrottlingHandlerTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport.netty4; @@ -17,43 +18,52 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.CompositeBytesReference; +import org.elasticsearch.common.network.ThreadWatchdog; +import org.elasticsearch.common.network.ThreadWatchdogHelper; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.transport.Transports; import org.junit.After; import org.junit.Before; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.ExecutionException; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.emptyIterable; +import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.oneOf; +import static org.hamcrest.Matchers.startsWith; public class Netty4WriteThrottlingHandlerTests extends ESTestCase { - private SharedGroupFactory.SharedGroup transportGroup; + private ThreadWatchdog threadWatchdog = new ThreadWatchdog(); @Before - public void createGroup() { - final SharedGroupFactory sharedGroupFactory = new SharedGroupFactory(Settings.EMPTY); - transportGroup = sharedGroupFactory.getTransportGroup(); + public void setFakeThreadName() { + // These tests interact with EmbeddedChannel instances directly on the test thread, so we rename it temporarily to satisfy checks + // that we're running on a transport thread + Thread.currentThread().setName(Transports.TEST_MOCK_TRANSPORT_THREAD_PREFIX + Thread.currentThread().getName()); } @After - public void stopGroup() { - transportGroup.shutdown(); + public void resetThreadName() { + final var threadName = Thread.currentThread().getName(); + assertThat(threadName, startsWith(Transports.TEST_MOCK_TRANSPORT_THREAD_PREFIX)); + Thread.currentThread().setName(threadName.substring(Transports.TEST_MOCK_TRANSPORT_THREAD_PREFIX.length())); } - public void testThrottlesLargeMessage() throws ExecutionException, InterruptedException { + public void testThrottlesLargeMessage() { final List seen = new CopyOnWriteArrayList<>(); final CapturingHandler capturingHandler = new CapturingHandler(seen); final EmbeddedChannel embeddedChannel = new EmbeddedChannel( capturingHandler, - new Netty4WriteThrottlingHandler(new ThreadContext(Settings.EMPTY)) + new Netty4WriteThrottlingHandler(new ThreadContext(Settings.EMPTY), threadWatchdog.getActivityTrackerForCurrentThread()) ); // we assume that the channel outbound buffer is smaller than Netty4WriteThrottlingHandler.MAX_BYTES_PER_WRITE final int writeableBytes = Math.toIntExact(embeddedChannel.bytesBeforeUnwritable()); @@ -65,11 +75,11 @@ public void testThrottlesLargeMessage() throws ExecutionException, InterruptedEx ); final Object message = wrapAsNettyOrEsBuffer(messageBytes); final ChannelPromise promise = embeddedChannel.newPromise(); - transportGroup.getLowLevelGroup().submit(() -> embeddedChannel.write(message, promise)).get(); + embeddedChannel.write(message, promise); assertThat(seen, hasSize(1)); assertSliceEquals(seen.get(0), message, 0, Netty4WriteThrottlingHandler.MAX_BYTES_PER_WRITE); assertFalse(promise.isDone()); - transportGroup.getLowLevelGroup().submit(embeddedChannel::flush).get(); + embeddedChannel.flush(); assertTrue(promise.isDone()); assertThat(seen, hasSize(fullSizeChunks + (extraChunkSize == 0 ? 0 : 1))); assertTrue(capturingHandler.didWriteAfterThrottled); @@ -83,12 +93,12 @@ public void testThrottlesLargeMessage() throws ExecutionException, InterruptedEx } } - public void testThrottleLargeCompositeMessage() throws ExecutionException, InterruptedException { + public void testThrottleLargeCompositeMessage() { final List seen = new CopyOnWriteArrayList<>(); final CapturingHandler capturingHandler = new CapturingHandler(seen); final EmbeddedChannel embeddedChannel = new EmbeddedChannel( capturingHandler, - new Netty4WriteThrottlingHandler(new ThreadContext(Settings.EMPTY)) + new Netty4WriteThrottlingHandler(new ThreadContext(Settings.EMPTY), threadWatchdog.getActivityTrackerForCurrentThread()) ); // we assume that the channel outbound buffer is smaller than Netty4WriteThrottlingHandler.MAX_BYTES_PER_WRITE final int writeableBytes = Math.toIntExact(embeddedChannel.bytesBeforeUnwritable()); @@ -104,51 +114,51 @@ public void testThrottleLargeCompositeMessage() throws ExecutionException, Inter new BytesArray(messageBytes, splitOffset, messageBytes.length - splitOffset) ); final ChannelPromise promise = embeddedChannel.newPromise(); - transportGroup.getLowLevelGroup().submit(() -> embeddedChannel.write(message, promise)).get(); + embeddedChannel.write(message, promise); assertThat(seen, hasSize(oneOf(1, 2))); assertSliceEquals(seen.get(0), message, 0, seen.get(0).readableBytes()); assertFalse(promise.isDone()); - transportGroup.getLowLevelGroup().submit(embeddedChannel::flush).get(); + embeddedChannel.flush(); assertTrue(promise.isDone()); assertThat(seen, hasSize(oneOf(fullSizeChunks, fullSizeChunks + 1))); assertTrue(capturingHandler.didWriteAfterThrottled); assertBufferEquals(Unpooled.compositeBuffer().addComponents(true, seen), message); } - public void testPassesSmallMessageDirectly() throws ExecutionException, InterruptedException { + public void testPassesSmallMessageDirectly() { final List seen = new CopyOnWriteArrayList<>(); final CapturingHandler capturingHandler = new CapturingHandler(seen); final EmbeddedChannel embeddedChannel = new EmbeddedChannel( capturingHandler, - new Netty4WriteThrottlingHandler(new ThreadContext(Settings.EMPTY)) + new Netty4WriteThrottlingHandler(new ThreadContext(Settings.EMPTY), threadWatchdog.getActivityTrackerForCurrentThread()) ); final int writeableBytes = Math.toIntExact(embeddedChannel.bytesBeforeUnwritable()); assertThat(writeableBytes, lessThan(Netty4WriteThrottlingHandler.MAX_BYTES_PER_WRITE)); final byte[] messageBytes = randomByteArrayOfLength(randomIntBetween(0, Netty4WriteThrottlingHandler.MAX_BYTES_PER_WRITE)); final Object message = wrapAsNettyOrEsBuffer(messageBytes); final ChannelPromise promise = embeddedChannel.newPromise(); - transportGroup.getLowLevelGroup().submit(() -> embeddedChannel.write(message, promise)).get(); + embeddedChannel.write(message, promise); assertThat(seen, hasSize(1)); // first message should be passed through straight away assertBufferEquals(seen.get(0), message); assertFalse(promise.isDone()); - transportGroup.getLowLevelGroup().submit(embeddedChannel::flush).get(); + embeddedChannel.flush(); assertTrue(promise.isDone()); assertThat(seen, hasSize(1)); assertFalse(capturingHandler.didWriteAfterThrottled); } - public void testThrottlesOnUnwritable() throws ExecutionException, InterruptedException { + public void testThrottlesOnUnwritable() { final List seen = new CopyOnWriteArrayList<>(); final EmbeddedChannel embeddedChannel = new EmbeddedChannel( new CapturingHandler(seen), - new Netty4WriteThrottlingHandler(new ThreadContext(Settings.EMPTY)) + new Netty4WriteThrottlingHandler(new ThreadContext(Settings.EMPTY), threadWatchdog.getActivityTrackerForCurrentThread()) ); final int writeableBytes = Math.toIntExact(embeddedChannel.bytesBeforeUnwritable()); assertThat(writeableBytes, lessThan(Netty4WriteThrottlingHandler.MAX_BYTES_PER_WRITE)); final byte[] messageBytes = randomByteArrayOfLength(writeableBytes + randomIntBetween(0, 10)); final Object message = wrapAsNettyOrEsBuffer(messageBytes); final ChannelPromise promise = embeddedChannel.newPromise(); - transportGroup.getLowLevelGroup().submit(() -> embeddedChannel.write(message, promise)).get(); + embeddedChannel.write(message, promise); assertThat(seen, hasSize(1)); // first message should be passed through straight away assertBufferEquals(seen.get(0), message); assertFalse(promise.isDone()); @@ -156,11 +166,11 @@ public void testThrottlesOnUnwritable() throws ExecutionException, InterruptedEx randomByteArrayOfLength(randomIntBetween(0, Netty4WriteThrottlingHandler.MAX_BYTES_PER_WRITE)) ); final ChannelPromise promiseForQueued = embeddedChannel.newPromise(); - transportGroup.getLowLevelGroup().submit(() -> embeddedChannel.write(messageToQueue, promiseForQueued)).get(); + embeddedChannel.write(messageToQueue, promiseForQueued); assertThat(seen, hasSize(1)); assertFalse(promiseForQueued.isDone()); assertFalse(promise.isDone()); - transportGroup.getLowLevelGroup().submit(embeddedChannel::flush).get(); + embeddedChannel.flush(); assertTrue(promise.isDone()); assertTrue(promiseForQueued.isDone()); } @@ -190,7 +200,7 @@ private static Object wrapAsNettyOrEsBuffer(byte[] messageBytes) { return new BytesArray(messageBytes); } - private static class CapturingHandler extends ChannelOutboundHandlerAdapter { + private class CapturingHandler extends ChannelOutboundHandlerAdapter { private final List seen; private boolean wasThrottled = false; @@ -203,6 +213,13 @@ private static class CapturingHandler extends ChannelOutboundHandlerAdapter { @Override public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception { + assertThat( + ThreadWatchdogHelper.getStuckThreadNames(threadWatchdog), + // writes are re-entrant so we might already be considered stuck due to an earlier check + anyOf(emptyIterable(), hasItem(Thread.currentThread().getName())) + ); + assertThat(ThreadWatchdogHelper.getStuckThreadNames(threadWatchdog), hasItem(Thread.currentThread().getName())); + assertTrue("should only write to writeable channel", ctx.channel().isWritable()); assertThat(msg, instanceOf(ByteBuf.class)); final ByteBuf buf = (ByteBuf) msg; diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NettyAllocatorTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NettyAllocatorTests.java new file mode 100644 index 0000000000000..b9e9b667e72fe --- /dev/null +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NettyAllocatorTests.java @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.transport.netty4; + +import io.netty.buffer.ByteBufAllocator; +import io.netty.buffer.Unpooled; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; + +import static org.elasticsearch.transport.netty4.NettyAllocator.TrashingByteBufAllocator; + +public class NettyAllocatorTests extends ESTestCase { + + static void assertBufferTrashed(BytesReference bytesRef) throws IOException { + var iter = bytesRef.iterator(); + BytesRef br; + while ((br = iter.next()) != null) { + for (var i = br.offset; i < br.offset + br.length; i++) { + assertEquals("off=" + br.offset + " len=" + br.length + " i=" + i, 0, br.bytes[i]); + } + } + } + + public void testTrashArrayByteBuf() { + var arr = randomByteArrayOfLength(between(1024, 2048)); + var buf = Unpooled.wrappedBuffer(arr); + var tBuf = new TrashingByteBuf(buf); + tBuf.release(); + var emptyArr = new byte[arr.length]; + assertArrayEquals(emptyArr, arr); + } + + public void testNioBufsTrashingByteBuf() { + var arrCnt = between(1, 16); + var byteArrs = new byte[arrCnt][]; + var byteBufs = new ByteBuffer[arrCnt]; + for (var i = 0; i < arrCnt; i++) { + byteArrs[i] = randomByteArrayOfLength(between(1024, 2048)); + byteBufs[i] = ByteBuffer.wrap(byteArrs[i]); + } + var buf = Unpooled.wrappedBuffer(byteBufs); + var tBuf = new TrashingByteBuf(buf); + tBuf.release(); + for (int i = 0; i < arrCnt; i++) { + for (int j = 0; j < byteArrs[i].length; j++) { + assertEquals(0, byteArrs[i][j]); + } + } + } + + public void testNioBufOffsetTrashingByteBuf() { + var arr = randomByteArrayOfLength(1024); + var off = 1; + var len = arr.length - 2; + arr[0] = 1; + arr[arr.length - 1] = 1; + var buf = Unpooled.wrappedBuffer(arr, off, len); + var tBuf = new TrashingByteBuf(buf); + tBuf.release(); + assertEquals(1, arr[0]); + assertEquals(1, arr[arr.length - 1]); + for (int i = 1; i < arr.length - 1; i++) { + assertEquals("at index " + i, 0, arr[i]); + } + } + + public void testTrashingByteBufAllocator() throws IOException { + var alloc = new TrashingByteBufAllocator(ByteBufAllocator.DEFAULT); + var size = between(1024 * 1024, 10 * 1024 * 1024); + + // use 3 different heap allocation methods + for (var buf : List.of(alloc.heapBuffer(), alloc.heapBuffer(1024), alloc.heapBuffer(1024, size))) { + buf.writeBytes(randomByteArrayOfLength(size)); + var bytesRef = Netty4Utils.toBytesReference(buf); + buf.release(); + assertBufferTrashed(bytesRef); + } + } + + public void testTrashingCompositeByteBuf() throws IOException { + var alloc = new TrashingByteBufAllocator(ByteBufAllocator.DEFAULT); + var compBuf = alloc.compositeHeapBuffer(); + for (var i = 0; i < between(1, 10); i++) { + var buf = alloc.heapBuffer().writeBytes(randomByteArrayOfLength(between(1024, 8192))); + compBuf.addComponent(true, buf); + } + var bytesRef = Netty4Utils.toBytesReference(compBuf); + compBuf.release(); + assertBufferTrashed(bytesRef); + } + +} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NettyTransportMultiPortTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NettyTransportMultiPortTests.java index 43b46554edb0e..1264949d5d626 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NettyTransportMultiPortTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NettyTransportMultiPortTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport.netty4; diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SharedGroupFactoryTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SharedGroupFactoryTests.java index a72e2c7b69465..85eb82f6dd5ed 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SharedGroupFactoryTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SharedGroupFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport.netty4; diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java index cedb68b25a4bf..cad839bed9555 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport.netty4; @@ -102,7 +103,7 @@ public void testDefaultKeepAliveSettings() throws IOException { MockTransportService serviceD = buildService("TS_D", VersionInformation.CURRENT, TransportVersion.current(), Settings.EMPTY) ) { - try (Transport.Connection connection = openConnection(serviceC, serviceD.getLocalDiscoNode(), TestProfiles.LIGHT_PROFILE)) { + try (Transport.Connection connection = openConnection(serviceC, serviceD.getLocalNode(), TestProfiles.LIGHT_PROFILE)) { assertThat(connection, instanceOf(StubbableTransport.WrappedConnection.class)); Transport.Connection conn = ((StubbableTransport.WrappedConnection) connection).getConnection(); assertThat(conn, instanceOf(TcpTransport.NodeChannels.class)); @@ -146,7 +147,7 @@ public void testTransportProfile() { MockTransportService serviceD = buildService("TS_D", VersionInformation.CURRENT, TransportVersion.current(), Settings.EMPTY) ) { - try (Transport.Connection connection = openConnection(serviceC, serviceD.getLocalDiscoNode(), connectionProfile)) { + try (Transport.Connection connection = openConnection(serviceC, serviceD.getLocalNode(), connectionProfile)) { assertThat(connection, instanceOf(StubbableTransport.WrappedConnection.class)); Transport.Connection conn = ((StubbableTransport.WrappedConnection) connection).getConnection(); assertThat(conn, instanceOf(TcpTransport.NodeChannels.class)); diff --git a/modules/transport-netty4/src/yamlRestTest/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java b/modules/transport-netty4/src/yamlRestTest/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java index 89e61b91397f5..38b914a1413c8 100644 --- a/modules/transport-netty4/src/yamlRestTest/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java +++ b/modules/transport-netty4/src/yamlRestTest/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.netty4; diff --git a/muted-tests.yml b/muted-tests.yml index ac35776db665f..b63c4705f18ef 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -1,25 +1,10 @@ tests: -- class: "org.elasticsearch.upgrades.SearchStatesIT" - issue: "https://github.com/elastic/elasticsearch/issues/108991" - method: "testCanMatch" - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT method: test {yaml=reference/esql/esql-async-query-api/line_17} issue: https://github.com/elastic/elasticsearch/issues/109260 -- class: "org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT" - issue: "https://github.com/elastic/elasticsearch/issues/109318" - method: "test {yaml=analysis-common/50_char_filters/pattern_replace error handling (too complex pattern)}" -- class: "org.elasticsearch.xpack.ml.integration.ClassificationHousePricingIT" - issue: "https://github.com/elastic/elasticsearch/issues/101598" - method: "testFeatureImportanceValues" - class: "org.elasticsearch.client.RestClientSingleHostIntegTests" issue: "https://github.com/elastic/elasticsearch/issues/102717" method: "testRequestResetAndAbort" -- class: "org.elasticsearch.xpack.deprecation.DeprecationHttpIT" - issue: "https://github.com/elastic/elasticsearch/issues/108628" - method: "testDeprecatedSettingsReturnWarnings" -- class: "org.elasticsearch.xpack.test.rest.XPackRestIT" - issue: "https://github.com/elastic/elasticsearch/issues/109687" - method: "test {p0=sql/translate/Translate SQL}" - class: org.elasticsearch.index.store.FsDirectoryFactoryTests method: testStoreDirectory issue: https://github.com/elastic/elasticsearch/issues/110210 @@ -29,15 +14,6 @@ tests: - class: org.elasticsearch.upgrades.SecurityIndexRolesMetadataMigrationIT method: testMetadataMigratedAfterUpgrade issue: https://github.com/elastic/elasticsearch/issues/110232 -- class: org.elasticsearch.backwards.SearchWithMinCompatibleSearchNodeIT - method: testMinVersionAsNewVersion - issue: https://github.com/elastic/elasticsearch/issues/95384 -- class: org.elasticsearch.backwards.SearchWithMinCompatibleSearchNodeIT - method: testCcsMinimizeRoundtripsIsFalse - issue: https://github.com/elastic/elasticsearch/issues/101974 -- class: "org.elasticsearch.xpack.searchablesnapshots.FrozenSearchableSnapshotsIntegTests" - issue: "https://github.com/elastic/elasticsearch/issues/110408" - method: "testCreateAndRestorePartialSearchableSnapshot" - class: org.elasticsearch.xpack.security.authz.store.NativePrivilegeStoreCacheTests method: testPopulationOfCacheWhenLoadingPrivilegesForAllApplications issue: https://github.com/elastic/elasticsearch/issues/110789 @@ -47,65 +23,23 @@ tests: - class: org.elasticsearch.nativeaccess.VectorSystemPropertyTests method: testSystemPropertyDisabled issue: https://github.com/elastic/elasticsearch/issues/110949 -- class: org.elasticsearch.multi_node.GlobalCheckpointSyncActionIT - issue: https://github.com/elastic/elasticsearch/issues/111124 -- class: org.elasticsearch.cluster.PrevalidateShardPathIT - method: testCheckShards - issue: https://github.com/elastic/elasticsearch/issues/111134 -- class: org.elasticsearch.packaging.test.DockerTests - method: test021InstallPlugin - issue: https://github.com/elastic/elasticsearch/issues/110343 - class: org.elasticsearch.xpack.security.authc.oidc.OpenIdConnectAuthIT method: testAuthenticateWithImplicitFlow issue: https://github.com/elastic/elasticsearch/issues/111191 -- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsRestIT - issue: https://github.com/elastic/elasticsearch/issues/111319 -- class: org.elasticsearch.xpack.ml.integration.InferenceIngestInputConfigIT - method: testIngestWithInputFields - issue: https://github.com/elastic/elasticsearch/issues/111383 -- class: org.elasticsearch.xpack.ml.integration.InferenceIngestInputConfigIT - method: testIngestWithMultipleInputFields - issue: https://github.com/elastic/elasticsearch/issues/111384 - class: org.elasticsearch.xpack.security.authc.oidc.OpenIdConnectAuthIT method: testAuthenticateWithCodeFlowAndClientPost issue: https://github.com/elastic/elasticsearch/issues/111396 -- class: org.elasticsearch.xpack.restart.FullClusterRestartIT - method: testSingleDoc {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/111434 -- class: org.elasticsearch.xpack.restart.FullClusterRestartIT - method: testDataStreams {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/111448 - class: org.elasticsearch.search.SearchServiceTests issue: https://github.com/elastic/elasticsearch/issues/111529 -- class: org.elasticsearch.xpack.test.rest.XPackRestIT - method: test {p0=rollup/security_tests/Index-based access} - issue: https://github.com/elastic/elasticsearch/issues/111631 -- class: org.elasticsearch.tdigest.ComparisonTests - method: testSparseGaussianDistribution - issue: https://github.com/elastic/elasticsearch/issues/111721 -- class: org.elasticsearch.upgrades.FullClusterRestartIT - method: testSnapshotRestore {cluster=OLD} - issue: https://github.com/elastic/elasticsearch/issues/111777 -- class: org.elasticsearch.xpack.restart.CoreFullClusterRestartIT - method: testSnapshotRestore {cluster=OLD} - issue: https://github.com/elastic/elasticsearch/issues/111775 - class: org.elasticsearch.upgrades.FullClusterRestartIT method: testSnapshotRestore {cluster=UPGRADED} issue: https://github.com/elastic/elasticsearch/issues/111798 -- class: org.elasticsearch.xpack.restart.CoreFullClusterRestartIT - method: testSnapshotRestore {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/111799 - class: org.elasticsearch.xpack.inference.InferenceRestIT method: test {p0=inference/80_random_rerank_retriever/Random rerank retriever predictably shuffles results} issue: https://github.com/elastic/elasticsearch/issues/111999 - class: org.elasticsearch.xpack.ml.integration.MlJobIT method: testDeleteJobAfterMissingIndex issue: https://github.com/elastic/elasticsearch/issues/112088 -- class: org.elasticsearch.xpack.test.rest.XPackRestIT - method: test {p0=transform/preview_transforms/Test preview transform latest} - issue: https://github.com/elastic/elasticsearch/issues/112144 -- class: org.elasticsearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT - issue: https://github.com/elastic/elasticsearch/issues/112147 - class: org.elasticsearch.smoketest.WatcherYamlRestIT method: test {p0=watcher/usage/10_basic/Test watcher usage stats output} issue: https://github.com/elastic/elasticsearch/issues/112189 @@ -115,72 +49,23 @@ tests: - class: org.elasticsearch.xpack.ml.integration.MlJobIT method: testDeleteJobAsync issue: https://github.com/elastic/elasticsearch/issues/112212 -- class: org.elasticsearch.search.retriever.RankDocRetrieverBuilderIT - method: testRankDocsRetrieverWithCollapse - issue: https://github.com/elastic/elasticsearch/issues/112254 -- class: org.elasticsearch.search.ccs.CCSUsageTelemetryIT - issue: https://github.com/elastic/elasticsearch/issues/112324 -- class: org.elasticsearch.datastreams.logsdb.qa.StandardVersusLogsIndexModeRandomDataChallengeRestIT - method: testMatchAllQuery - issue: https://github.com/elastic/elasticsearch/issues/112374 -- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT - method: test {yaml=reference/rest-api/watcher/put-watch/line_120} - issue: https://github.com/elastic/elasticsearch/issues/99517 - class: org.elasticsearch.xpack.ml.integration.MlJobIT method: testMultiIndexDelete issue: https://github.com/elastic/elasticsearch/issues/112381 -- class: org.elasticsearch.action.admin.cluster.stats.CCSTelemetrySnapshotTests - method: testToXContent - issue: https://github.com/elastic/elasticsearch/issues/112325 -- class: org.elasticsearch.search.retriever.RankDocRetrieverBuilderIT - method: testRankDocsRetrieverWithNestedQuery - issue: https://github.com/elastic/elasticsearch/issues/112421 -- class: org.elasticsearch.indices.mapping.UpdateMappingIntegrationIT - issue: https://github.com/elastic/elasticsearch/issues/112423 - class: org.elasticsearch.xpack.esql.expression.function.aggregate.SpatialCentroidTests method: "testAggregateIntermediate {TestCase= #2}" issue: https://github.com/elastic/elasticsearch/issues/112461 - class: org.elasticsearch.xpack.esql.expression.function.aggregate.SpatialCentroidTests method: testAggregateIntermediate {TestCase=} issue: https://github.com/elastic/elasticsearch/issues/112463 -- class: org.elasticsearch.xpack.esql.action.ManyShardsIT - method: testRejection - issue: https://github.com/elastic/elasticsearch/issues/112406 -- class: org.elasticsearch.xpack.esql.action.ManyShardsIT - method: testConcurrentQueries - issue: https://github.com/elastic/elasticsearch/issues/112424 - class: org.elasticsearch.xpack.inference.external.http.RequestBasedTaskRunnerTests method: testLoopOneAtATime issue: https://github.com/elastic/elasticsearch/issues/112471 - class: org.elasticsearch.ingest.geoip.IngestGeoIpClientYamlTestSuiteIT issue: https://github.com/elastic/elasticsearch/issues/111497 -- class: org.elasticsearch.smoketest.SmokeTestIngestWithAllDepsClientYamlTestSuiteIT - method: test {yaml=ingest/80_ingest_simulate/Test ingest simulate with reroute and mapping validation from templates} - issue: https://github.com/elastic/elasticsearch/issues/112575 -- class: org.elasticsearch.xpack.security.authc.kerberos.SimpleKdcLdapServerTests - method: testClientServiceMutualAuthentication - issue: https://github.com/elastic/elasticsearch/issues/112529 -- class: org.elasticsearch.search.basic.SearchWhileRelocatingIT - method: testSearchAndRelocateConcurrentlyRandomReplicas - issue: https://github.com/elastic/elasticsearch/issues/112515 -- class: org.elasticsearch.xpack.test.rest.XPackRestIT - method: test {p0=terms_enum/10_basic/Test search after on unconfigured constant keyword field} - issue: https://github.com/elastic/elasticsearch/issues/112624 -- class: org.elasticsearch.xpack.esql.EsqlAsyncSecurityIT - method: testIndexPatternErrorMessageComparison_ESQL_SearchDSL - issue: https://github.com/elastic/elasticsearch/issues/112630 - class: org.elasticsearch.xpack.ml.integration.MlJobIT method: testPutJob_GivenFarequoteConfig issue: https://github.com/elastic/elasticsearch/issues/112382 -- class: org.elasticsearch.xpack.security.authc.kerberos.KerberosTicketValidatorTests - method: testWhenKeyTabWithInvalidContentFailsValidation - issue: https://github.com/elastic/elasticsearch/issues/112631 -- class: org.elasticsearch.xpack.security.authc.kerberos.KerberosTicketValidatorTests - method: testValidKebrerosTicket - issue: https://github.com/elastic/elasticsearch/issues/112632 -- class: org.elasticsearch.xpack.security.authc.kerberos.KerberosTicketValidatorTests - method: testKerbTicketGeneratedForDifferentServerFailsValidation - issue: https://github.com/elastic/elasticsearch/issues/112639 - class: org.elasticsearch.packaging.test.PackagesSecurityAutoConfigurationTests method: test20SecurityNotAutoConfiguredOnReInstallation issue: https://github.com/elastic/elasticsearch/issues/112635 @@ -205,9 +90,233 @@ tests: - class: org.elasticsearch.xpack.ml.integration.MlJobIT method: testGetJob_GivenNoSuchJob issue: https://github.com/elastic/elasticsearch/issues/112730 -- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT - method: test {yaml=reference/cluster/stats/line_1450} - issue: https://github.com/elastic/elasticsearch/issues/112732 +- class: org.elasticsearch.xpack.ml.integration.MlJobIT + method: testDeleteJobAfterMissingAliases + issue: https://github.com/elastic/elasticsearch/issues/112823 +- class: org.elasticsearch.xpack.ml.integration.MlJobIT + method: testCreateJob_WithClashingFieldMappingsFails + issue: https://github.com/elastic/elasticsearch/issues/113046 +- class: org.elasticsearch.xpack.sql.qa.security.JdbcSqlSpecIT + method: test {case-functions.testUcaseInline1} + issue: https://github.com/elastic/elasticsearch/issues/112641 +- class: org.elasticsearch.xpack.sql.qa.security.JdbcSqlSpecIT + method: test {case-functions.testUcaseInline3} + issue: https://github.com/elastic/elasticsearch/issues/112643 +- class: org.elasticsearch.xpack.sql.qa.security.JdbcSqlSpecIT + method: test {case-functions.testUpperCasingTheSecondLetterFromTheRightFromFirstName} + issue: https://github.com/elastic/elasticsearch/issues/112640 +- class: org.elasticsearch.xpack.sql.qa.security.JdbcSqlSpecIT + method: test {case-functions.testSelectInsertWithLcaseAndLengthWithOrderBy} + issue: https://github.com/elastic/elasticsearch/issues/112642 +- class: org.elasticsearch.xpack.inference.rest.ServerSentEventsRestActionListenerTests + method: testResponse + issue: https://github.com/elastic/elasticsearch/issues/113148 +- class: org.elasticsearch.packaging.test.WindowsServiceTests + method: test30StartStop + issue: https://github.com/elastic/elasticsearch/issues/113160 +- class: org.elasticsearch.packaging.test.WindowsServiceTests + method: test33JavaChanged + issue: https://github.com/elastic/elasticsearch/issues/113177 +- class: org.elasticsearch.xpack.inference.rest.ServerSentEventsRestActionListenerTests + method: testErrorMidStream + issue: https://github.com/elastic/elasticsearch/issues/113179 +- class: org.elasticsearch.smoketest.MlWithSecurityIT + method: test {yaml=ml/sparse_vector_search/Test sparse_vector search with query vector and pruning config} + issue: https://github.com/elastic/elasticsearch/issues/108997 +- class: org.elasticsearch.packaging.test.WindowsServiceTests + method: test80JavaOptsInEnvVar + issue: https://github.com/elastic/elasticsearch/issues/113219 +- class: org.elasticsearch.packaging.test.WindowsServiceTests + method: test81JavaOptsInJvmOptions + issue: https://github.com/elastic/elasticsearch/issues/113313 +- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT + method: test {p0=mtermvectors/10_basic/Tests catching other exceptions per item} + issue: https://github.com/elastic/elasticsearch/issues/113325 +- class: org.elasticsearch.xpack.ml.integration.MlJobIT + method: testDeleteJob_TimingStatsDocumentIsDeleted + issue: https://github.com/elastic/elasticsearch/issues/113370 +- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT + method: test {p0=search/500_date_range/from, to, include_lower, include_upper deprecated} + issue: https://github.com/elastic/elasticsearch/pull/113286 +- class: org.elasticsearch.index.mapper.extras.TokenCountFieldMapperTests + method: testBlockLoaderFromRowStrideReaderWithSyntheticSource + issue: https://github.com/elastic/elasticsearch/issues/113427 +- class: org.elasticsearch.xpack.ml.integration.MlJobIT + method: testOutOfOrderData + issue: https://github.com/elastic/elasticsearch/issues/113477 +- class: org.elasticsearch.xpack.ml.integration.MlJobIT + method: testCreateJobsWithIndexNameOption + issue: https://github.com/elastic/elasticsearch/issues/113528 +- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT + method: test {p0=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates} + issue: https://github.com/elastic/elasticsearch/issues/113537 +- class: org.elasticsearch.xpack.ml.integration.MlJobIT + method: testCantCreateJobWithSameID + issue: https://github.com/elastic/elasticsearch/issues/113581 +- class: org.elasticsearch.xpack.transform.integration.TransformIT + method: testStopWaitForCheckpoint + issue: https://github.com/elastic/elasticsearch/issues/106113 +- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT + method: test {p0=search/540_ignore_above_synthetic_source/ignore_above mapping level setting on arrays} + issue: https://github.com/elastic/elasticsearch/issues/113648 +- class: org.elasticsearch.xpack.ml.integration.MlJobIT + method: testGetJobs_GivenMultipleJobs + issue: https://github.com/elastic/elasticsearch/issues/113654 +- class: org.elasticsearch.xpack.ml.integration.MlJobIT + method: testGetJobs_GivenSingleJob + issue: https://github.com/elastic/elasticsearch/issues/113655 +- class: org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDateNanosTests + issue: https://github.com/elastic/elasticsearch/issues/113661 +- class: org.elasticsearch.search.retriever.RankDocsRetrieverBuilderTests + method: testRewrite + issue: https://github.com/elastic/elasticsearch/issues/114467 +- class: org.elasticsearch.gradle.internal.PublishPluginFuncTest + issue: https://github.com/elastic/elasticsearch/issues/114492 +- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT + method: test {p0=indices.split/40_routing_partition_size/nested} + issue: https://github.com/elastic/elasticsearch/issues/113842 +- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT + method: test {p0=indices.split/40_routing_partition_size/more than 1} + issue: https://github.com/elastic/elasticsearch/issues/113841 +- class: org.elasticsearch.kibana.KibanaThreadPoolIT + method: testBlockedThreadPoolsRejectUserRequests + issue: https://github.com/elastic/elasticsearch/issues/113939 +- class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT + method: testPutE5WithTrainedModelAndInference + issue: https://github.com/elastic/elasticsearch/issues/114023 +- class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT + method: testPutE5Small_withPlatformAgnosticVariant + issue: https://github.com/elastic/elasticsearch/issues/113983 +- class: org.elasticsearch.datastreams.LazyRolloverDuringDisruptionIT + method: testRolloverIsExecutedOnce + issue: https://github.com/elastic/elasticsearch/issues/112634 +- class: org.elasticsearch.xpack.rank.rrf.RRFRankClientYamlTestSuiteIT + method: test {yaml=rrf/800_rrf_with_text_similarity_reranker_retriever/explain using rrf retriever and text-similarity} + issue: https://github.com/elastic/elasticsearch/issues/114757 +- class: org.elasticsearch.xpack.remotecluster.RemoteClusterSecurityWithApmTracingRestIT + method: testTracingCrossCluster + issue: https://github.com/elastic/elasticsearch/issues/112731 +- class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT + method: testPutE5Small_withPlatformSpecificVariant + issue: https://github.com/elastic/elasticsearch/issues/113950 +- class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT + method: test {p0=search.vectors/42_knn_search_int4_flat/Vector similarity with filter only} + issue: https://github.com/elastic/elasticsearch/issues/115475 +- class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests + method: testProcessFileChanges + issue: https://github.com/elastic/elasticsearch/issues/115280 +- class: org.elasticsearch.xpack.inference.DefaultEndPointsIT + method: testInferDeploysDefaultE5 + issue: https://github.com/elastic/elasticsearch/issues/115361 +- class: org.elasticsearch.xpack.inference.InferenceCrudIT + method: testSupportedStream + issue: https://github.com/elastic/elasticsearch/issues/113430 +- class: org.elasticsearch.xpack.spatial.search.GeoGridAggAndQueryConsistencyIT + method: testGeoShapeGeoHash + issue: https://github.com/elastic/elasticsearch/issues/115664 +- class: org.elasticsearch.indices.mapping.UpdateMappingIntegrationIT + issue: https://github.com/elastic/elasticsearch/issues/116126 +- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsRestIT + issue: https://github.com/elastic/elasticsearch/issues/111319 +- class: org.elasticsearch.upgrades.FullClusterRestartIT + method: testSnapshotRestore {cluster=OLD} + issue: https://github.com/elastic/elasticsearch/issues/111777 +- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsRestIT + method: testLookbackWithIndicesOptions + issue: https://github.com/elastic/elasticsearch/issues/116127 +- class: org.elasticsearch.xpack.restart.CoreFullClusterRestartIT + method: testSnapshotRestore {cluster=UPGRADED} + issue: https://github.com/elastic/elasticsearch/issues/111799 +- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT + method: test {p0=search/380_sort_segments_on_timestamp/Test that index segments are NOT sorted on timestamp field when @timestamp field is dynamically added} + issue: https://github.com/elastic/elasticsearch/issues/116221 +- class: org.elasticsearch.ingest.common.IngestCommonClientYamlTestSuiteIT + method: test {yaml=ingest/310_reroute_processor/Test remove then add reroute processor with and without lazy rollover} + issue: https://github.com/elastic/elasticsearch/issues/116158 +- class: org.elasticsearch.ingest.common.IngestCommonClientYamlTestSuiteIT + method: test {yaml=ingest/310_reroute_processor/Test data stream with lazy rollover obtains pipeline from template} + issue: https://github.com/elastic/elasticsearch/issues/116157 +- class: org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloaderIT + method: testEnterpriseDownloaderTask + issue: https://github.com/elastic/elasticsearch/issues/115163 +- class: org.elasticsearch.xpack.deprecation.DeprecationHttpIT + method: testDeprecatedSettingsReturnWarnings + issue: https://github.com/elastic/elasticsearch/issues/108628 +- class: org.elasticsearch.xpack.apmdata.APMYamlTestSuiteIT + method: test {yaml=/10_apm/Test template reinstallation} + issue: https://github.com/elastic/elasticsearch/issues/116445 +- class: org.elasticsearch.action.admin.HotThreadsIT + method: testHotThreadsDontFail + issue: https://github.com/elastic/elasticsearch/issues/115754 +- class: org.elasticsearch.action.search.PointInTimeIT + method: testPITTiebreak + issue: https://github.com/elastic/elasticsearch/issues/115810 +- class: org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT + method: test {p0=esql/61_enrich_ip/IP strings} + issue: https://github.com/elastic/elasticsearch/issues/116529 +- class: org.elasticsearch.xpack.inference.DefaultEndPointsIT + method: testInferDeploysDefaultElser + issue: https://github.com/elastic/elasticsearch/issues/114913 +- class: org.elasticsearch.threadpool.SimpleThreadPoolIT + method: testThreadPoolMetrics + issue: https://github.com/elastic/elasticsearch/issues/108320 +- class: org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT + method: test {p0=esql/60_enrich/Enrich on keyword with fields alias} + issue: https://github.com/elastic/elasticsearch/issues/116592 +- class: org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT + method: test {p0=esql/60_enrich/Enrich on keyword with fields} + issue: https://github.com/elastic/elasticsearch/issues/116593 +- class: org.elasticsearch.xpack.spatial.search.GeoGridAggAndQueryConsistencyIT + method: testGeoShapeGeoTile + issue: https://github.com/elastic/elasticsearch/issues/115717 +- class: org.elasticsearch.search.StressSearchServiceReaperIT + method: testStressReaper + issue: https://github.com/elastic/elasticsearch/issues/115816 +- class: org.elasticsearch.xpack.spatial.search.GeoGridAggAndQueryConsistencyIT + method: testGeoShapeGeoHex + issue: https://github.com/elastic/elasticsearch/issues/115705 +- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT + method: test {spatial.CentroidFromAirportsAfterIntersectsCompoundPredicateNoDocValues SYNC} + issue: https://github.com/elastic/elasticsearch/issues/116945 +- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT + method: test {spatial.CentroidFromAirportsAfterIntersectsCompoundPredicateNoDocValues ASYNC} + issue: https://github.com/elastic/elasticsearch/issues/116945 +- class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT + method: test {spatial.CentroidFromAirportsAfterIntersectsCompoundPredicateNoDocValues} + issue: https://github.com/elastic/elasticsearch/issues/116945 +- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT + method: test {spatial.CentroidFromAirportsAfterIntersectsCompoundPredicateNotIndexedNorDocValues SYNC} + issue: https://github.com/elastic/elasticsearch/issues/116945 +- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT + method: test {spatial.CentroidFromAirportsAfterIntersectsCompoundPredicateNotIndexedNorDocValues ASYNC} + issue: https://github.com/elastic/elasticsearch/issues/116945 +- class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT + method: test {spatial.CentroidFromAirportsAfterIntersectsCompoundPredicateNotIndexedNorDocValues} + issue: https://github.com/elastic/elasticsearch/issues/116945 +- class: org.elasticsearch.search.basic.SearchWithRandomIOExceptionsIT + method: testRandomDirectoryIOExceptions + issue: https://github.com/elastic/elasticsearch/issues/114824 +- class: org.elasticsearch.xpack.inference.InferenceRestIT + method: test {p0=inference/40_semantic_text_query/Query a field that uses the default ELSER 2 endpoint} + issue: https://github.com/elastic/elasticsearch/issues/117027 +- class: org.elasticsearch.xpack.inference.InferenceRestIT + method: test {p0=inference/30_semantic_text_inference/Calculates embeddings using the default ELSER 2 endpoint} + issue: https://github.com/elastic/elasticsearch/issues/117349 +- class: org.elasticsearch.xpack.inference.InferenceRestIT + method: test {p0=inference/30_semantic_text_inference_bwc/Calculates embeddings using the default ELSER 2 endpoint} + issue: https://github.com/elastic/elasticsearch/issues/117349 +- class: org.elasticsearch.search.basic.SearchWithRandomDisconnectsIT + method: testSearchWithRandomDisconnects + issue: https://github.com/elastic/elasticsearch/issues/116175 +- class: org.elasticsearch.action.search.SearchQueryThenFetchAsyncActionTests + method: testMinimumVersionBetweenNewAndOldVersion + issue: https://github.com/elastic/elasticsearch/issues/117485 +- class: org.elasticsearch.discovery.ClusterDisruptionIT + method: testAckedIndexing + issue: https://github.com/elastic/elasticsearch/issues/117024 +- class: org.elasticsearch.xpack.inference.DefaultEndPointsIT + method: testMultipleInferencesTriggeringDownloadAndDeploy + issue: https://github.com/elastic/elasticsearch/issues/117208 # Examples: # @@ -247,3 +356,96 @@ tests: # - class: "org.elasticsearch.xpack.esql.**" # method: "test {union_types.MultiIndexIpStringStatsInline *}" # issue: "https://github.com/elastic/elasticsearch/..." +- class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests + method: testStopWorksInMiddleOfProcessing + issue: https://github.com/elastic/elasticsearch/issues/117591 +- class: org.elasticsearch.repositories.s3.RepositoryS3ClientYamlTestSuiteIT + issue: https://github.com/elastic/elasticsearch/issues/117596 +- class: org.elasticsearch.search.ccs.CrossClusterIT + method: testCancel + issue: https://github.com/elastic/elasticsearch/issues/108061 +- class: org.elasticsearch.xpack.ml.integration.RegressionIT + method: testTwoJobsWithSameRandomizeSeedUseSameTrainingSet + issue: https://github.com/elastic/elasticsearch/issues/117805 +- class: org.elasticsearch.xpack.security.authc.ldap.UserAttributeGroupsResolverTests + issue: https://github.com/elastic/elasticsearch/issues/116537 +- class: org.elasticsearch.xpack.esql.plugin.ClusterRequestTests + method: testFallbackIndicesOptions + issue: https://github.com/elastic/elasticsearch/issues/117937 +- class: org.elasticsearch.repositories.s3.RepositoryS3EcsCredentialsRestIT + method: testNonexistentBucketReadonlyFalse + issue: https://github.com/elastic/elasticsearch/issues/118225 +- class: org.elasticsearch.xpack.esql.action.EsqlActionTaskIT + method: testCancelRequestWhenFailingFetchingPages + issue: https://github.com/elastic/elasticsearch/issues/118213 +- class: org.elasticsearch.upgrades.HealthNodeUpgradeIT + method: testHealthNode {upgradedNodes=1} + issue: https://github.com/elastic/elasticsearch/issues/118157 +- class: org.elasticsearch.upgrades.HealthNodeUpgradeIT + method: testHealthNode {upgradedNodes=2} + issue: https://github.com/elastic/elasticsearch/issues/118158 +- class: org.elasticsearch.xpack.inference.DefaultEndPointsIT + method: testInferDeploysDefaultRerank + issue: https://github.com/elastic/elasticsearch/issues/118184 +- class: org.elasticsearch.reservedstate.service.RepositoriesFileSettingsIT + method: testSettingsApplied + issue: https://github.com/elastic/elasticsearch/issues/116694 +- class: org.elasticsearch.repositories.blobstore.testkit.analyze.MinioRepositoryAnalysisRestIT + issue: https://github.com/elastic/elasticsearch/issues/118548 +- class: org.elasticsearch.xpack.test.rest.XPackRestIT + method: test {p0=migrate/20_reindex_status/Test Reindex With Existing Data Stream} + issue: https://github.com/elastic/elasticsearch/issues/118576 +- class: org.elasticsearch.discovery.ec2.DiscoveryEc2AvailabilityZoneAttributeNoImdsIT + method: testAvailabilityZoneAttribute + issue: https://github.com/elastic/elasticsearch/issues/118564 +- class: org.elasticsearch.xpack.searchablesnapshots.RetrySearchIntegTests + method: testRetryPointInTime + issue: https://github.com/elastic/elasticsearch/issues/118514 +- class: org.elasticsearch.xpack.apmdata.APMYamlTestSuiteIT + method: test {yaml=/20_metrics_ingest/Test metrics-apm.app-* setting event.ingested via ingest pipeline} + issue: https://github.com/elastic/elasticsearch/issues/118875 +- class: org.elasticsearch.xpack.ml.integration.ForecastIT + method: testOverflowToDisk + issue: https://github.com/elastic/elasticsearch/issues/117740 +- class: org.elasticsearch.aggregations.bucket.SearchCancellationIT + method: testCancellationDuringTimeSeriesAggregation + issue: https://github.com/elastic/elasticsearch/issues/118992 +- class: org.elasticsearch.versioning.ConcurrentSeqNoVersioningIT + method: testSeqNoCASLinearizability + issue: https://github.com/elastic/elasticsearch/issues/117249 +- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT + method: test {p0=synonyms/90_synonyms_reloading_for_synset/Reload analyzers for specific synonym set} + issue: https://github.com/elastic/elasticsearch/issues/116777 +- class: org.elasticsearch.xpack.ml.integration.InferenceIngestInputConfigIT + method: testIngestWithInputFields + issue: https://github.com/elastic/elasticsearch/issues/118092 +- class: org.elasticsearch.xpack.ml.integration.InferenceIngestInputConfigIT + method: testIngestWithMultipleInputFields + issue: https://github.com/elastic/elasticsearch/issues/118093 +- class: org.elasticsearch.search.fieldcaps.FieldCapabilitiesIT + method: testRelocation + issue: https://github.com/elastic/elasticsearch/issues/119280 +- class: org.elasticsearch.xpack.esql.action.EsqlNodeFailureIT + method: testFailureLoadingFields + issue: https://github.com/elastic/elasticsearch/issues/118000 +- class: org.elasticsearch.xpack.restart.FullClusterRestartIT + method: testWatcherWithApiKey {cluster=UPGRADED} + issue: https://github.com/elastic/elasticsearch/issues/119396 +- class: org.elasticsearch.index.engine.LuceneSyntheticSourceChangesSnapshotTests + method: testSkipNonRootOfNestedDocuments + issue: https://github.com/elastic/elasticsearch/issues/119553 +- class: org.elasticsearch.xpack.restart.FullClusterRestartIT + method: testWatcher {cluster=UPGRADED} + issue: https://github.com/elastic/elasticsearch/issues/119259 +- class: org.elasticsearch.xpack.restart.FullClusterRestartIT + method: testServiceAccountApiKey {cluster=UPGRADED} + issue: https://github.com/elastic/elasticsearch/issues/119435 +- class: org.elasticsearch.xpack.restart.FullClusterRestartIT + method: testSecurityNativeRealm {cluster=UPGRADED} + issue: https://github.com/elastic/elasticsearch/issues/119423 +- class: org.elasticsearch.xpack.restart.FullClusterRestartIT + method: testSingleDoc {cluster=UPGRADED} + issue: https://github.com/elastic/elasticsearch/issues/119434 +- class: org.elasticsearch.upgrades.SearchStatesIT + method: testCanMatch + issue: https://github.com/elastic/elasticsearch/issues/118718 diff --git a/plugins/analysis-icu/build.gradle b/plugins/analysis-icu/build.gradle index 1c7db6d040be5..6fe503f0f5336 100644 --- a/plugins/analysis-icu/build.gradle +++ b/plugins/analysis-icu/build.gradle @@ -2,18 +2,19 @@ import org.elasticsearch.gradle.internal.info.BuildParams /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.legacy-yaml-rest-test' apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { - description 'The ICU Analysis plugin integrates the Lucene ICU module into Elasticsearch, adding ICU-related analysis components.' - classname 'org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin' + description = 'The ICU Analysis plugin integrates the Lucene ICU module into Elasticsearch, adding ICU-related analysis components.' + classname ='org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin' } tasks.named("forbiddenApisMain").configure { @@ -27,7 +28,7 @@ dependencies { api "com.ibm.icu:icu4j:${versions.icu4j}" } -if (BuildParams.isSnapshotBuild() == false) { +if (buildParams.isSnapshotBuild() == false) { tasks.named("test").configure { systemProperty 'es.index_mode_feature_flag_registered', 'true' } diff --git a/plugins/analysis-icu/src/internalClusterTest/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperIT.java b/plugins/analysis-icu/src/internalClusterTest/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperIT.java index 955bcaf8f0352..ee81fad0c425d 100644 --- a/plugins/analysis-icu/src/internalClusterTest/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperIT.java +++ b/plugins/analysis-icu/src/internalClusterTest/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper; diff --git a/plugins/analysis-icu/src/main/java/module-info.java b/plugins/analysis-icu/src/main/java/module-info.java index 78d9e9d0ec64b..cc1e4d468ed09 100644 --- a/plugins/analysis-icu/src/main/java/module-info.java +++ b/plugins/analysis-icu/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.analysis.icu { diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java index 8ec4e7791333e..240aa27b51bc2 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.icu; diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java index 7a0caf56d6066..6c4a4ee42ed00 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.icu; diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuAnalyzerProvider.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuAnalyzerProvider.java index f5f946e575fec..3fea1918252ea 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuAnalyzerProvider.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.icu; diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuCollationTokenFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuCollationTokenFilterFactory.java index 2e7268bbafbd0..ae8ead523b7ea 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuCollationTokenFilterFactory.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuCollationTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.icu; diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuFoldingTokenFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuFoldingTokenFilterFactory.java index 8b3140fbc596c..6cffbb7e0a17e 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuFoldingTokenFilterFactory.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuFoldingTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.icu; diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuNormalizerCharFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuNormalizerCharFilterFactory.java index 63b865dac1e27..102f27cd855b2 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuNormalizerCharFilterFactory.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuNormalizerCharFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.icu; diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuNormalizerTokenFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuNormalizerTokenFilterFactory.java index 5352a33cf517c..23b2c355b7a68 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuNormalizerTokenFilterFactory.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuNormalizerTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.icu; diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuTokenizerFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuTokenizerFactory.java index 0a73eb38bd45c..62ab6d8792905 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuTokenizerFactory.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuTokenizerFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.icu; diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuTransformTokenFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuTransformTokenFilterFactory.java index 67e755a4fd54a..785b083c4c31e 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuTransformTokenFilterFactory.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuTransformTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.icu; diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUFactoryTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUFactoryTests.java index 4922fbc32045d..3fc1505f45eb1 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUFactoryTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.icu; diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/CollationFieldTypeTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/CollationFieldTypeTests.java index 5a03632b0052e..7742963f7e097 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/CollationFieldTypeTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/CollationFieldTypeTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.icu; diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapperTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapperTests.java index 99017733dd989..034a25e854298 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapperTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapperTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.icu; diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldTypeTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldTypeTests.java index a7e78cb0fdbde..388e941ea1c9a 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldTypeTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldTypeTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.icu; diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/IcuAnalyzerTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/IcuAnalyzerTests.java index fef40ca59940f..7e02f69e0025b 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/IcuAnalyzerTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/IcuAnalyzerTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.icu; diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/IcuTokenizerFactoryTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/IcuTokenizerFactoryTests.java index 30c908abbca3a..b508542cbf0bc 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/IcuTokenizerFactoryTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/IcuTokenizerFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.icu; diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/SimpleIcuAnalysisTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/SimpleIcuAnalysisTests.java index af4e53b1e38aa..5fa18c3a2132a 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/SimpleIcuAnalysisTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/SimpleIcuAnalysisTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.icu; diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/SimpleIcuCollationTokenFilterTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/SimpleIcuCollationTokenFilterTests.java index 9e88371a681f6..9265fac32cb78 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/SimpleIcuCollationTokenFilterTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/SimpleIcuCollationTokenFilterTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.icu; diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/SimpleIcuNormalizerCharFilterTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/SimpleIcuNormalizerCharFilterTests.java index 0064ee56e5f6f..2b2c155e6f6d7 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/SimpleIcuNormalizerCharFilterTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/SimpleIcuNormalizerCharFilterTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.icu; diff --git a/plugins/analysis-icu/src/yamlRestTest/java/org/elasticsearch/index/analysis/IcuClientYamlTestSuiteIT.java b/plugins/analysis-icu/src/yamlRestTest/java/org/elasticsearch/index/analysis/IcuClientYamlTestSuiteIT.java index 3a4ad7e259480..db2e70ab624ae 100644 --- a/plugins/analysis-icu/src/yamlRestTest/java/org/elasticsearch/index/analysis/IcuClientYamlTestSuiteIT.java +++ b/plugins/analysis-icu/src/yamlRestTest/java/org/elasticsearch/index/analysis/IcuClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.analysis; diff --git a/plugins/analysis-kuromoji/build.gradle b/plugins/analysis-kuromoji/build.gradle index a91e34018179f..82fa59e5773c3 100644 --- a/plugins/analysis-kuromoji/build.gradle +++ b/plugins/analysis-kuromoji/build.gradle @@ -1,16 +1,17 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.legacy-yaml-rest-test' apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' esplugin { - description 'The Japanese (kuromoji) Analysis plugin integrates Lucene kuromoji analysis module into elasticsearch.' - classname 'org.elasticsearch.plugin.analysis.kuromoji.AnalysisKuromojiPlugin' + description = 'The Japanese (kuromoji) Analysis plugin integrates Lucene kuromoji analysis module into elasticsearch.' + classname ='org.elasticsearch.plugin.analysis.kuromoji.AnalysisKuromojiPlugin' } dependencies { diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java index da572a9ad9b44..4942e895e0785 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.kuromoji; diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/HiraganaUppercaseFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/HiraganaUppercaseFilterFactory.java index f257eed743c07..b22757af22372 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/HiraganaUppercaseFilterFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/HiraganaUppercaseFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.kuromoji; diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/JapaneseStopTokenFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/JapaneseStopTokenFilterFactory.java index 50e37df9700a4..e34d5246dd092 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/JapaneseStopTokenFilterFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/JapaneseStopTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.kuromoji; diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KatakanaUppercaseFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KatakanaUppercaseFilterFactory.java index dfa58aecc34ae..1f72f1d57d2c5 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KatakanaUppercaseFilterFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KatakanaUppercaseFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.kuromoji; diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiAnalyzerProvider.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiAnalyzerProvider.java index 2edfe3e4725b1..db336bec997c4 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiAnalyzerProvider.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.kuromoji; diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiBaseFormFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiBaseFormFilterFactory.java index 81f51e5ec4247..536e6fe993d06 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiBaseFormFilterFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiBaseFormFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.kuromoji; diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiCompletionAnalyzerProvider.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiCompletionAnalyzerProvider.java index 70052994519e1..c4970251d2bb6 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiCompletionAnalyzerProvider.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiCompletionAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.kuromoji; diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiCompletionFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiCompletionFilterFactory.java index a5a5bfd75da93..3ec6d08145e69 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiCompletionFilterFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiCompletionFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.kuromoji; diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiIterationMarkCharFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiIterationMarkCharFilterFactory.java index 54d22ac1b0c8f..37265f3ff9228 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiIterationMarkCharFilterFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiIterationMarkCharFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.kuromoji; diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiKatakanaStemmerFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiKatakanaStemmerFactory.java index f620fbc39c580..7c4d3138381fb 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiKatakanaStemmerFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiKatakanaStemmerFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.kuromoji; diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiNumberFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiNumberFilterFactory.java index f3b66e929e3fa..089b7e10bfae0 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiNumberFilterFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiNumberFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.kuromoji; diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiPartOfSpeechFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiPartOfSpeechFilterFactory.java index f5fb8479ed7ba..e8efa781726f8 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiPartOfSpeechFilterFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiPartOfSpeechFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.kuromoji; diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiReadingFormFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiReadingFormFilterFactory.java index ddd7f04db68dd..09ab0bbd4b8d8 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiReadingFormFilterFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiReadingFormFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.kuromoji; diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiTokenizerFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiTokenizerFactory.java index 65276c04bed56..edb29a8f4c98e 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiTokenizerFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiTokenizerFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.kuromoji; @@ -32,6 +33,7 @@ public class KuromojiTokenizerFactory extends AbstractTokenizerFactory { private static final String NBEST_COST = "nbest_cost"; private static final String NBEST_EXAMPLES = "nbest_examples"; private static final String DISCARD_COMPOUND_TOKEN = "discard_compound_token"; + private static final String LENIENT = "lenient"; private final UserDictionary userDictionary; private final Mode mode; @@ -57,7 +59,15 @@ public static UserDictionary getUserDictionary(Environment env, Settings setting "It is not allowed to use [" + USER_DICT_PATH_OPTION + "] in conjunction" + " with [" + USER_DICT_RULES_OPTION + "]" ); } - List ruleList = Analysis.getWordList(env, settings, USER_DICT_PATH_OPTION, USER_DICT_RULES_OPTION, false, true); + List ruleList = Analysis.getWordList( + env, + settings, + USER_DICT_PATH_OPTION, + USER_DICT_RULES_OPTION, + LENIENT, + false, // typically don't want to remove comments as deduplication will provide better feedback + true + ); if (ruleList == null || ruleList.isEmpty()) { return null; } @@ -65,6 +75,7 @@ public static UserDictionary getUserDictionary(Environment env, Settings setting for (String line : ruleList) { sb.append(line).append(System.lineSeparator()); } + try (Reader rulesReader = new StringReader(sb.toString())) { return UserDictionary.open(rulesReader); } catch (IOException e) { diff --git a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiFactoryTests.java b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiFactoryTests.java index 901db28820b71..c00668ebba731 100644 --- a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiFactoryTests.java +++ b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.kuromoji; diff --git a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiAnalysisTests.java b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiAnalysisTests.java index f1a594621fb3e..f26213d86c5a9 100644 --- a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiAnalysisTests.java +++ b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiAnalysisTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.kuromoji; @@ -444,7 +445,26 @@ public void testKuromojiAnalyzerDuplicateUserDictRule() throws Exception { ) .build(); IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> createTestAnalysis(settings)); - assertThat(exc.getMessage(), containsString("[制限スピード] in user dictionary at line [3]")); + assertThat(exc.getMessage(), containsString("[制限スピード] in user dictionary at line [4]")); + } + + public void testKuromojiAnalyzerDuplicateUserDictRuleDeduplication() throws Exception { + Settings settings = Settings.builder() + .put("index.analysis.analyzer.my_analyzer.type", "kuromoji") + .put("index.analysis.analyzer.my_analyzer.lenient", "true") + .putList( + "index.analysis.analyzer.my_analyzer.user_dictionary_rules", + "c++,c++,w,w", + "#comment", + "制限スピード,制限スピード,セイゲンスピード,テスト名詞", + "制限スピード,制限スピード,セイゲンスピード,テスト名詞" + ) + .build(); + TestAnalysis analysis = createTestAnalysis(settings); + Analyzer analyzer = analysis.indexAnalyzers.get("my_analyzer"); + try (TokenStream stream = analyzer.tokenStream("", "制限スピード")) { + assertTokenStreamContents(stream, new String[] { "制限スピード" }); + } } public void testDiscardCompoundToken() throws Exception { diff --git a/plugins/analysis-kuromoji/src/yamlRestTest/java/org/elasticsearch/index/analysis/KuromojiClientYamlTestSuiteIT.java b/plugins/analysis-kuromoji/src/yamlRestTest/java/org/elasticsearch/index/analysis/KuromojiClientYamlTestSuiteIT.java index 41e2e420abb05..75ae8a744c8fb 100644 --- a/plugins/analysis-kuromoji/src/yamlRestTest/java/org/elasticsearch/index/analysis/KuromojiClientYamlTestSuiteIT.java +++ b/plugins/analysis-kuromoji/src/yamlRestTest/java/org/elasticsearch/index/analysis/KuromojiClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.analysis; diff --git a/plugins/analysis-nori/build.gradle b/plugins/analysis-nori/build.gradle index 51e93bf6cc2c4..6254a56f0657f 100644 --- a/plugins/analysis-nori/build.gradle +++ b/plugins/analysis-nori/build.gradle @@ -1,16 +1,17 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.legacy-yaml-rest-test' apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' esplugin { - description 'The Korean (nori) Analysis plugin integrates Lucene nori analysis module into elasticsearch.' - classname 'org.elasticsearch.plugin.analysis.nori.AnalysisNoriPlugin' + description = 'The Korean (nori) Analysis plugin integrates Lucene nori analysis module into elasticsearch.' + classname ='org.elasticsearch.plugin.analysis.nori.AnalysisNoriPlugin' } dependencies { diff --git a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/AnalysisNoriPlugin.java b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/AnalysisNoriPlugin.java index fd544ef42b027..3e418852f9ecd 100644 --- a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/AnalysisNoriPlugin.java +++ b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/AnalysisNoriPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.nori; diff --git a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriAnalyzerProvider.java b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriAnalyzerProvider.java index 65c9bb9833177..180f6aa0a7f96 100644 --- a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriAnalyzerProvider.java +++ b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.nori; diff --git a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriNumberFilterFactory.java b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriNumberFilterFactory.java index d75ead01bf1f0..cbe92156cd765 100644 --- a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriNumberFilterFactory.java +++ b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriNumberFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.nori; diff --git a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriPartOfSpeechStopFilterFactory.java b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriPartOfSpeechStopFilterFactory.java index 84dc8acbb32ac..dddb485ab0df3 100644 --- a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriPartOfSpeechStopFilterFactory.java +++ b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriPartOfSpeechStopFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.nori; diff --git a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriReadingFormFilterFactory.java b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriReadingFormFilterFactory.java index 7a23039b3d810..1e1b211fdea09 100644 --- a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriReadingFormFilterFactory.java +++ b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriReadingFormFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.nori; diff --git a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriTokenizerFactory.java b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriTokenizerFactory.java index eedb4c2011af3..ed8458bc94043 100644 --- a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriTokenizerFactory.java +++ b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriTokenizerFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.nori; @@ -30,6 +31,7 @@ public class NoriTokenizerFactory extends AbstractTokenizerFactory { private static final String USER_DICT_PATH_OPTION = "user_dictionary"; private static final String USER_DICT_RULES_OPTION = "user_dictionary_rules"; + private static final String LENIENT = "lenient"; private final UserDictionary userDictionary; private final KoreanTokenizer.DecompoundMode decompoundMode; @@ -53,7 +55,8 @@ public static UserDictionary getUserDictionary(Environment env, Settings setting settings, USER_DICT_PATH_OPTION, USER_DICT_RULES_OPTION, - true, + LENIENT, + false, // typically don't want to remove comments as deduplication will provide better feedback isSupportDuplicateCheck(indexSettings) ); if (ruleList == null || ruleList.isEmpty()) { diff --git a/plugins/analysis-nori/src/test/java/org/elasticsearch/plugin/analysis/nori/AnalysisNoriFactoryTests.java b/plugins/analysis-nori/src/test/java/org/elasticsearch/plugin/analysis/nori/AnalysisNoriFactoryTests.java index 44ba5e4148115..4eb76b3582d9f 100644 --- a/plugins/analysis-nori/src/test/java/org/elasticsearch/plugin/analysis/nori/AnalysisNoriFactoryTests.java +++ b/plugins/analysis-nori/src/test/java/org/elasticsearch/plugin/analysis/nori/AnalysisNoriFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.nori; diff --git a/plugins/analysis-nori/src/test/java/org/elasticsearch/plugin/analysis/nori/NoriAnalysisTests.java b/plugins/analysis-nori/src/test/java/org/elasticsearch/plugin/analysis/nori/NoriAnalysisTests.java index 642ed19c520d7..f9a0b7d5c5216 100644 --- a/plugins/analysis-nori/src/test/java/org/elasticsearch/plugin/analysis/nori/NoriAnalysisTests.java +++ b/plugins/analysis-nori/src/test/java/org/elasticsearch/plugin/analysis/nori/NoriAnalysisTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.nori; @@ -23,7 +24,6 @@ import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.analysis.TokenizerFactory; -import org.elasticsearch.test.ESTestCase.TestAnalysis; import org.elasticsearch.test.ESTokenStreamTestCase; import java.io.IOException; @@ -32,6 +32,7 @@ import java.nio.file.Files; import java.nio.file.Path; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.instanceOf; @@ -126,7 +127,7 @@ public void testNoriAnalyzerDuplicateUserDictRule() throws Exception { .build(); final IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> createTestAnalysis(settings)); - assertThat(exc.getMessage(), containsString("[세종] in user dictionary at line [3]")); + assertThat(exc.getMessage(), containsString("[세종] in user dictionary at line [4]")); } public void testNoriAnalyzerDuplicateUserDictRuleWithLegacyVersion() throws IOException { @@ -143,6 +144,20 @@ public void testNoriAnalyzerDuplicateUserDictRuleWithLegacyVersion() throws IOEx } } + public void testNoriAnalyzerDuplicateUserDictRuleDeduplication() throws Exception { + Settings settings = Settings.builder() + .put("index.analysis.analyzer.my_analyzer.type", "nori") + .put("index.analysis.analyzer.my_analyzer.lenient", "true") + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersions.NORI_DUPLICATES) + .putList("index.analysis.analyzer.my_analyzer.user_dictionary_rules", "c++", "C쁠쁠", "세종", "세종", "세종시 세종 시") + .build(); + TestAnalysis analysis = createTestAnalysis(settings); + Analyzer analyzer = analysis.indexAnalyzers.get("my_analyzer"); + try (TokenStream stream = analyzer.tokenStream("", "세종시")) { + assertTokenStreamContents(stream, new String[] { "세종", "시" }); + } + } + public void testNoriTokenizer() throws Exception { Settings settings = Settings.builder() .put("index.analysis.tokenizer.my_tokenizer.type", "nori_tokenizer") diff --git a/plugins/analysis-nori/src/yamlRestTest/java/org/elasticsearch/index/analysis/NoriClientYamlTestSuiteIT.java b/plugins/analysis-nori/src/yamlRestTest/java/org/elasticsearch/index/analysis/NoriClientYamlTestSuiteIT.java index 4c7238ebc9e2e..52f3fa5944277 100644 --- a/plugins/analysis-nori/src/yamlRestTest/java/org/elasticsearch/index/analysis/NoriClientYamlTestSuiteIT.java +++ b/plugins/analysis-nori/src/yamlRestTest/java/org/elasticsearch/index/analysis/NoriClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.analysis; diff --git a/plugins/analysis-phonetic/build.gradle b/plugins/analysis-phonetic/build.gradle index 7646c0ee874dc..018e2c0e52b8d 100644 --- a/plugins/analysis-phonetic/build.gradle +++ b/plugins/analysis-phonetic/build.gradle @@ -1,16 +1,17 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.legacy-yaml-rest-test' apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' esplugin { - description 'The Phonetic Analysis plugin integrates phonetic token filter analysis with elasticsearch.' - classname 'org.elasticsearch.plugin.analysis.phonetic.AnalysisPhoneticPlugin' + description = 'The Phonetic Analysis plugin integrates phonetic token filter analysis with elasticsearch.' + classname ='org.elasticsearch.plugin.analysis.phonetic.AnalysisPhoneticPlugin' } dependencies { diff --git a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/phonetic/AnalysisPhoneticPlugin.java b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/phonetic/AnalysisPhoneticPlugin.java index 47562ea372b34..39e0a5c94cc0a 100644 --- a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/phonetic/AnalysisPhoneticPlugin.java +++ b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/phonetic/AnalysisPhoneticPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.phonetic; diff --git a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/phonetic/HaasePhonetik.java b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/phonetic/HaasePhonetik.java index 3132c6bed2b49..6896cb4159c51 100644 --- a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/phonetic/HaasePhonetik.java +++ b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/phonetic/HaasePhonetik.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.phonetic; diff --git a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/phonetic/KoelnerPhonetik.java b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/phonetic/KoelnerPhonetik.java index 0076ecc705759..42cf4d8c0cb58 100644 --- a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/phonetic/KoelnerPhonetik.java +++ b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/phonetic/KoelnerPhonetik.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.phonetic; diff --git a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/phonetic/PhoneticTokenFilterFactory.java b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/phonetic/PhoneticTokenFilterFactory.java index 65a790dc576c0..786c6230349a4 100644 --- a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/phonetic/PhoneticTokenFilterFactory.java +++ b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/phonetic/PhoneticTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.phonetic; diff --git a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/plugin/analysis/phonetic/AnalysisPhoneticFactoryTests.java b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/plugin/analysis/phonetic/AnalysisPhoneticFactoryTests.java index 348e9f5fae7c8..4606bc0e0a8d2 100644 --- a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/plugin/analysis/phonetic/AnalysisPhoneticFactoryTests.java +++ b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/plugin/analysis/phonetic/AnalysisPhoneticFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.phonetic; diff --git a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/plugin/analysis/phonetic/SimplePhoneticAnalysisTests.java b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/plugin/analysis/phonetic/SimplePhoneticAnalysisTests.java index a6c9035b8ecc1..ad06f1036120b 100644 --- a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/plugin/analysis/phonetic/SimplePhoneticAnalysisTests.java +++ b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/plugin/analysis/phonetic/SimplePhoneticAnalysisTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.phonetic; diff --git a/plugins/analysis-phonetic/src/yamlRestTest/java/org/elasticsearch/index/analysis/PhoneticClientYamlTestSuiteIT.java b/plugins/analysis-phonetic/src/yamlRestTest/java/org/elasticsearch/index/analysis/PhoneticClientYamlTestSuiteIT.java index ae9f25d4bd0da..66589ee523972 100644 --- a/plugins/analysis-phonetic/src/yamlRestTest/java/org/elasticsearch/index/analysis/PhoneticClientYamlTestSuiteIT.java +++ b/plugins/analysis-phonetic/src/yamlRestTest/java/org/elasticsearch/index/analysis/PhoneticClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.analysis; diff --git a/plugins/analysis-smartcn/build.gradle b/plugins/analysis-smartcn/build.gradle index c75c5c304048b..b4ac03935aab5 100644 --- a/plugins/analysis-smartcn/build.gradle +++ b/plugins/analysis-smartcn/build.gradle @@ -1,16 +1,17 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.legacy-yaml-rest-test' apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' esplugin { - description 'Smart Chinese Analysis plugin integrates Lucene Smart Chinese analysis module into elasticsearch.' - classname 'org.elasticsearch.plugin.analysis.smartcn.AnalysisSmartChinesePlugin' + description = 'Smart Chinese Analysis plugin integrates Lucene Smart Chinese analysis module into elasticsearch.' + classname ='org.elasticsearch.plugin.analysis.smartcn.AnalysisSmartChinesePlugin' } dependencies { diff --git a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChinesePlugin.java b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChinesePlugin.java index 555cef1cc95dc..9f2063b817003 100644 --- a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChinesePlugin.java +++ b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChinesePlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.smartcn; diff --git a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseAnalyzerProvider.java b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseAnalyzerProvider.java index 55e2c967fd438..668b94d0ca972 100644 --- a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseAnalyzerProvider.java +++ b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.smartcn; diff --git a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseNoOpTokenFilterFactory.java b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseNoOpTokenFilterFactory.java index 677b483dfcec3..55cda67852272 100644 --- a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseNoOpTokenFilterFactory.java +++ b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseNoOpTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.smartcn; diff --git a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseStopTokenFilterFactory.java b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseStopTokenFilterFactory.java index a88740282f267..2463ad4a2c188 100644 --- a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseStopTokenFilterFactory.java +++ b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseStopTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.smartcn; diff --git a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseTokenizerTokenizerFactory.java b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseTokenizerTokenizerFactory.java index 9266f0515491f..2545c9c7d94e8 100644 --- a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseTokenizerTokenizerFactory.java +++ b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/SmartChineseTokenizerTokenizerFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.smartcn; diff --git a/plugins/analysis-smartcn/src/test/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChineseFactoryTests.java b/plugins/analysis-smartcn/src/test/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChineseFactoryTests.java index 23d0b1a21a204..7956014baf4f7 100644 --- a/plugins/analysis-smartcn/src/test/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChineseFactoryTests.java +++ b/plugins/analysis-smartcn/src/test/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChineseFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.smartcn; diff --git a/plugins/analysis-smartcn/src/test/java/org/elasticsearch/plugin/analysis/smartcn/SimpleSmartChineseAnalysisTests.java b/plugins/analysis-smartcn/src/test/java/org/elasticsearch/plugin/analysis/smartcn/SimpleSmartChineseAnalysisTests.java index 6187f65e3cd30..09c27ff127885 100644 --- a/plugins/analysis-smartcn/src/test/java/org/elasticsearch/plugin/analysis/smartcn/SimpleSmartChineseAnalysisTests.java +++ b/plugins/analysis-smartcn/src/test/java/org/elasticsearch/plugin/analysis/smartcn/SimpleSmartChineseAnalysisTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.smartcn; diff --git a/plugins/analysis-smartcn/src/yamlRestTest/java/org/elasticsearch/index/analysis/SmartCNClientYamlTestSuiteIT.java b/plugins/analysis-smartcn/src/yamlRestTest/java/org/elasticsearch/index/analysis/SmartCNClientYamlTestSuiteIT.java index 1d83ba8156916..e0eff356f8bb7 100644 --- a/plugins/analysis-smartcn/src/yamlRestTest/java/org/elasticsearch/index/analysis/SmartCNClientYamlTestSuiteIT.java +++ b/plugins/analysis-smartcn/src/yamlRestTest/java/org/elasticsearch/index/analysis/SmartCNClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.analysis; diff --git a/plugins/analysis-stempel/build.gradle b/plugins/analysis-stempel/build.gradle index ca6d91df5cd11..0fb15ec7d36d1 100644 --- a/plugins/analysis-stempel/build.gradle +++ b/plugins/analysis-stempel/build.gradle @@ -1,16 +1,17 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.legacy-yaml-rest-test' apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' esplugin { - description 'The Stempel (Polish) Analysis plugin integrates Lucene stempel (polish) analysis module into elasticsearch.' - classname 'org.elasticsearch.plugin.analysis.stempel.AnalysisStempelPlugin' + description = 'The Stempel (Polish) Analysis plugin integrates Lucene stempel (polish) analysis module into elasticsearch.' + classname ='org.elasticsearch.plugin.analysis.stempel.AnalysisStempelPlugin' } dependencies { diff --git a/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishAnalyzerProvider.java b/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishAnalyzerProvider.java index 1d57c65717cc6..68e7298473cd4 100644 --- a/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishAnalyzerProvider.java +++ b/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.analysis.pl; diff --git a/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishStemTokenFilterFactory.java b/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishStemTokenFilterFactory.java index 6887dc661af85..76ca03e6228a4 100644 --- a/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishStemTokenFilterFactory.java +++ b/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishStemTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.analysis.pl; diff --git a/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishStopTokenFilterFactory.java b/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishStopTokenFilterFactory.java index 3ec480a4f85b2..814bc7dfe56c6 100644 --- a/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishStopTokenFilterFactory.java +++ b/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishStopTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.analysis.pl; diff --git a/plugins/analysis-stempel/src/main/java/org/elasticsearch/plugin/analysis/stempel/AnalysisStempelPlugin.java b/plugins/analysis-stempel/src/main/java/org/elasticsearch/plugin/analysis/stempel/AnalysisStempelPlugin.java index 5d3b9e873b3e0..df4b8d16af096 100644 --- a/plugins/analysis-stempel/src/main/java/org/elasticsearch/plugin/analysis/stempel/AnalysisStempelPlugin.java +++ b/plugins/analysis-stempel/src/main/java/org/elasticsearch/plugin/analysis/stempel/AnalysisStempelPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.stempel; diff --git a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/AnalysisPolishFactoryTests.java b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/AnalysisPolishFactoryTests.java index 10590c8ef6b11..aa7f773b86206 100644 --- a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/AnalysisPolishFactoryTests.java +++ b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/AnalysisPolishFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.analysis; diff --git a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/PolishAnalysisTests.java b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/PolishAnalysisTests.java index fafbcafb3189b..458b4bacab3fa 100644 --- a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/PolishAnalysisTests.java +++ b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/PolishAnalysisTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.analysis; diff --git a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/SimplePolishTokenFilterTests.java b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/SimplePolishTokenFilterTests.java index 671cb2e4f15cc..071a36bdb52b5 100644 --- a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/SimplePolishTokenFilterTests.java +++ b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/SimplePolishTokenFilterTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.analysis; diff --git a/plugins/analysis-stempel/src/yamlRestTest/java/org/elasticsearch/index/analysis/StempelClientYamlTestSuiteIT.java b/plugins/analysis-stempel/src/yamlRestTest/java/org/elasticsearch/index/analysis/StempelClientYamlTestSuiteIT.java index faee6493920c8..6ef5d037577dd 100644 --- a/plugins/analysis-stempel/src/yamlRestTest/java/org/elasticsearch/index/analysis/StempelClientYamlTestSuiteIT.java +++ b/plugins/analysis-stempel/src/yamlRestTest/java/org/elasticsearch/index/analysis/StempelClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.analysis; diff --git a/plugins/analysis-ukrainian/build.gradle b/plugins/analysis-ukrainian/build.gradle index 709b8831628b3..2be48240a8875 100644 --- a/plugins/analysis-ukrainian/build.gradle +++ b/plugins/analysis-ukrainian/build.gradle @@ -1,16 +1,17 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.legacy-yaml-rest-test' apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' esplugin { - description 'The Ukrainian Analysis plugin integrates the Lucene UkrainianMorfologikAnalyzer into elasticsearch.' - classname 'org.elasticsearch.plugin.analysis.ukrainian.AnalysisUkrainianPlugin' + description = 'The Ukrainian Analysis plugin integrates the Lucene UkrainianMorfologikAnalyzer into elasticsearch.' + classname ='org.elasticsearch.plugin.analysis.ukrainian.AnalysisUkrainianPlugin' } dependencies { diff --git a/plugins/analysis-ukrainian/src/main/java/org/elasticsearch/plugin/analysis/ukrainian/AnalysisUkrainianPlugin.java b/plugins/analysis-ukrainian/src/main/java/org/elasticsearch/plugin/analysis/ukrainian/AnalysisUkrainianPlugin.java index 8200fd03a5b83..178b498dbb8e1 100644 --- a/plugins/analysis-ukrainian/src/main/java/org/elasticsearch/plugin/analysis/ukrainian/AnalysisUkrainianPlugin.java +++ b/plugins/analysis-ukrainian/src/main/java/org/elasticsearch/plugin/analysis/ukrainian/AnalysisUkrainianPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.ukrainian; diff --git a/plugins/analysis-ukrainian/src/main/java/org/elasticsearch/plugin/analysis/ukrainian/UkrainianAnalyzerProvider.java b/plugins/analysis-ukrainian/src/main/java/org/elasticsearch/plugin/analysis/ukrainian/UkrainianAnalyzerProvider.java index 455a0b0464889..8ea493b90be3c 100644 --- a/plugins/analysis-ukrainian/src/main/java/org/elasticsearch/plugin/analysis/ukrainian/UkrainianAnalyzerProvider.java +++ b/plugins/analysis-ukrainian/src/main/java/org/elasticsearch/plugin/analysis/ukrainian/UkrainianAnalyzerProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.ukrainian; diff --git a/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/plugin/analysis/ukrainian/SimpleUkrainianAnalyzerTests.java b/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/plugin/analysis/ukrainian/SimpleUkrainianAnalyzerTests.java index c774405aa36d1..18ba55615f63a 100644 --- a/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/plugin/analysis/ukrainian/SimpleUkrainianAnalyzerTests.java +++ b/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/plugin/analysis/ukrainian/SimpleUkrainianAnalyzerTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.ukrainian; diff --git a/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/plugin/analysis/ukrainian/UkrainianAnalysisTests.java b/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/plugin/analysis/ukrainian/UkrainianAnalysisTests.java index 4b4ea95849b75..baca4d2419cb5 100644 --- a/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/plugin/analysis/ukrainian/UkrainianAnalysisTests.java +++ b/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/plugin/analysis/ukrainian/UkrainianAnalysisTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.analysis.ukrainian; diff --git a/plugins/analysis-ukrainian/src/yamlRestTest/java/org/elasticsearch/index/analysis/UkrainianClientYamlTestSuiteIT.java b/plugins/analysis-ukrainian/src/yamlRestTest/java/org/elasticsearch/index/analysis/UkrainianClientYamlTestSuiteIT.java index e02036548c0ad..9a1d14a6ceee2 100644 --- a/plugins/analysis-ukrainian/src/yamlRestTest/java/org/elasticsearch/index/analysis/UkrainianClientYamlTestSuiteIT.java +++ b/plugins/analysis-ukrainian/src/yamlRestTest/java/org/elasticsearch/index/analysis/UkrainianClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.analysis; diff --git a/plugins/build.gradle b/plugins/build.gradle index 273c299a6e65f..32fd646ef0be8 100644 --- a/plugins/build.gradle +++ b/plugins/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ subprojects { @@ -21,10 +22,10 @@ configure(subprojects.findAll { it.parent.path == project.path }) { esplugin { // for local ES plugins, the name of the plugin is the same as the directory - name project.name + name = project.name - licenseFile rootProject.file('licenses/SSPL-1.0+ELASTIC-LICENSE-2.0.txt') - noticeFile rootProject.file('NOTICE.txt') + licenseFile = rootProject.file('licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt') + noticeFile = rootProject.file('NOTICE.txt') } diff --git a/plugins/discovery-azure-classic/build.gradle b/plugins/discovery-azure-classic/build.gradle index e6a4f7e207309..8608ee05c7712 100644 --- a/plugins/discovery-azure-classic/build.gradle +++ b/plugins/discovery-azure-classic/build.gradle @@ -3,17 +3,18 @@ import org.elasticsearch.gradle.internal.info.BuildParams /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.legacy-yaml-rest-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { - description 'The Azure Classic Discovery plugin allows to use Azure Classic API for the unicast discovery mechanism' - classname 'org.elasticsearch.plugin.discovery.azure.classic.AzureDiscoveryPlugin' + description = 'The Azure Classic Discovery plugin allows to use Azure Classic API for the unicast discovery mechanism' + classname ='org.elasticsearch.plugin.discovery.azure.classic.AzureDiscoveryPlugin' } def localVersions = versions + [ @@ -62,7 +63,7 @@ TaskProvider createKey = tasks.register("createKey", LoggedExec) { keystore.parentFile.mkdirs() } outputs.file(keystore).withPropertyName('keystoreFile') - executable = "${BuildParams.runtimeJavaHome}/bin/keytool" + executable = "${buildParams.runtimeJavaHome.get()}/bin/keytool" getStandardInput().set('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n') args '-genkey', '-alias', 'test-node', diff --git a/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/cloud/azure/classic/AbstractAzureComputeServiceTestCase.java b/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/cloud/azure/classic/AbstractAzureComputeServiceTestCase.java index dbcf64bef33e9..58c47c63ddfb5 100644 --- a/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/cloud/azure/classic/AbstractAzureComputeServiceTestCase.java +++ b/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/cloud/azure/classic/AbstractAzureComputeServiceTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cloud.azure.classic; diff --git a/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java b/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java index 349e54f4dd25f..30759f27c5356 100644 --- a/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java +++ b/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.azure.classic; diff --git a/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/discovery/azure/classic/AzureSimpleTests.java b/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/discovery/azure/classic/AzureSimpleTests.java index a3e8a3a02f93b..c8447bc9757df 100644 --- a/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/discovery/azure/classic/AzureSimpleTests.java +++ b/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/discovery/azure/classic/AzureSimpleTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.azure.classic; diff --git a/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/discovery/azure/classic/AzureTwoStartedNodesTests.java b/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/discovery/azure/classic/AzureTwoStartedNodesTests.java index 6c07670266278..a4ba0338010b1 100644 --- a/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/discovery/azure/classic/AzureTwoStartedNodesTests.java +++ b/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/discovery/azure/classic/AzureTwoStartedNodesTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.azure.classic; diff --git a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/AzureServiceDisableException.java b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/AzureServiceDisableException.java index e50190ca4971a..3917a2309f45b 100644 --- a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/AzureServiceDisableException.java +++ b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/AzureServiceDisableException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cloud.azure.classic; diff --git a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/AzureServiceRemoteException.java b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/AzureServiceRemoteException.java index bd9615e538af9..6713d2ad96b00 100644 --- a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/AzureServiceRemoteException.java +++ b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/AzureServiceRemoteException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cloud.azure.classic; diff --git a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeService.java b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeService.java index 6579efa1d8733..6d2df9231a090 100644 --- a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeService.java +++ b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cloud.azure.classic.management; diff --git a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceImpl.java b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceImpl.java index ebcbfd027d386..3c5213c2c49cd 100644 --- a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceImpl.java +++ b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceImpl.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cloud.azure.classic.management; diff --git a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/discovery/azure/classic/AzureSeedHostsProvider.java b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/discovery/azure/classic/AzureSeedHostsProvider.java index 0cb46f1509fcf..bd8e344d35b7e 100644 --- a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/discovery/azure/classic/AzureSeedHostsProvider.java +++ b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/discovery/azure/classic/AzureSeedHostsProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.azure.classic; diff --git a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/plugin/discovery/azure/classic/AzureDiscoveryPlugin.java b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/plugin/discovery/azure/classic/AzureDiscoveryPlugin.java index a32c7dc638bbb..7d40ee73ca97e 100644 --- a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/plugin/discovery/azure/classic/AzureDiscoveryPlugin.java +++ b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/plugin/discovery/azure/classic/AzureDiscoveryPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.discovery.azure.classic; diff --git a/plugins/discovery-azure-classic/src/main/plugin-metadata/plugin-security.policy b/plugins/discovery-azure-classic/src/main/plugin-metadata/plugin-security.policy index 0d42ee2a4ba81..d766397d7b1b4 100644 --- a/plugins/discovery-azure-classic/src/main/plugin-metadata/plugin-security.policy +++ b/plugins/discovery-azure-classic/src/main/plugin-metadata/plugin-security.policy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ grant { diff --git a/plugins/discovery-azure-classic/src/yamlRestTest/java/org/elasticsearch/discovery/azure/classic/DiscoveryAzureClassicClientYamlTestSuiteIT.java b/plugins/discovery-azure-classic/src/yamlRestTest/java/org/elasticsearch/discovery/azure/classic/DiscoveryAzureClassicClientYamlTestSuiteIT.java index cc20fdbbf5fd3..711d033ecfca2 100644 --- a/plugins/discovery-azure-classic/src/yamlRestTest/java/org/elasticsearch/discovery/azure/classic/DiscoveryAzureClassicClientYamlTestSuiteIT.java +++ b/plugins/discovery-azure-classic/src/yamlRestTest/java/org/elasticsearch/discovery/azure/classic/DiscoveryAzureClassicClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.azure.classic; diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle index b57d6bce26633..a833c5d6c4642 100644 --- a/plugins/discovery-ec2/build.gradle +++ b/plugins/discovery-ec2/build.gradle @@ -2,26 +2,23 @@ import org.elasticsearch.gradle.internal.info.BuildParams /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ -apply plugin: 'elasticsearch.legacy-yaml-rest-test' +apply plugin: 'elasticsearch.internal-java-rest-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { - description 'The EC2 discovery plugin allows to use AWS API for the unicast discovery mechanism.' - classname 'org.elasticsearch.discovery.ec2.Ec2DiscoveryPlugin' + description = 'The EC2 discovery plugin allows to use AWS API for the unicast discovery mechanism.' + classname ='org.elasticsearch.discovery.ec2.Ec2DiscoveryPlugin' } -versions << [ - 'aws': '1.12.270' -] - dependencies { - api "com.amazonaws:aws-java-sdk-ec2:${versions.aws}" - api "com.amazonaws:aws-java-sdk-core:${versions.aws}" + api "com.amazonaws:aws-java-sdk-ec2:${versions.awsv1sdk}" + api "com.amazonaws:aws-java-sdk-core:${versions.awsv1sdk}" api "org.apache.httpcomponents:httpclient:${versions.httpclient}" api "org.apache.httpcomponents:httpcore:${versions.httpcore}" api "commons-logging:commons-logging:${versions.commonslogging}" @@ -32,12 +29,11 @@ dependencies { api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" api "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}" api "joda-time:joda-time:2.10.10" -} -restResources { - restApi { - include '_common', 'cluster', 'nodes' - } + javaRestTestImplementation project(':plugins:discovery-ec2') + javaRestTestImplementation project(':test:fixtures:ec2-imds-fixture') + + internalClusterTestImplementation project(':test:fixtures:ec2-imds-fixture') } tasks.named("dependencyLicenses").configure { @@ -56,7 +52,7 @@ tasks.register("writeTestJavaPolicy") { throw new GradleException("failed to create temporary directory [${tmp}]") } final File javaPolicy = file("${tmp}/java.policy") - if (BuildParams.inFipsJvm) { + if (buildParams.inFipsJvm) { javaPolicy.write( [ "grant {", @@ -76,6 +72,7 @@ tasks.register("writeTestJavaPolicy") { "permission java.security.SecurityPermission \"getProperty.jdk.tls.disabledAlgorithms\";", "permission java.security.SecurityPermission \"getProperty.jdk.certpath.disabledAlgorithms\";", "permission java.security.SecurityPermission \"getProperty.keystore.type.compat\";", + "permission java.security.SecurityPermission \"getProperty.org.bouncycastle.ec.max_f2m_field_size\";", "};" ].join("\n") ) @@ -90,14 +87,14 @@ tasks.register("writeTestJavaPolicy") { } } -tasks.named("test").configure { +tasks.withType(Test).configureEach { dependsOn "writeTestJavaPolicy" // this is needed for insecure plugins, remove if possible! systemProperty 'tests.artifact', project.name // this is needed to manipulate com.amazonaws.sdk.ec2MetadataServiceEndpointOverride system property // it is better rather disable security manager at all with `systemProperty 'tests.security.manager', 'false'` - if (BuildParams.inFipsJvm){ + if (buildParams.inFipsJvm){ nonInputProperties.systemProperty 'java.security.policy', "=file://${buildDir}/tmp/java.policy" } else { nonInputProperties.systemProperty 'java.security.policy', "file://${buildDir}/tmp/java.policy" diff --git a/plugins/discovery-ec2/qa/amazon-ec2/build.gradle b/plugins/discovery-ec2/qa/amazon-ec2/build.gradle index c7c4ecba40fec..5f0fee6636256 100644 --- a/plugins/discovery-ec2/qa/amazon-ec2/build.gradle +++ b/plugins/discovery-ec2/qa/amazon-ec2/build.gradle @@ -1,13 +1,13 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.apache.tools.ant.filters.ReplaceTokens -import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.internal.test.AntFixture import org.elasticsearch.gradle.internal.test.RestIntegTestTask import org.elasticsearch.gradle.internal.test.rest.LegacyYamlRestTestPlugin @@ -54,8 +54,9 @@ tasks.named("yamlRestTest").configure { enabled = false } ['KeyStore', 'EnvVariables', 'SystemProperties', 'ContainerCredentials', 'InstanceProfile'].forEach { action -> TaskProvider fixture = tasks.register("ec2Fixture${action}", AntFixture) { dependsOn project.sourceSets.yamlRestTest.runtimeClasspath - env 'CLASSPATH', "${-> project.sourceSets.yamlRestTest.runtimeClasspath.asPath}" - executable = "${BuildParams.runtimeJavaHome}/bin/java" + FileCollection cp = project.sourceSets.yamlRestTest.runtimeClasspath + env 'CLASSPATH', "${-> cp.asPath}" + executable = "${buildParams.runtimeJavaHome.get() }/bin/java" args 'org.elasticsearch.discovery.ec2.AmazonEC2Fixture', baseDir, "${buildDir}/testclusters/yamlRestTest${action}-1/config/unicast_hosts.txt" } @@ -67,9 +68,18 @@ tasks.named("yamlRestTest").configure { enabled = false } classpath = yamlRestTestSourceSet.getRuntimeClasspath() } + if(action == 'ContainerCredentials') { + def addressAndPortSource = fixture.get().addressAndPortSource + testClusters.matching { it.name == "yamlRestTestContainerCredentials" }.configureEach { + environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', + () -> addressAndPortSource.map{ addr -> "http://${addr}/ecs_credentials_endpoint" }.get(), IGNORE_VALUE + } + } + tasks.named("check").configure { dependsOn(yamlRestTestTask) } + def addressAndPortSource = fixture.get().addressAndPortSource testClusters.matching { it.name == yamlRestTestTask.name}.configureEach { numberOfNodes = ec2NumberOfNodes @@ -77,9 +87,9 @@ tasks.named("yamlRestTest").configure { enabled = false } setting 'discovery.seed_providers', 'ec2' setting 'network.host', '_ec2_' - setting 'discovery.ec2.endpoint', { "http://${-> fixture.get().addressAndPort}" }, IGNORE_VALUE + setting 'discovery.ec2.endpoint', { "http://${-> addressAndPortSource.get()}" }, IGNORE_VALUE - systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "http://${-> fixture.get().addressAndPort}" }, IGNORE_VALUE + systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "http://${-> addressAndPortSource.get()}" }, IGNORE_VALUE } } @@ -106,11 +116,6 @@ tasks.named("ec2FixtureContainerCredentials").configure { env 'ACTIVATE_CONTAINER_CREDENTIALS', true } -testClusters.matching { it.name == "yamlRestTestContainerCredentials" }.configureEach { - environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', - { "http://${-> tasks.findByName("ec2FixtureContainerCredentials").addressAndPort}/ecs_credentials_endpoint" }, IGNORE_VALUE -} - // Extra config for InstanceProfile tasks.named("ec2FixtureInstanceProfile").configure { env 'ACTIVATE_INSTANCE_PROFILE', true diff --git a/plugins/discovery-ec2/qa/amazon-ec2/src/yamlRestTest/java/org/elasticsearch/discovery/ec2/AmazonEC2DiscoveryClientYamlTestSuiteIT.java b/plugins/discovery-ec2/qa/amazon-ec2/src/yamlRestTest/java/org/elasticsearch/discovery/ec2/AmazonEC2DiscoveryClientYamlTestSuiteIT.java index b5853b9fefc5a..a7cb6113502e7 100644 --- a/plugins/discovery-ec2/qa/amazon-ec2/src/yamlRestTest/java/org/elasticsearch/discovery/ec2/AmazonEC2DiscoveryClientYamlTestSuiteIT.java +++ b/plugins/discovery-ec2/qa/amazon-ec2/src/yamlRestTest/java/org/elasticsearch/discovery/ec2/AmazonEC2DiscoveryClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.ec2; diff --git a/plugins/discovery-ec2/qa/amazon-ec2/src/yamlRestTest/java/org/elasticsearch/discovery/ec2/AmazonEC2Fixture.java b/plugins/discovery-ec2/qa/amazon-ec2/src/yamlRestTest/java/org/elasticsearch/discovery/ec2/AmazonEC2Fixture.java index c9a7e68414e3c..2718580b5ff68 100644 --- a/plugins/discovery-ec2/qa/amazon-ec2/src/yamlRestTest/java/org/elasticsearch/discovery/ec2/AmazonEC2Fixture.java +++ b/plugins/discovery-ec2/qa/amazon-ec2/src/yamlRestTest/java/org/elasticsearch/discovery/ec2/AmazonEC2Fixture.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.ec2; diff --git a/plugins/discovery-ec2/src/internalClusterTest/java/org/elasticsearch/discovery/ec2/AbstractAwsTestCase.java b/plugins/discovery-ec2/src/internalClusterTest/java/org/elasticsearch/discovery/ec2/AbstractAwsTestCase.java deleted file mode 100644 index 0f946732f4350..0000000000000 --- a/plugins/discovery-ec2/src/internalClusterTest/java/org/elasticsearch/discovery/ec2/AbstractAwsTestCase.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.discovery.ec2; - -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.SettingsException; -import org.elasticsearch.core.PathUtils; -import org.elasticsearch.env.Environment; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.ESIntegTestCase.ThirdParty; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; - -/** - * Base class for AWS tests that require credentials. - *

    - * You must specify {@code -Dtests.thirdparty=true -Dtests.config=/path/to/config} - * in order to run these tests. - */ -@ThirdParty -public abstract class AbstractAwsTestCase extends ESIntegTestCase { - - @Override - protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { - Settings.Builder settings = Settings.builder() - .put(super.nodeSettings(nodeOrdinal, otherSettings)) - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()); - - // if explicit, just load it and don't load from env - try { - if (Strings.hasText(System.getProperty("tests.config"))) { - try { - settings.loadFromPath(PathUtils.get(System.getProperty("tests.config"))); - } catch (IOException e) { - throw new IllegalArgumentException("could not load aws tests config", e); - } - } else { - throw new IllegalStateException( - "to run integration tests, you need to set -Dtests.thirdparty=true and -Dtests.config=/path/to/elasticsearch.yml" - ); - } - } catch (SettingsException exception) { - throw new IllegalStateException("your test configuration file is incorrect: " + System.getProperty("tests.config"), exception); - } - return settings.build(); - } - - @Override - protected Collection> nodePlugins() { - return Arrays.asList(Ec2DiscoveryPlugin.class); - } -} diff --git a/plugins/discovery-ec2/src/internalClusterTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2NetworkAddressesTestCase.java b/plugins/discovery-ec2/src/internalClusterTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2NetworkAddressesTestCase.java new file mode 100644 index 0000000000000..b6a4845977b09 --- /dev/null +++ b/plugins/discovery-ec2/src/internalClusterTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2NetworkAddressesTestCase.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.discovery.ec2; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.http.HttpServerTransport; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; + +import java.io.IOException; +import java.util.Collection; + +@ESIntegTestCase.ClusterScope(numDataNodes = 0) +public abstract class DiscoveryEc2NetworkAddressesTestCase extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return CollectionUtils.appendToCopyNoNullElements(super.nodePlugins(), Ec2DiscoveryPlugin.class); + } + + @Override + protected boolean addMockHttpTransport() { + return false; + } + + void verifyPublishAddress(String publishAddressSetting, String expectedAddress) throws IOException { + final var node = internalCluster().startNode(Settings.builder().put("http.publish_host", publishAddressSetting)); + assertEquals( + expectedAddress, + internalCluster().getInstance(HttpServerTransport.class, node).boundAddress().publishAddress().getAddress() + ); + internalCluster().stopNode(node); + } +} diff --git a/plugins/discovery-ec2/src/internalClusterTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2RegularNetworkAddressesIT.java b/plugins/discovery-ec2/src/internalClusterTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2RegularNetworkAddressesIT.java new file mode 100644 index 0000000000000..491fa37a4c87d --- /dev/null +++ b/plugins/discovery-ec2/src/internalClusterTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2RegularNetworkAddressesIT.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.discovery.ec2; + +import fixture.aws.imds.Ec2ImdsHttpFixture; +import fixture.aws.imds.Ec2ImdsServiceBuilder; +import fixture.aws.imds.Ec2ImdsVersion; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.transport.BindTransportException; + +import java.io.IOException; +import java.util.concurrent.ExecutionException; + +import static org.hamcrest.Matchers.containsString; + +public class DiscoveryEc2RegularNetworkAddressesIT extends DiscoveryEc2NetworkAddressesTestCase { + public void testLocalIgnoresImds() { + Ec2ImdsHttpFixture.runWithFixture(new Ec2ImdsServiceBuilder(randomFrom(Ec2ImdsVersion.values())), imdsFixture -> { + try (var ignored = Ec2ImdsHttpFixture.withEc2MetadataServiceEndpointOverride(imdsFixture.getAddress())) { + verifyPublishAddress("_local_", "127.0.0.1"); + } + }); + } + + public void testImdsNotAvailable() throws IOException { + try (var ignored = Ec2ImdsHttpFixture.withEc2MetadataServiceEndpointOverride("http://127.0.0.1")) { + // if IMDS is not running, regular values like `_local_` should still work + verifyPublishAddress("_local_", "127.0.0.1"); + + // but EC2 addresses will cause the node to fail to start + final var assertionError = expectThrows( + AssertionError.class, + () -> internalCluster().startNode(Settings.builder().put("http.publish_host", "_ec2_")) + ); + final var executionException = asInstanceOf(ExecutionException.class, assertionError.getCause()); + final var bindTransportException = asInstanceOf(BindTransportException.class, executionException.getCause()); + assertEquals("Failed to resolve publish address", bindTransportException.getMessage()); + final var ioException = asInstanceOf(IOException.class, bindTransportException.getCause()); + assertThat(ioException.getMessage(), containsString("/latest/meta-data/local-ipv4")); + } + } +} diff --git a/plugins/discovery-ec2/src/internalClusterTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2SpecialNetworkAddressesIT.java b/plugins/discovery-ec2/src/internalClusterTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2SpecialNetworkAddressesIT.java new file mode 100644 index 0000000000000..f541c4cdd979b --- /dev/null +++ b/plugins/discovery-ec2/src/internalClusterTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2SpecialNetworkAddressesIT.java @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.discovery.ec2; + +import fixture.aws.imds.Ec2ImdsHttpFixture; +import fixture.aws.imds.Ec2ImdsServiceBuilder; +import fixture.aws.imds.Ec2ImdsVersion; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import java.util.Map; +import java.util.stream.Stream; + +public class DiscoveryEc2SpecialNetworkAddressesIT extends DiscoveryEc2NetworkAddressesTestCase { + + private final String imdsAddressName; + private final String elasticsearchAddressName; + private final Ec2ImdsVersion imdsVersion; + + public DiscoveryEc2SpecialNetworkAddressesIT( + @Name("imdsAddressName") String imdsAddressName, + @Name("elasticsearchAddressName") String elasticsearchAddressName, + @Name("imdsVersion") Ec2ImdsVersion imdsVersion + ) { + this.imdsAddressName = imdsAddressName; + this.elasticsearchAddressName = elasticsearchAddressName; + this.imdsVersion = imdsVersion; + } + + @ParametersFactory + public static Iterable parameters() { + return Map.of( + "_ec2:privateIpv4_", + "local-ipv4", + "_ec2:privateDns_", + "local-hostname", + "_ec2:publicIpv4_", + "public-ipv4", + "_ec2:publicDns_", + "public-hostname", + "_ec2:publicIp_", + "public-ipv4", + "_ec2:privateIp_", + "local-ipv4", + "_ec2_", + "local-ipv4" + ) + .entrySet() + .stream() + .flatMap( + addresses -> Stream.of(Ec2ImdsVersion.values()) + .map(ec2ImdsVersion -> new Object[] { addresses.getValue(), addresses.getKey(), ec2ImdsVersion }) + ) + .toList(); + } + + public void testSpecialNetworkAddresses() { + final var publishAddress = "10.0." + between(0, 255) + "." + between(0, 255); + Ec2ImdsHttpFixture.runWithFixture( + new Ec2ImdsServiceBuilder(imdsVersion).addInstanceAddress(imdsAddressName, publishAddress), + imdsFixture -> { + try (var ignored = Ec2ImdsHttpFixture.withEc2MetadataServiceEndpointOverride(imdsFixture.getAddress())) { + verifyPublishAddress(elasticsearchAddressName, publishAddress); + } + } + ); + } + +} diff --git a/plugins/discovery-ec2/src/internalClusterTest/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsTests.java b/plugins/discovery-ec2/src/internalClusterTest/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsTests.java deleted file mode 100644 index 0ed530c9ee3de..0000000000000 --- a/plugins/discovery-ec2/src/internalClusterTest/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsTests.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.discovery.ec2; - -import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; -import org.elasticsearch.common.UUIDs; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.discovery.DiscoveryModule; -import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.elasticsearch.test.ESIntegTestCase.Scope; - -import static org.hamcrest.CoreMatchers.is; - -/** - * Just an empty Node Start test to check eveything if fine when - * starting. - * This test requires AWS to run. - */ -@ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0) -public class Ec2DiscoveryUpdateSettingsTests extends AbstractAwsTestCase { - public void testMinimumMasterNodesStart() { - Settings nodeSettings = Settings.builder().put(DiscoveryModule.DISCOVERY_SEED_PROVIDERS_SETTING.getKey(), "ec2").build(); - internalCluster().startNode(nodeSettings); - - // We try to update a setting now - final String expectedValue = UUIDs.randomBase64UUID(random()); - final String settingName = "cluster.routing.allocation.exclude.any_attribute"; - final ClusterUpdateSettingsResponse response = clusterAdmin().prepareUpdateSettings(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT) - .setPersistentSettings(Settings.builder().put(settingName, expectedValue)) - .get(); - - final String value = response.getPersistentSettings().get(settingName); - assertThat(value, is(expectedValue)); - } -} diff --git a/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeImdsV1IT.java b/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeImdsV1IT.java new file mode 100644 index 0000000000000..32291236ea158 --- /dev/null +++ b/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeImdsV1IT.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.discovery.ec2; + +import fixture.aws.imds.Ec2ImdsHttpFixture; +import fixture.aws.imds.Ec2ImdsServiceBuilder; +import fixture.aws.imds.Ec2ImdsVersion; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +public class DiscoveryEc2AvailabilityZoneAttributeImdsV1IT extends DiscoveryEc2AvailabilityZoneAttributeTestCase { + private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture( + new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V1).availabilityZoneSupplier( + DiscoveryEc2AvailabilityZoneAttributeTestCase::getAvailabilityZone + ) + ); + + public static ElasticsearchCluster cluster = buildCluster(ec2ImdsHttpFixture::getAddress); + + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(ec2ImdsHttpFixture).around(cluster); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } +} diff --git a/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeImdsV2IT.java b/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeImdsV2IT.java new file mode 100644 index 0000000000000..8b785d688e7c4 --- /dev/null +++ b/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeImdsV2IT.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.discovery.ec2; + +import fixture.aws.imds.Ec2ImdsHttpFixture; +import fixture.aws.imds.Ec2ImdsServiceBuilder; +import fixture.aws.imds.Ec2ImdsVersion; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +public class DiscoveryEc2AvailabilityZoneAttributeImdsV2IT extends DiscoveryEc2AvailabilityZoneAttributeTestCase { + private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture( + new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V2).availabilityZoneSupplier( + DiscoveryEc2AvailabilityZoneAttributeTestCase::getAvailabilityZone + ) + ); + + public static ElasticsearchCluster cluster = buildCluster(ec2ImdsHttpFixture::getAddress); + + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(ec2ImdsHttpFixture).around(cluster); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } +} diff --git a/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeNoImdsIT.java b/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeNoImdsIT.java new file mode 100644 index 0000000000000..af3b3d67951dd --- /dev/null +++ b/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeNoImdsIT.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.discovery.ec2; + +import org.elasticsearch.client.Request; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.junit.ClassRule; + +import java.io.IOException; + +public class DiscoveryEc2AvailabilityZoneAttributeNoImdsIT extends DiscoveryEc2AvailabilityZoneAttributeTestCase { + @ClassRule + // use an address which definitely isn't running an IMDS, just in case we're running these tests in EC2 + public static ElasticsearchCluster cluster = DiscoveryEc2AvailabilityZoneAttributeTestCase.buildCluster(() -> "http://127.0.0.1:1"); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @Override // the base class asserts that the attribute is set, but we don't want that here + public void testAvailabilityZoneAttribute() throws IOException { + final var nodesInfoResponse = assertOKAndCreateObjectPath(client().performRequest(new Request("GET", "/_nodes/_all/_none"))); + for (final var nodeId : nodesInfoResponse.evaluateMapKeys("nodes")) { + assertNull(nodesInfoResponse.evaluateExact("nodes", nodeId, "attributes", "aws_availability_zone")); + } + } +} diff --git a/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeTestCase.java b/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeTestCase.java new file mode 100644 index 0000000000000..178c5c3ad4cae --- /dev/null +++ b/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeTestCase.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.discovery.ec2; + +import fixture.aws.imds.Ec2ImdsHttpFixture; + +import org.elasticsearch.client.Request; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.hamcrest.Matchers; + +import java.io.IOException; +import java.util.Objects; +import java.util.Set; +import java.util.function.Supplier; + +public abstract class DiscoveryEc2AvailabilityZoneAttributeTestCase extends ESRestTestCase { + + private static final Set createdAvailabilityZones = ConcurrentCollections.newConcurrentSet(); + + protected static String getAvailabilityZone() { + final var zoneName = randomIdentifier(); + createdAvailabilityZones.add(zoneName); + return zoneName; + } + + protected static ElasticsearchCluster buildCluster(Supplier imdsFixtureAddressSupplier) { + return ElasticsearchCluster.local() + .plugin("discovery-ec2") + .setting(AwsEc2Service.AUTO_ATTRIBUTE_SETTING.getKey(), "true") + .systemProperty(Ec2ImdsHttpFixture.ENDPOINT_OVERRIDE_SYSPROP_NAME, imdsFixtureAddressSupplier) + .build(); + } + + public void testAvailabilityZoneAttribute() throws IOException { + final var nodesInfoResponse = assertOKAndCreateObjectPath(client().performRequest(new Request("GET", "/_nodes/_all/_none"))); + for (final var nodeId : nodesInfoResponse.evaluateMapKeys("nodes")) { + assertThat( + createdAvailabilityZones, + Matchers.hasItem( + Objects.requireNonNull(nodesInfoResponse.evaluateExact("nodes", nodeId, "attributes", "aws_availability_zone")) + ) + ); + } + } +} diff --git a/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2PluginLoadedIT.java b/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2PluginLoadedIT.java new file mode 100644 index 0000000000000..f0f22b2136fc7 --- /dev/null +++ b/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2PluginLoadedIT.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.discovery.ec2; + +import org.elasticsearch.client.Request; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.ClassRule; + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; + +import static org.hamcrest.Matchers.hasItem; + +public class DiscoveryEc2PluginLoadedIT extends ESRestTestCase { + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().plugin("discovery-ec2").build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + public void testPluginLoaded() throws IOException { + final var nodesInfoResponse = assertOKAndCreateObjectPath(client().performRequest(new Request("GET", "/_nodes/plugins"))); + for (final var nodeId : nodesInfoResponse.evaluateMapKeys("nodes")) { + final var pluginCount = asInstanceOf(List.class, nodesInfoResponse.evaluateExact("nodes", nodeId, "plugins")).size(); + final var pluginNames = new HashSet(); + for (int i = 0; i < pluginCount; i++) { + pluginNames.add( + Objects.requireNonNull(nodesInfoResponse.evaluateExact("nodes", nodeId, "plugins", Integer.toString(i), "name")) + ); + } + assertThat(pluginNames, hasItem("discovery-ec2")); + } + } + +} diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AmazonEc2Reference.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AmazonEc2Reference.java index b4d40dd009888..0a4d90047dfeb 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AmazonEc2Reference.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AmazonEc2Reference.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.ec2; diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2SeedHostsProvider.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2SeedHostsProvider.java index 28d9bd71287c5..0717e9bec4c6f 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2SeedHostsProvider.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2SeedHostsProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.ec2; diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2Service.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2Service.java index ef6a9f982db98..8922264552d7a 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2Service.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2Service.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.ec2; diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java index 94aa05288a55c..a9ad82dd2a389 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.ec2; diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2Utils.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2Utils.java index b2475216a9ce7..2e4dc5a9181f6 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2Utils.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2Utils.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.ec2; diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2ClientSettings.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2ClientSettings.java index 3a1cd1f1d33e6..9421e53c97260 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2ClientSettings.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2ClientSettings.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.ec2; diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java index 69447e800d4ac..c089b9dfcc0c9 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.ec2; diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2NameResolver.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2NameResolver.java index 5c8a2a8fb92f9..b4fbce7bb693a 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2NameResolver.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2NameResolver.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.ec2; diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/SocketAccess.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/SocketAccess.java index 1153d045c046f..a9c42aed86d91 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/SocketAccess.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/SocketAccess.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.ec2; diff --git a/plugins/discovery-ec2/src/main/plugin-metadata/plugin-security.policy b/plugins/discovery-ec2/src/main/plugin-metadata/plugin-security.policy index 40a08f3c7d48f..4fb7eb7dda2e4 100644 --- a/plugins/discovery-ec2/src/main/plugin-metadata/plugin-security.policy +++ b/plugins/discovery-ec2/src/main/plugin-metadata/plugin-security.policy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ grant { diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AbstractEC2MockAPITestCase.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AbstractEC2MockAPITestCase.java index 138eb9fe7d14c..7e5ac435a412a 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AbstractEC2MockAPITestCase.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AbstractEC2MockAPITestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.ec2; diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java index aa4a5bd6e54ea..cf12ebb302248 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.ec2; diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/EC2RetriesTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/EC2RetriesTests.java index 9fcf7e51ba30a..78ea619d81f84 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/EC2RetriesTests.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/EC2RetriesTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.ec2; @@ -100,6 +101,7 @@ public void testEC2DiscoveryRetriesOnRateLimiting() throws IOException { exchange.getResponseHeaders().set("Content-Type", "text/xml; charset=UTF-8"); exchange.sendResponseHeaders(HttpStatus.SC_OK, responseBody.length); exchange.getResponseBody().write(responseBody); + exchange.getResponseBody().flush(); return; } } diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java index 41b848954b551..a5bcb10cdd468 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.ec2; diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java index a9ade11d6a7e3..135ddcee8da44 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.ec2; @@ -58,7 +59,7 @@ public class Ec2DiscoveryTests extends AbstractEC2MockAPITestCase { private static final String PREFIX_PUBLIC_IP = "8.8.8."; private static final String PREFIX_PRIVATE_IP = "10.0.0."; - private Map poorMansDNS = new ConcurrentHashMap<>(); + private final Map poorMansDNS = new ConcurrentHashMap<>(); protected MockTransportService createTransportService() { final Transport transport = new Netty4Transport( @@ -132,7 +133,7 @@ protected List buildDynamicHosts(Settings nodeSettings, int no .stream() .filter(t -> t.getKey().equals(entry.getKey())) .map(Tag::getValue) - .collect(Collectors.toList()) + .toList() .containsAll(entry.getValue()) ) ) @@ -143,6 +144,7 @@ protected List buildDynamicHosts(Settings nodeSettings, int no exchange.getResponseHeaders().set("Content-Type", "text/xml; charset=UTF-8"); exchange.sendResponseHeaders(HttpStatus.SC_OK, responseBody.length); exchange.getResponseBody().write(responseBody); + exchange.getResponseBody().flush(); return; } } @@ -159,14 +161,14 @@ protected List buildDynamicHosts(Settings nodeSettings, int no } } - public void testDefaultSettings() throws InterruptedException { + public void testDefaultSettings() { int nodes = randomInt(10); Settings nodeSettings = Settings.builder().build(); List discoveryNodes = buildDynamicHosts(nodeSettings, nodes); assertThat(discoveryNodes, hasSize(nodes)); } - public void testPrivateIp() throws InterruptedException { + public void testPrivateIp() { int nodes = randomInt(10); for (int i = 0; i < nodes; i++) { poorMansDNS.put(PREFIX_PRIVATE_IP + (i + 1), buildNewFakeTransportAddress()); @@ -182,7 +184,7 @@ public void testPrivateIp() throws InterruptedException { } } - public void testPublicIp() throws InterruptedException { + public void testPublicIp() { int nodes = randomInt(10); for (int i = 0; i < nodes; i++) { poorMansDNS.put(PREFIX_PUBLIC_IP + (i + 1), buildNewFakeTransportAddress()); @@ -198,7 +200,7 @@ public void testPublicIp() throws InterruptedException { } } - public void testPrivateDns() throws InterruptedException { + public void testPrivateDns() { int nodes = randomInt(10); for (int i = 0; i < nodes; i++) { String instanceId = "node" + (i + 1); @@ -216,7 +218,7 @@ public void testPrivateDns() throws InterruptedException { } } - public void testPublicDns() throws InterruptedException { + public void testPublicDns() { int nodes = randomInt(10); for (int i = 0; i < nodes; i++) { String instanceId = "node" + (i + 1); @@ -234,14 +236,14 @@ public void testPublicDns() throws InterruptedException { } } - public void testInvalidHostType() throws InterruptedException { + public void testInvalidHostType() { Settings nodeSettings = Settings.builder().put(AwsEc2Service.HOST_TYPE_SETTING.getKey(), "does_not_exist").build(); IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> { buildDynamicHosts(nodeSettings, 1); }); assertThat(exception.getMessage(), containsString("does_not_exist is unknown for discovery.ec2.host_type")); } - public void testFilterByTags() throws InterruptedException { + public void testFilterByTags() { int nodes = randomIntBetween(5, 10); Settings nodeSettings = Settings.builder().put(AwsEc2Service.TAG_SETTING.getKey() + "stage", "prod").build(); @@ -264,7 +266,7 @@ public void testFilterByTags() throws InterruptedException { assertThat(dynamicHosts, hasSize(prodInstances)); } - public void testFilterByMultipleTags() throws InterruptedException { + public void testFilterByMultipleTags() { int nodes = randomIntBetween(5, 10); Settings nodeSettings = Settings.builder().putList(AwsEc2Service.TAG_SETTING.getKey() + "stage", "prod", "preprod").build(); diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2NetworkTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2NetworkTests.java deleted file mode 100644 index eed6ae76a4573..0000000000000 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2NetworkTests.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.discovery.ec2; - -import com.sun.net.httpserver.HttpServer; - -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.network.NetworkService; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.SuppressForbidden; -import org.elasticsearch.mocksocket.MockHttpServer; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.test.ESTestCase; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; - -import java.io.IOException; -import java.io.OutputStream; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.security.AccessController; -import java.security.PrivilegedAction; -import java.util.Arrays; -import java.util.Collections; -import java.util.function.BiConsumer; - -import static com.amazonaws.SDKGlobalConfiguration.EC2_METADATA_SERVICE_OVERRIDE_SYSTEM_PROPERTY; -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.hamcrest.Matchers.arrayContaining; -import static org.hamcrest.Matchers.equalTo; - -/** - * Test for EC2 network.host settings. - *

    - * Warning: This test doesn't assert that the exceptions are thrown. - * They aren't. - */ -@SuppressForbidden(reason = "use http server") -public class Ec2NetworkTests extends ESTestCase { - - private static HttpServer httpServer; - - @BeforeClass - public static void startHttp() throws Exception { - httpServer = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress().getHostAddress(), 0), 0); - - BiConsumer registerContext = (path, v) -> { - final byte[] message = v.getBytes(UTF_8); - httpServer.createContext(path, (s) -> { - s.sendResponseHeaders(RestStatus.OK.getStatus(), message.length); - OutputStream responseBody = s.getResponseBody(); - responseBody.write(message); - responseBody.close(); - }); - }; - registerContext.accept("/latest/meta-data/local-ipv4", "127.0.0.1"); - registerContext.accept("/latest/meta-data/public-ipv4", "165.168.10.2"); - registerContext.accept("/latest/meta-data/public-hostname", "165.168.10.3"); - registerContext.accept("/latest/meta-data/local-hostname", "10.10.10.5"); - - httpServer.start(); - } - - @Before - public void setup() { - // redirect EC2 metadata service to httpServer - AccessController.doPrivileged( - (PrivilegedAction) () -> System.setProperty( - EC2_METADATA_SERVICE_OVERRIDE_SYSTEM_PROPERTY, - "http://" + httpServer.getAddress().getHostName() + ":" + httpServer.getAddress().getPort() - ) - ); - } - - @AfterClass - public static void stopHttp() { - httpServer.stop(0); - httpServer = null; - } - - /** - * Test for network.host: _ec2_ - */ - public void testNetworkHostEc2() throws IOException { - resolveEc2("_ec2_", InetAddress.getByName("127.0.0.1")); - } - - /** - * Test for network.host: _ec2_ - */ - public void testNetworkHostUnableToResolveEc2() { - // redirect EC2 metadata service to unknown location - AccessController.doPrivileged( - (PrivilegedAction) () -> System.setProperty(EC2_METADATA_SERVICE_OVERRIDE_SYSTEM_PROPERTY, "http://127.0.0.1/") - ); - - try { - resolveEc2("_ec2_", (InetAddress[]) null); - } catch (IOException e) { - assertThat( - e.getMessage(), - equalTo("IOException caught when fetching InetAddress from [http://127.0.0.1//latest/meta-data/local-ipv4]") - ); - } - } - - /** - * Test for network.host: _ec2:publicIp_ - */ - public void testNetworkHostEc2PublicIp() throws IOException { - resolveEc2("_ec2:publicIp_", InetAddress.getByName("165.168.10.2")); - } - - /** - * Test for network.host: _ec2:privateIp_ - */ - public void testNetworkHostEc2PrivateIp() throws IOException { - resolveEc2("_ec2:privateIp_", InetAddress.getByName("127.0.0.1")); - } - - /** - * Test for network.host: _ec2:privateIpv4_ - */ - public void testNetworkHostEc2PrivateIpv4() throws IOException { - resolveEc2("_ec2:privateIpv4_", InetAddress.getByName("127.0.0.1")); - } - - /** - * Test for network.host: _ec2:privateDns_ - */ - public void testNetworkHostEc2PrivateDns() throws IOException { - resolveEc2("_ec2:privateDns_", InetAddress.getByName("10.10.10.5")); - } - - /** - * Test for network.host: _ec2:publicIpv4_ - */ - public void testNetworkHostEc2PublicIpv4() throws IOException { - resolveEc2("_ec2:publicIpv4_", InetAddress.getByName("165.168.10.2")); - } - - /** - * Test for network.host: _ec2:publicDns_ - */ - public void testNetworkHostEc2PublicDns() throws IOException { - resolveEc2("_ec2:publicDns_", InetAddress.getByName("165.168.10.3")); - } - - private InetAddress[] resolveEc2(String host, InetAddress... expected) throws IOException { - Settings nodeSettings = Settings.builder().put("network.host", host).build(); - - NetworkService networkService = new NetworkService(Collections.singletonList(new Ec2NameResolver())); - - InetAddress[] addresses = networkService.resolveBindHostAddresses( - NetworkService.GLOBAL_NETWORK_BIND_HOST_SETTING.get(nodeSettings).toArray(Strings.EMPTY_ARRAY) - ); - if (expected == null) { - fail("We should get an IOException, resolved addressed:" + Arrays.toString(addresses)); - } - assertThat(addresses, arrayContaining(expected)); - return addresses; - } - - /** - * Test that we don't have any regression with network host core settings such as - * network.host: _local_ - */ - public void testNetworkHostCoreLocal() throws IOException { - NetworkService networkService = new NetworkService(Collections.singletonList(new Ec2NameResolver())); - InetAddress[] addresses = networkService.resolveBindHostAddresses(null); - assertThat(addresses, arrayContaining(networkService.resolveBindHostAddresses(new String[] { "_local_" }))); - } -} diff --git a/plugins/discovery-ec2/src/yamlRestTest/java/org/elasticsearch/discovery/ec2/CloudAwsClientYamlTestSuiteIT.java b/plugins/discovery-ec2/src/yamlRestTest/java/org/elasticsearch/discovery/ec2/CloudAwsClientYamlTestSuiteIT.java deleted file mode 100644 index 2110baae5ec9d..0000000000000 --- a/plugins/discovery-ec2/src/yamlRestTest/java/org/elasticsearch/discovery/ec2/CloudAwsClientYamlTestSuiteIT.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.discovery.ec2; - -import com.carrotsearch.randomizedtesting.annotations.Name; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - -import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; -import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; - -public class CloudAwsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { - - public CloudAwsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { - super(testCandidate); - } - - @ParametersFactory - public static Iterable parameters() throws Exception { - return ESClientYamlSuiteTestCase.createParameters(); - } -} diff --git a/plugins/discovery-ec2/src/yamlRestTest/resources/rest-api-spec/test/discovery_ec2/10_basic.yml b/plugins/discovery-ec2/src/yamlRestTest/resources/rest-api-spec/test/discovery_ec2/10_basic.yml deleted file mode 100644 index ba51c623fe888..0000000000000 --- a/plugins/discovery-ec2/src/yamlRestTest/resources/rest-api-spec/test/discovery_ec2/10_basic.yml +++ /dev/null @@ -1,16 +0,0 @@ -# Integration tests for Discovery EC2 component -# -"Discovery EC2 loaded": - - skip: - reason: "contains is a newly added assertion" - features: contains - - do: - cluster.state: {} - - # Get master node id - - set: { master_node: master } - - - do: - nodes.info: {} - - - contains: { nodes.$master.plugins: { name: discovery-ec2 } } diff --git a/plugins/discovery-gce/build.gradle b/plugins/discovery-gce/build.gradle index 530070f9e0073..c6beaf3f332ca 100644 --- a/plugins/discovery-gce/build.gradle +++ b/plugins/discovery-gce/build.gradle @@ -2,8 +2,8 @@ apply plugin: 'elasticsearch.legacy-yaml-rest-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { - description 'The Google Compute Engine (GCE) Discovery plugin allows to use GCE API for the unicast discovery mechanism.' - classname 'org.elasticsearch.plugin.discovery.gce.GceDiscoveryPlugin' + description = 'The Google Compute Engine (GCE) Discovery plugin allows to use GCE API for the unicast discovery mechanism.' + classname ='org.elasticsearch.plugin.discovery.gce.GceDiscoveryPlugin' } versions << [ diff --git a/plugins/discovery-gce/qa/gce/build.gradle b/plugins/discovery-gce/qa/gce/build.gradle index 7eb6e85903ba5..b55f17202d0cb 100644 --- a/plugins/discovery-gce/qa/gce/build.gradle +++ b/plugins/discovery-gce/qa/gce/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ @@ -29,9 +30,10 @@ restResources { /** A task to start the GCEFixture which emulates a GCE service **/ def gceFixtureProvider = tasks.register("gceFixture", AntFixture) { - dependsOn project.sourceSets.yamlRestTest.runtimeClasspath - env 'CLASSPATH', "${-> project.sourceSets.yamlRestTest.runtimeClasspath.asPath}" - executable = "${BuildParams.runtimeJavaHome}/bin/java" + def runtimeClasspath = project.sourceSets.yamlRestTest.runtimeClasspath + dependsOn runtimeClasspath + env 'CLASSPATH', "${-> runtimeClasspath.asPath}" + executable = "${buildParams.runtimeJavaHome.get()}/bin/java" args 'org.elasticsearch.cloud.gce.GCEFixture', baseDir, "${buildDir}/testclusters/yamlRestTest-1/config/unicast_hosts.txt" } diff --git a/plugins/discovery-gce/qa/gce/src/yamlRestTest/java/org/elasticsearch/cloud/gce/GCEDiscoveryClientYamlTestSuiteIT.java b/plugins/discovery-gce/qa/gce/src/yamlRestTest/java/org/elasticsearch/cloud/gce/GCEDiscoveryClientYamlTestSuiteIT.java index 18fd0ff5acc2b..c645d0114331a 100644 --- a/plugins/discovery-gce/qa/gce/src/yamlRestTest/java/org/elasticsearch/cloud/gce/GCEDiscoveryClientYamlTestSuiteIT.java +++ b/plugins/discovery-gce/qa/gce/src/yamlRestTest/java/org/elasticsearch/cloud/gce/GCEDiscoveryClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cloud.gce; diff --git a/plugins/discovery-gce/qa/gce/src/yamlRestTest/java/org/elasticsearch/cloud/gce/GCEFixture.java b/plugins/discovery-gce/qa/gce/src/yamlRestTest/java/org/elasticsearch/cloud/gce/GCEFixture.java index 9829f39390970..7a5764e1fbd87 100644 --- a/plugins/discovery-gce/qa/gce/src/yamlRestTest/java/org/elasticsearch/cloud/gce/GCEFixture.java +++ b/plugins/discovery-gce/qa/gce/src/yamlRestTest/java/org/elasticsearch/cloud/gce/GCEFixture.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cloud.gce; diff --git a/plugins/discovery-gce/src/internalClusterTest/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java b/plugins/discovery-gce/src/internalClusterTest/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java index ca8a4449c4d6d..9e3a708dc3ba3 100644 --- a/plugins/discovery-gce/src/internalClusterTest/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java +++ b/plugins/discovery-gce/src/internalClusterTest/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.gce; diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesService.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesService.java index 05f3df75cf841..1bf5896426540 100644 --- a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesService.java +++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cloud.gce; diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesServiceImpl.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesServiceImpl.java index d4369429babf3..530c8230e3f00 100644 --- a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesServiceImpl.java +++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesServiceImpl.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cloud.gce; diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceMetadataService.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceMetadataService.java index 34c3bf1b4329e..85385207241b5 100644 --- a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceMetadataService.java +++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceMetadataService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cloud.gce; diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/network/GceNameResolver.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/network/GceNameResolver.java index 6c3244f947c87..debf8ecb36b28 100644 --- a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/network/GceNameResolver.java +++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/network/GceNameResolver.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cloud.gce.network; diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/util/Access.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/util/Access.java index 0f328baedadff..8d2134b907e14 100644 --- a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/util/Access.java +++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/util/Access.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cloud.gce.util; diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceSeedHostsProvider.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceSeedHostsProvider.java index 38f5d1ca07365..57723bb541c70 100644 --- a/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceSeedHostsProvider.java +++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceSeedHostsProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.gce; diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapper.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapper.java index ae8b85b201a56..a41eec89fe7f8 100644 --- a/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapper.java +++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.gce; diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java index c277e9d2c30e1..735d74b767bb9 100644 --- a/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java +++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.discovery.gce; diff --git a/plugins/discovery-gce/src/main/plugin-metadata/entitlement-policy.yaml b/plugins/discovery-gce/src/main/plugin-metadata/entitlement-policy.yaml new file mode 100644 index 0000000000000..a1ff54f02d969 --- /dev/null +++ b/plugins/discovery-gce/src/main/plugin-metadata/entitlement-policy.yaml @@ -0,0 +1,2 @@ +ALL-UNNAMED: + - set_https_connection_properties # required by google-http-client diff --git a/plugins/discovery-gce/src/main/plugin-metadata/plugin-security.policy b/plugins/discovery-gce/src/main/plugin-metadata/plugin-security.policy index f7a198dad8dd3..467d6d4502869 100644 --- a/plugins/discovery-gce/src/main/plugin-metadata/plugin-security.policy +++ b/plugins/discovery-gce/src/main/plugin-metadata/plugin-security.policy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ grant { diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/cloud/gce/GceInstancesServiceImplTests.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/cloud/gce/GceInstancesServiceImplTests.java index cae2150cc4d29..662d840d9c5be 100644 --- a/plugins/discovery-gce/src/test/java/org/elasticsearch/cloud/gce/GceInstancesServiceImplTests.java +++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/cloud/gce/GceInstancesServiceImplTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cloud.gce; diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java index 0ce0952b6a7f0..ffa2fc40546e6 100644 --- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java +++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.gce; diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceInstancesServiceMock.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceInstancesServiceMock.java index ac74034a8ebfd..473a625a29e98 100644 --- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceInstancesServiceMock.java +++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceInstancesServiceMock.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.gce; diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceMetadataServiceMock.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceMetadataServiceMock.java index bc5b355b2a87b..1fa39b9e27f92 100644 --- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceMetadataServiceMock.java +++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceMetadataServiceMock.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.gce; diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceMockUtils.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceMockUtils.java index 7dd626494a9c2..cb9aa96dd609a 100644 --- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceMockUtils.java +++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceMockUtils.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.gce; diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceNetworkTests.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceNetworkTests.java index bdb6abd8d1bc7..3cfaa2237365f 100644 --- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceNetworkTests.java +++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceNetworkTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.gce; diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapperTests.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapperTests.java index 2c29523ff4c91..eb06096d44c6b 100644 --- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapperTests.java +++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapperTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.gce; diff --git a/plugins/discovery-gce/src/yamlRestTest/java/org/elasticsearch/discovery/gce/DiscoveryGceClientYamlTestSuiteIT.java b/plugins/discovery-gce/src/yamlRestTest/java/org/elasticsearch/discovery/gce/DiscoveryGceClientYamlTestSuiteIT.java index af3fd118b5061..718e8f511bdf7 100644 --- a/plugins/discovery-gce/src/yamlRestTest/java/org/elasticsearch/discovery/gce/DiscoveryGceClientYamlTestSuiteIT.java +++ b/plugins/discovery-gce/src/yamlRestTest/java/org/elasticsearch/discovery/gce/DiscoveryGceClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.gce; diff --git a/plugins/examples/custom-processor/build.gradle b/plugins/examples/custom-processor/build.gradle index 69da64d8ebe86..6c0281d899a4e 100644 --- a/plugins/examples/custom-processor/build.gradle +++ b/plugins/examples/custom-processor/build.gradle @@ -1,19 +1,20 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.esplugin' apply plugin: 'elasticsearch.yaml-rest-test' esplugin { - name 'custom-processor' - description 'An example plugin showing how to register a custom ingest processor' - classname 'org.elasticsearch.example.customprocessor.ExampleProcessorPlugin' - licenseFile rootProject.file('SSPL-1.0+ELASTIC-LICENSE-2.0.txt') - noticeFile rootProject.file('NOTICE.txt') + name = 'custom-processor' + description = 'An example plugin showing how to register a custom ingest processor' + classname ='org.elasticsearch.example.customprocessor.ExampleProcessorPlugin' + licenseFile = rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt') + noticeFile = rootProject.file('NOTICE.txt') } dependencies { diff --git a/plugins/examples/custom-processor/src/main/java/org/elasticsearch/example/customprocessor/ExampleProcessorPlugin.java b/plugins/examples/custom-processor/src/main/java/org/elasticsearch/example/customprocessor/ExampleProcessorPlugin.java index 1ba145a92ca7d..3ca812a31bbc6 100644 --- a/plugins/examples/custom-processor/src/main/java/org/elasticsearch/example/customprocessor/ExampleProcessorPlugin.java +++ b/plugins/examples/custom-processor/src/main/java/org/elasticsearch/example/customprocessor/ExampleProcessorPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.customprocessor; diff --git a/plugins/examples/custom-processor/src/yamlRestTest/java/org/elasticsearch/example/customprocessor/ExampleProcessorClientYamlTestSuiteIT.java b/plugins/examples/custom-processor/src/yamlRestTest/java/org/elasticsearch/example/customprocessor/ExampleProcessorClientYamlTestSuiteIT.java index ac08df358fe5e..803734953f2b6 100644 --- a/plugins/examples/custom-processor/src/yamlRestTest/java/org/elasticsearch/example/customprocessor/ExampleProcessorClientYamlTestSuiteIT.java +++ b/plugins/examples/custom-processor/src/yamlRestTest/java/org/elasticsearch/example/customprocessor/ExampleProcessorClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.customprocessor; diff --git a/plugins/examples/custom-settings/build.gradle b/plugins/examples/custom-settings/build.gradle index 56a2dd5cee724..2774bf6e75c78 100644 --- a/plugins/examples/custom-settings/build.gradle +++ b/plugins/examples/custom-settings/build.gradle @@ -1,19 +1,20 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.esplugin' apply plugin: 'elasticsearch.yaml-rest-test' esplugin { - name 'custom-settings' - description 'An example plugin showing how to register custom settings' - classname 'org.elasticsearch.example.customsettings.ExampleCustomSettingsPlugin' - licenseFile rootProject.file('SSPL-1.0+ELASTIC-LICENSE-2.0.txt') - noticeFile rootProject.file('NOTICE.txt') + name = 'custom-settings' + description = 'An example plugin showing how to register custom settings' + classname ='org.elasticsearch.example.customsettings.ExampleCustomSettingsPlugin' + licenseFile = rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt') + noticeFile = rootProject.file('NOTICE.txt') } testClusters.configureEach { @@ -23,4 +24,4 @@ testClusters.configureEach { dependencies { yamlRestTestRuntimeOnly "org.apache.logging.log4j:log4j-core:${log4jVersion}" -} \ No newline at end of file +} diff --git a/plugins/examples/custom-settings/src/main/java/org/elasticsearch/example/customsettings/ExampleCustomSettingsConfig.java b/plugins/examples/custom-settings/src/main/java/org/elasticsearch/example/customsettings/ExampleCustomSettingsConfig.java index 8211a6170a183..243201f632c8f 100644 --- a/plugins/examples/custom-settings/src/main/java/org/elasticsearch/example/customsettings/ExampleCustomSettingsConfig.java +++ b/plugins/examples/custom-settings/src/main/java/org/elasticsearch/example/customsettings/ExampleCustomSettingsConfig.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.customsettings; diff --git a/plugins/examples/custom-settings/src/main/java/org/elasticsearch/example/customsettings/ExampleCustomSettingsPlugin.java b/plugins/examples/custom-settings/src/main/java/org/elasticsearch/example/customsettings/ExampleCustomSettingsPlugin.java index 12398857ad3cc..d8f16dcf14194 100644 --- a/plugins/examples/custom-settings/src/main/java/org/elasticsearch/example/customsettings/ExampleCustomSettingsPlugin.java +++ b/plugins/examples/custom-settings/src/main/java/org/elasticsearch/example/customsettings/ExampleCustomSettingsPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.customsettings; diff --git a/plugins/examples/custom-settings/src/test/java/org/elasticsearch/example/customsettings/ExampleCustomSettingsConfigTests.java b/plugins/examples/custom-settings/src/test/java/org/elasticsearch/example/customsettings/ExampleCustomSettingsConfigTests.java index 23a134d58b8ad..1afdde2f4da1f 100644 --- a/plugins/examples/custom-settings/src/test/java/org/elasticsearch/example/customsettings/ExampleCustomSettingsConfigTests.java +++ b/plugins/examples/custom-settings/src/test/java/org/elasticsearch/example/customsettings/ExampleCustomSettingsConfigTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.customsettings; diff --git a/plugins/examples/custom-settings/src/yamlRestTest/java/org/elasticsearch/example/customsettings/ExampleCustomSettingsClientYamlTestSuiteIT.java b/plugins/examples/custom-settings/src/yamlRestTest/java/org/elasticsearch/example/customsettings/ExampleCustomSettingsClientYamlTestSuiteIT.java index 9377cc7afd47a..f0ce5dbb0f97d 100644 --- a/plugins/examples/custom-settings/src/yamlRestTest/java/org/elasticsearch/example/customsettings/ExampleCustomSettingsClientYamlTestSuiteIT.java +++ b/plugins/examples/custom-settings/src/yamlRestTest/java/org/elasticsearch/example/customsettings/ExampleCustomSettingsClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.customsettings; diff --git a/plugins/examples/custom-significance-heuristic/build.gradle b/plugins/examples/custom-significance-heuristic/build.gradle index 536fe0dac1cf9..f2f0cefa6d6f5 100644 --- a/plugins/examples/custom-significance-heuristic/build.gradle +++ b/plugins/examples/custom-significance-heuristic/build.gradle @@ -1,21 +1,22 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.esplugin' apply plugin: 'elasticsearch.yaml-rest-test' esplugin { - name 'custom-significance-heuristic' - description 'An example plugin showing how to write and register a custom significance heuristic' - classname 'org.elasticsearch.example.customsigheuristic.CustomSignificanceHeuristicPlugin' - licenseFile rootProject.file('SSPL-1.0+ELASTIC-LICENSE-2.0.txt') - noticeFile rootProject.file('NOTICE.txt') + name = 'custom-significance-heuristic' + description = 'An example plugin showing how to write and register a custom significance heuristic' + classname ='org.elasticsearch.example.customsigheuristic.CustomSignificanceHeuristicPlugin' + licenseFile = rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt') + noticeFile = rootProject.file('NOTICE.txt') } dependencies { yamlRestTestRuntimeOnly "org.apache.logging.log4j:log4j-core:${log4jVersion}" -} \ No newline at end of file +} diff --git a/plugins/examples/custom-significance-heuristic/src/main/java/org/elasticsearch/example/customsigheuristic/CustomSignificanceHeuristicPlugin.java b/plugins/examples/custom-significance-heuristic/src/main/java/org/elasticsearch/example/customsigheuristic/CustomSignificanceHeuristicPlugin.java index df37380bbee1d..74dd22d5e1fac 100644 --- a/plugins/examples/custom-significance-heuristic/src/main/java/org/elasticsearch/example/customsigheuristic/CustomSignificanceHeuristicPlugin.java +++ b/plugins/examples/custom-significance-heuristic/src/main/java/org/elasticsearch/example/customsigheuristic/CustomSignificanceHeuristicPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.customsigheuristic; diff --git a/plugins/examples/custom-significance-heuristic/src/main/java/org/elasticsearch/example/customsigheuristic/SimpleHeuristic.java b/plugins/examples/custom-significance-heuristic/src/main/java/org/elasticsearch/example/customsigheuristic/SimpleHeuristic.java index 61c87edd66ab3..2ca387181fe07 100644 --- a/plugins/examples/custom-significance-heuristic/src/main/java/org/elasticsearch/example/customsigheuristic/SimpleHeuristic.java +++ b/plugins/examples/custom-significance-heuristic/src/main/java/org/elasticsearch/example/customsigheuristic/SimpleHeuristic.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.customsigheuristic; diff --git a/plugins/examples/custom-significance-heuristic/src/test/java/org/elasticsearch/example/customsigheuristic/SimpleHeuristicWireTests.java b/plugins/examples/custom-significance-heuristic/src/test/java/org/elasticsearch/example/customsigheuristic/SimpleHeuristicWireTests.java index d43374b3b5c60..3f44d98d69bde 100644 --- a/plugins/examples/custom-significance-heuristic/src/test/java/org/elasticsearch/example/customsigheuristic/SimpleHeuristicWireTests.java +++ b/plugins/examples/custom-significance-heuristic/src/test/java/org/elasticsearch/example/customsigheuristic/SimpleHeuristicWireTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.customsigheuristic; diff --git a/plugins/examples/custom-significance-heuristic/src/yamlRestTest/java/org/elasticsearch/example/customsigheuristic/CustomSignificanceHeuristicClientYamlTestSuiteIT.java b/plugins/examples/custom-significance-heuristic/src/yamlRestTest/java/org/elasticsearch/example/customsigheuristic/CustomSignificanceHeuristicClientYamlTestSuiteIT.java index 51922d5a06bed..08b2d9e296f9e 100644 --- a/plugins/examples/custom-significance-heuristic/src/yamlRestTest/java/org/elasticsearch/example/customsigheuristic/CustomSignificanceHeuristicClientYamlTestSuiteIT.java +++ b/plugins/examples/custom-significance-heuristic/src/yamlRestTest/java/org/elasticsearch/example/customsigheuristic/CustomSignificanceHeuristicClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.customsigheuristic; diff --git a/plugins/examples/custom-suggester/build.gradle b/plugins/examples/custom-suggester/build.gradle index c8132ce1eaf0c..a1cf345f5e819 100644 --- a/plugins/examples/custom-suggester/build.gradle +++ b/plugins/examples/custom-suggester/build.gradle @@ -1,19 +1,20 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.esplugin' apply plugin: 'elasticsearch.yaml-rest-test' esplugin { - name 'custom-suggester' - description 'An example plugin showing how to write and register a custom suggester' - classname 'org.elasticsearch.example.customsuggester.CustomSuggesterPlugin' - licenseFile rootProject.file('SSPL-1.0+ELASTIC-LICENSE-2.0.txt') - noticeFile rootProject.file('NOTICE.txt') + name = 'custom-suggester' + description = 'An example plugin showing how to write and register a custom suggester' + classname ='org.elasticsearch.example.customsuggester.CustomSuggesterPlugin' + licenseFile = rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt') + noticeFile = rootProject.file('NOTICE.txt') } testClusters.configureEach { diff --git a/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggester.java b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggester.java index c69f1751d4c64..148324be55654 100644 --- a/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggester.java +++ b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggester.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.customsuggester; diff --git a/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggesterPlugin.java b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggesterPlugin.java index 85ebaa94a33b5..51f876a65457e 100644 --- a/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggesterPlugin.java +++ b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggesterPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.customsuggester; diff --git a/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestion.java b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestion.java index afffd8266c11e..b04d62065edc2 100644 --- a/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestion.java +++ b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestion.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.customsuggester; diff --git a/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestionBuilder.java b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestionBuilder.java index 298ddb7253eec..8aaa9ee5ad5e7 100644 --- a/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestionBuilder.java +++ b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestionBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.customsuggester; diff --git a/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestionContext.java b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestionContext.java index 33f4606cceff2..190129f0a3743 100644 --- a/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestionContext.java +++ b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestionContext.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.customsuggester; diff --git a/plugins/examples/custom-suggester/src/yamlRestTest/java/org/elasticsearch/example/customsuggester/CustomSuggesterClientYamlTestSuiteIT.java b/plugins/examples/custom-suggester/src/yamlRestTest/java/org/elasticsearch/example/customsuggester/CustomSuggesterClientYamlTestSuiteIT.java index a928f6ffc9d0d..bc656d0e7214a 100644 --- a/plugins/examples/custom-suggester/src/yamlRestTest/java/org/elasticsearch/example/customsuggester/CustomSuggesterClientYamlTestSuiteIT.java +++ b/plugins/examples/custom-suggester/src/yamlRestTest/java/org/elasticsearch/example/customsuggester/CustomSuggesterClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.customsuggester; diff --git a/plugins/examples/gradle.properties b/plugins/examples/gradle.properties index 7e11c8077f808..8995eeba2a09e 100644 --- a/plugins/examples/gradle.properties +++ b/plugins/examples/gradle.properties @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # version=1.0.0-SNAPSHOT group=org.elasticsearch.examples diff --git a/plugins/examples/gradle/wrapper/gradle-wrapper.properties b/plugins/examples/gradle/wrapper/gradle-wrapper.properties index 9036682bf0f0c..e712035eabc7b 100644 --- a/plugins/examples/gradle/wrapper/gradle-wrapper.properties +++ b/plugins/examples/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=682b4df7fe5accdca84a4d1ef6a3a6ab096b3efd5edf7de2bd8c758d95a93703 -distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-all.zip +distributionSha256Sum=7ebdac923867a3cec0098302416d1e3c6c0c729fc4e2e05c10637a8af33a76c5 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.12-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/plugins/examples/painless-whitelist/build.gradle b/plugins/examples/painless-whitelist/build.gradle index 5e0bf88819e59..f501bd466ebe5 100644 --- a/plugins/examples/painless-whitelist/build.gradle +++ b/plugins/examples/painless-whitelist/build.gradle @@ -1,20 +1,21 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.esplugin' apply plugin: 'elasticsearch.yaml-rest-test' esplugin { - name 'painless-whitelist' - description 'An example whitelisting additional classes and methods in painless' - classname 'org.elasticsearch.example.painlesswhitelist.MyWhitelistPlugin' + name = 'painless-whitelist' + description = 'An example whitelisting additional classes and methods in painless' + classname ='org.elasticsearch.example.painlesswhitelist.MyWhitelistPlugin' extendedPlugins = ['lang-painless'] - licenseFile rootProject.file('SSPL-1.0+ELASTIC-LICENSE-2.0.txt') - noticeFile rootProject.file('NOTICE.txt') + licenseFile = rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt') + noticeFile = rootProject.file('NOTICE.txt') } dependencies { diff --git a/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExamplePainlessAnnotation.java b/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExamplePainlessAnnotation.java index 1040d8e7d60d0..0f1854e0277ac 100644 --- a/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExamplePainlessAnnotation.java +++ b/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExamplePainlessAnnotation.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.painlesswhitelist; diff --git a/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleStaticMethodClass.java b/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleStaticMethodClass.java index 6f5d8db8348c6..c3b4d44ada48b 100644 --- a/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleStaticMethodClass.java +++ b/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleStaticMethodClass.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.painlesswhitelist; diff --git a/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleWhitelistAnnotationParser.java b/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleWhitelistAnnotationParser.java index a09b9e003c812..ef877be517a17 100644 --- a/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleWhitelistAnnotationParser.java +++ b/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleWhitelistAnnotationParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.painlesswhitelist; diff --git a/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleWhitelistExtension.java b/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleWhitelistExtension.java index fb4612623b2d2..b003704eebdc2 100644 --- a/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleWhitelistExtension.java +++ b/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleWhitelistExtension.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.painlesswhitelist; diff --git a/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleWhitelistedClass.java b/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleWhitelistedClass.java index bcc38de7a0f28..5dd2009ee3731 100644 --- a/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleWhitelistedClass.java +++ b/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleWhitelistedClass.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.painlesswhitelist; diff --git a/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleWhitelistedInstance.java b/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleWhitelistedInstance.java index e841e6cedc425..7b4d62357cc84 100644 --- a/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleWhitelistedInstance.java +++ b/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleWhitelistedInstance.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.painlesswhitelist; diff --git a/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/MyWhitelistPlugin.java b/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/MyWhitelistPlugin.java index a32ef8e56c6ab..c13e7de704e28 100644 --- a/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/MyWhitelistPlugin.java +++ b/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/MyWhitelistPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.painlesswhitelist; diff --git a/plugins/examples/painless-whitelist/src/main/resources/org/elasticsearch/example/painlesswhitelist/example_whitelist.txt b/plugins/examples/painless-whitelist/src/main/resources/org/elasticsearch/example/painlesswhitelist/example_whitelist.txt index 341080be02440..2b0ee781b4947 100644 --- a/plugins/examples/painless-whitelist/src/main/resources/org/elasticsearch/example/painlesswhitelist/example_whitelist.txt +++ b/plugins/examples/painless-whitelist/src/main/resources/org/elasticsearch/example/painlesswhitelist/example_whitelist.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # This file contains a whitelist for an example class which may be access from painless diff --git a/plugins/examples/painless-whitelist/src/yamlRestTest/java/org/elasticsearch/example/painlesswhitelist/PainlessWhitelistClientYamlTestSuiteIT.java b/plugins/examples/painless-whitelist/src/yamlRestTest/java/org/elasticsearch/example/painlesswhitelist/PainlessWhitelistClientYamlTestSuiteIT.java index 269284f1389e2..a21f3eaafcd5d 100644 --- a/plugins/examples/painless-whitelist/src/yamlRestTest/java/org/elasticsearch/example/painlesswhitelist/PainlessWhitelistClientYamlTestSuiteIT.java +++ b/plugins/examples/painless-whitelist/src/yamlRestTest/java/org/elasticsearch/example/painlesswhitelist/PainlessWhitelistClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.painlesswhitelist; diff --git a/plugins/examples/rescore/build.gradle b/plugins/examples/rescore/build.gradle index d27f9f0be1dab..023033349dd8c 100644 --- a/plugins/examples/rescore/build.gradle +++ b/plugins/examples/rescore/build.gradle @@ -1,19 +1,20 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.esplugin' apply plugin: 'elasticsearch.yaml-rest-test' esplugin { - name 'example-rescore' - description 'An example plugin implementing rescore and verifying that plugins *can* implement rescore' - classname 'org.elasticsearch.example.rescore.ExampleRescorePlugin' - licenseFile rootProject.file('SSPL-1.0+ELASTIC-LICENSE-2.0.txt') - noticeFile rootProject.file('NOTICE.txt') + name = 'example-rescore' + description = 'An example plugin implementing rescore and verifying that plugins *can* implement rescore' + classname ='org.elasticsearch.example.rescore.ExampleRescorePlugin' + licenseFile = rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt') + noticeFile = rootProject.file('NOTICE.txt') } dependencies { diff --git a/plugins/examples/rescore/src/main/java/org/elasticsearch/example/rescore/ExampleRescoreBuilder.java b/plugins/examples/rescore/src/main/java/org/elasticsearch/example/rescore/ExampleRescoreBuilder.java index ae920d2d4873b..4beb926bea752 100644 --- a/plugins/examples/rescore/src/main/java/org/elasticsearch/example/rescore/ExampleRescoreBuilder.java +++ b/plugins/examples/rescore/src/main/java/org/elasticsearch/example/rescore/ExampleRescoreBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.rescore; diff --git a/plugins/examples/rescore/src/main/java/org/elasticsearch/example/rescore/ExampleRescorePlugin.java b/plugins/examples/rescore/src/main/java/org/elasticsearch/example/rescore/ExampleRescorePlugin.java index 34ef333119640..55a523727f555 100644 --- a/plugins/examples/rescore/src/main/java/org/elasticsearch/example/rescore/ExampleRescorePlugin.java +++ b/plugins/examples/rescore/src/main/java/org/elasticsearch/example/rescore/ExampleRescorePlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.rescore; diff --git a/plugins/examples/rescore/src/test/java/org/elasticsearch/example/rescore/ExampleRescoreBuilderFieldDataTests.java b/plugins/examples/rescore/src/test/java/org/elasticsearch/example/rescore/ExampleRescoreBuilderFieldDataTests.java index b486a6730a9ad..4d71c70087bcf 100644 --- a/plugins/examples/rescore/src/test/java/org/elasticsearch/example/rescore/ExampleRescoreBuilderFieldDataTests.java +++ b/plugins/examples/rescore/src/test/java/org/elasticsearch/example/rescore/ExampleRescoreBuilderFieldDataTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.rescore; diff --git a/plugins/examples/rescore/src/test/java/org/elasticsearch/example/rescore/ExampleRescoreBuilderTests.java b/plugins/examples/rescore/src/test/java/org/elasticsearch/example/rescore/ExampleRescoreBuilderTests.java index f01466b2c9216..d1f0a67ea36c6 100644 --- a/plugins/examples/rescore/src/test/java/org/elasticsearch/example/rescore/ExampleRescoreBuilderTests.java +++ b/plugins/examples/rescore/src/test/java/org/elasticsearch/example/rescore/ExampleRescoreBuilderTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.rescore; diff --git a/plugins/examples/rescore/src/yamlRestTest/java/org/elasticsearch/example/rescore/ExampleRescoreClientYamlTestSuiteIT.java b/plugins/examples/rescore/src/yamlRestTest/java/org/elasticsearch/example/rescore/ExampleRescoreClientYamlTestSuiteIT.java index 1380f5e26cfc3..1dd046a52fd83 100644 --- a/plugins/examples/rescore/src/yamlRestTest/java/org/elasticsearch/example/rescore/ExampleRescoreClientYamlTestSuiteIT.java +++ b/plugins/examples/rescore/src/yamlRestTest/java/org/elasticsearch/example/rescore/ExampleRescoreClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.rescore; diff --git a/plugins/examples/rest-handler/build.gradle b/plugins/examples/rest-handler/build.gradle index 2e0a26cb767cf..43590b166a545 100644 --- a/plugins/examples/rest-handler/build.gradle +++ b/plugins/examples/rest-handler/build.gradle @@ -1,19 +1,20 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.esplugin' apply plugin: 'elasticsearch.yaml-rest-test' esplugin { - name 'rest-handler' - description 'An example plugin showing how to register a REST handler' - classname 'org.elasticsearch.example.resthandler.ExampleRestHandlerPlugin' - licenseFile rootProject.file('SSPL-1.0+ELASTIC-LICENSE-2.0.txt') - noticeFile rootProject.file('NOTICE.txt') + name = 'rest-handler' + description = 'An example plugin showing how to register a REST handler' + classname ='org.elasticsearch.example.resthandler.ExampleRestHandlerPlugin' + licenseFile = rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt') + noticeFile = rootProject.file('NOTICE.txt') } dependencies { diff --git a/plugins/examples/rest-handler/src/main/java/org/elasticsearch/example/resthandler/ExampleCatAction.java b/plugins/examples/rest-handler/src/main/java/org/elasticsearch/example/resthandler/ExampleCatAction.java index d9479349a3d07..a2ae81dd94058 100644 --- a/plugins/examples/rest-handler/src/main/java/org/elasticsearch/example/resthandler/ExampleCatAction.java +++ b/plugins/examples/rest-handler/src/main/java/org/elasticsearch/example/resthandler/ExampleCatAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.resthandler; diff --git a/plugins/examples/rest-handler/src/main/java/org/elasticsearch/example/resthandler/ExampleRestHandlerPlugin.java b/plugins/examples/rest-handler/src/main/java/org/elasticsearch/example/resthandler/ExampleRestHandlerPlugin.java index a820973c19ca3..b85f66bc04804 100644 --- a/plugins/examples/rest-handler/src/main/java/org/elasticsearch/example/resthandler/ExampleRestHandlerPlugin.java +++ b/plugins/examples/rest-handler/src/main/java/org/elasticsearch/example/resthandler/ExampleRestHandlerPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.resthandler; diff --git a/plugins/examples/rest-handler/src/yamlRestTest/java/org/elasticsearch/example/resthandler/ExampleRestHandlerClientYamlTestSuiteIT.java b/plugins/examples/rest-handler/src/yamlRestTest/java/org/elasticsearch/example/resthandler/ExampleRestHandlerClientYamlTestSuiteIT.java index 9ebfc5ebfe9de..67163ff3ab9df 100644 --- a/plugins/examples/rest-handler/src/yamlRestTest/java/org/elasticsearch/example/resthandler/ExampleRestHandlerClientYamlTestSuiteIT.java +++ b/plugins/examples/rest-handler/src/yamlRestTest/java/org/elasticsearch/example/resthandler/ExampleRestHandlerClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.resthandler; diff --git a/plugins/examples/script-expert-scoring/build.gradle b/plugins/examples/script-expert-scoring/build.gradle index 81cc92e8c653e..0fb1baaea2f03 100644 --- a/plugins/examples/script-expert-scoring/build.gradle +++ b/plugins/examples/script-expert-scoring/build.gradle @@ -1,19 +1,20 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.esplugin' apply plugin: 'elasticsearch.yaml-rest-test' esplugin { - name 'script-expert-scoring' - description 'An example script engine to use low level Lucene internals for expert scoring' - classname 'org.elasticsearch.example.expertscript.ExpertScriptPlugin' - licenseFile rootProject.file('SSPL-1.0+ELASTIC-LICENSE-2.0.txt') - noticeFile rootProject.file('NOTICE.txt') + name = 'script-expert-scoring' + description = 'An example script engine to use low level Lucene internals for expert scoring' + classname ='org.elasticsearch.example.expertscript.ExpertScriptPlugin' + licenseFile = rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt') + noticeFile = rootProject.file('NOTICE.txt') } dependencies { diff --git a/plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java b/plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java index 7e177c4b7b6a0..94bfd0d35f377 100644 --- a/plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java +++ b/plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.expertscript; diff --git a/plugins/examples/script-expert-scoring/src/yamlRestTest/java/org/elasticsearch/example/expertscript/ExpertScriptClientYamlTestSuiteIT.java b/plugins/examples/script-expert-scoring/src/yamlRestTest/java/org/elasticsearch/example/expertscript/ExpertScriptClientYamlTestSuiteIT.java index 11d8c0409360b..5f28191db498f 100644 --- a/plugins/examples/script-expert-scoring/src/yamlRestTest/java/org/elasticsearch/example/expertscript/ExpertScriptClientYamlTestSuiteIT.java +++ b/plugins/examples/script-expert-scoring/src/yamlRestTest/java/org/elasticsearch/example/expertscript/ExpertScriptClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.expertscript; diff --git a/plugins/examples/security-authorization-engine/build.gradle b/plugins/examples/security-authorization-engine/build.gradle index 4ab90717eddde..faf32774a20ac 100644 --- a/plugins/examples/security-authorization-engine/build.gradle +++ b/plugins/examples/security-authorization-engine/build.gradle @@ -2,12 +2,12 @@ apply plugin: 'elasticsearch.esplugin' apply plugin: 'elasticsearch.java-rest-test' esplugin { - name 'security-authorization-engine' - description 'An example spi extension plugin for security that implements an Authorization Engine' - classname 'org.elasticsearch.example.AuthorizationEnginePlugin' + name = 'security-authorization-engine' + description = 'An example spi extension plugin for security that implements an Authorization Engine' + classname ='org.elasticsearch.example.AuthorizationEnginePlugin' extendedPlugins = ['x-pack-security'] - licenseFile rootProject.file('SSPL-1.0+ELASTIC-LICENSE-2.0.txt') - noticeFile rootProject.file('NOTICE.txt') + licenseFile = rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt') + noticeFile = rootProject.file('NOTICE.txt') } dependencies { diff --git a/plugins/examples/security-authorization-engine/src/javaRestTest/java/org/elasticsearch/example/CustomAuthorizationEngineIT.java b/plugins/examples/security-authorization-engine/src/javaRestTest/java/org/elasticsearch/example/CustomAuthorizationEngineIT.java index df2911cfdd3d6..b05e30554fb58 100644 --- a/plugins/examples/security-authorization-engine/src/javaRestTest/java/org/elasticsearch/example/CustomAuthorizationEngineIT.java +++ b/plugins/examples/security-authorization-engine/src/javaRestTest/java/org/elasticsearch/example/CustomAuthorizationEngineIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example; diff --git a/plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/AuthorizationEnginePlugin.java b/plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/AuthorizationEnginePlugin.java index 33e6890ee4537..f79b048714770 100644 --- a/plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/AuthorizationEnginePlugin.java +++ b/plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/AuthorizationEnginePlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example; diff --git a/plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/CustomAuthorizationEngine.java b/plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/CustomAuthorizationEngine.java index 3ffefd0a5abe4..523fdc51f6423 100644 --- a/plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/CustomAuthorizationEngine.java +++ b/plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/CustomAuthorizationEngine.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example; diff --git a/plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/ExampleAuthorizationEngineExtension.java b/plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/ExampleAuthorizationEngineExtension.java index d987f7a880ceb..0751941420292 100644 --- a/plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/ExampleAuthorizationEngineExtension.java +++ b/plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/ExampleAuthorizationEngineExtension.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example; diff --git a/plugins/examples/security-authorization-engine/src/test/java/org/elasticsearch/example/CustomAuthorizationEngineTests.java b/plugins/examples/security-authorization-engine/src/test/java/org/elasticsearch/example/CustomAuthorizationEngineTests.java index 4f14771007fa4..dbf797e3d0899 100644 --- a/plugins/examples/security-authorization-engine/src/test/java/org/elasticsearch/example/CustomAuthorizationEngineTests.java +++ b/plugins/examples/security-authorization-engine/src/test/java/org/elasticsearch/example/CustomAuthorizationEngineTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example; diff --git a/plugins/examples/settings.gradle b/plugins/examples/settings.gradle index 09abbfa6b5863..bbeb024957b75 100644 --- a/plugins/examples/settings.gradle +++ b/plugins/examples/settings.gradle @@ -1,13 +1,14 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ plugins { - id "com.gradle.develocity" version "3.17.4" + id "com.gradle.develocity" version "3.18.1" } // Include all subdirectories as example projects @@ -36,6 +37,10 @@ gradle.projectsEvaluated { configurations.all { resolutionStrategy.dependencySubstitution { // When using composite builds we need to tell Gradle to use the project names since we rename the published artifacts + substitute module('org.elasticsearch.plugin:elasticsearch-plugin-api') using module("org.elasticsearch.plugin:plugin-api:${elasticsearchVersion}") + substitute module('org.elasticsearch.plugin:elasticsearch-plugin-analysis-api') using module("org.elasticsearch.plugin:plugin-analysis-api:${elasticsearchVersion}") + substitute module('org.elasticsearch:elasticsearch-plugin-scanner') using module("org.elasticsearch:plugin-scanner:${elasticsearchVersion}") + substitute module('org.elasticsearch:elasticsearch-core') using module("org.elasticsearch:core:${elasticsearchVersion}") substitute module('org.elasticsearch:elasticsearch') using module("org.elasticsearch:server:${elasticsearchVersion}") substitute module('org.elasticsearch.client:elasticsearch-rest-client') using module("org.elasticsearch.client:rest:${elasticsearchVersion}") substitute module('org.elasticsearch.plugin:x-pack-core') using module("org.elasticsearch.plugin:core:${elasticsearchVersion}") diff --git a/plugins/examples/stable-analysis/build.gradle b/plugins/examples/stable-analysis/build.gradle index 358e2585623e3..57126853da8e8 100644 --- a/plugins/examples/stable-analysis/build.gradle +++ b/plugins/examples/stable-analysis/build.gradle @@ -2,8 +2,8 @@ apply plugin: 'elasticsearch.stable-esplugin' apply plugin: 'elasticsearch.yaml-rest-test' esplugin { - name 'stable-analysis-plugin' - description 'An example analysis plugin using stable plugin api' + name = 'stable-analysis-plugin' + description = 'An example analysis plugin using stable plugin api' } //TODO write module-info diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/CharacterSkippingTokenizerFactory.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/CharacterSkippingTokenizerFactory.java index 570dc55c25381..2c1cae7a36b31 100644 --- a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/CharacterSkippingTokenizerFactory.java +++ b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/CharacterSkippingTokenizerFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.analysis; diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/CustomAnalyzerFactory.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/CustomAnalyzerFactory.java index b546a1da8c6fc..b7e6aaec951fe 100644 --- a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/CustomAnalyzerFactory.java +++ b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/CustomAnalyzerFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.analysis; diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleAnalysisSettings.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleAnalysisSettings.java index 024b28e468da0..8f00ce6e85714 100644 --- a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleAnalysisSettings.java +++ b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleAnalysisSettings.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.analysis; diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ReplacingCharFilterFactory.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ReplacingCharFilterFactory.java index 91b509353b6d2..78716932999de 100644 --- a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ReplacingCharFilterFactory.java +++ b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ReplacingCharFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.analysis; diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/SkippingTokenFilterFactory.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/SkippingTokenFilterFactory.java index 75de12b5f6ca4..4e4aa4a588d37 100644 --- a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/SkippingTokenFilterFactory.java +++ b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/SkippingTokenFilterFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.analysis; diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/CharSkippingTokenizer.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/CharSkippingTokenizer.java index 326ae870e5e9c..52a25a52cab4d 100644 --- a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/CharSkippingTokenizer.java +++ b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/CharSkippingTokenizer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.analysis.lucene; diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/ReplaceCharToNumber.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/ReplaceCharToNumber.java index 262748a129ff3..d35d7c7306ec7 100644 --- a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/ReplaceCharToNumber.java +++ b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/ReplaceCharToNumber.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.analysis.lucene; diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/SkipStartingWithDigitTokenFilter.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/SkipStartingWithDigitTokenFilter.java index 0ddfa51af5620..b8343cc004ef7 100644 --- a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/SkipStartingWithDigitTokenFilter.java +++ b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/SkipStartingWithDigitTokenFilter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.analysis.lucene; diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/package-info.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/package-info.java index 5368b1f72d471..b49226afb92d1 100644 --- a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/package-info.java +++ b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/plugins/examples/stable-analysis/src/test/java/org/elasticsearch/example/analysis/ReplacingCharFilterFactoryTests.java b/plugins/examples/stable-analysis/src/test/java/org/elasticsearch/example/analysis/ReplacingCharFilterFactoryTests.java index 9859e9a595857..4f5708d6310c7 100644 --- a/plugins/examples/stable-analysis/src/test/java/org/elasticsearch/example/analysis/ReplacingCharFilterFactoryTests.java +++ b/plugins/examples/stable-analysis/src/test/java/org/elasticsearch/example/analysis/ReplacingCharFilterFactoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.analysis; diff --git a/plugins/examples/stable-analysis/src/yamlRestTest/java/org/elasticsearch/example/stable/client/ExampleStableAnalysisClientYamlTestSuiteIT.java b/plugins/examples/stable-analysis/src/yamlRestTest/java/org/elasticsearch/example/stable/client/ExampleStableAnalysisClientYamlTestSuiteIT.java index 95c63c8f79d0a..71e62ded4ec23 100644 --- a/plugins/examples/stable-analysis/src/yamlRestTest/java/org/elasticsearch/example/stable/client/ExampleStableAnalysisClientYamlTestSuiteIT.java +++ b/plugins/examples/stable-analysis/src/yamlRestTest/java/org/elasticsearch/example/stable/client/ExampleStableAnalysisClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.example.stable.client; diff --git a/plugins/mapper-annotated-text/build.gradle b/plugins/mapper-annotated-text/build.gradle index 275e7f5267e51..d475a36f8d62d 100644 --- a/plugins/mapper-annotated-text/build.gradle +++ b/plugins/mapper-annotated-text/build.gradle @@ -2,23 +2,18 @@ import org.elasticsearch.gradle.internal.info.BuildParams /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ -apply plugin: 'elasticsearch.legacy-yaml-rest-test' -apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { - description 'The Mapper Annotated_text plugin adds support for text fields with markup used to inject annotation tokens into the index.' - classname 'org.elasticsearch.index.mapper.annotatedtext.AnnotatedTextPlugin' -} - -if (BuildParams.isSnapshotBuild() == false) { - tasks.named("test").configure { - systemProperty 'es.index_mode_feature_flag_registered', 'true' - } + description = 'The Mapper Annotated_text plugin adds support for text fields with markup used to inject annotation tokens into the index.' + classname ='org.elasticsearch.index.mapper.annotatedtext.AnnotatedTextPlugin' } restResources { diff --git a/plugins/mapper-annotated-text/src/main/java/module-info.java b/plugins/mapper-annotated-text/src/main/java/module-info.java index 3aa8e46e2980c..13f2bd66418be 100644 --- a/plugins/mapper-annotated-text/src/main/java/module-info.java +++ b/plugins/mapper-annotated-text/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.index.mapper.annotatedtext { diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedPassageFormatter.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedPassageFormatter.java index 6a6e378834706..cfec72fa91444 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedPassageFormatter.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedPassageFormatter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.annotatedtext; diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index b66ce41d3259d..c12849d545b33 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.annotatedtext; @@ -30,6 +31,7 @@ import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.StringStoredFieldFieldLoader; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TextParams; @@ -90,15 +92,10 @@ public static class Builder extends FieldMapper.Builder { private final IndexVersion indexCreatedVersion; private final TextParams.Analyzers analyzers; - private final boolean isSyntheticSourceEnabledViaIndexMode; + private final boolean isSyntheticSourceEnabled; private final Parameter store; - public Builder( - String name, - IndexVersion indexCreatedVersion, - IndexAnalyzers indexAnalyzers, - boolean isSyntheticSourceEnabledViaIndexMode - ) { + public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers indexAnalyzers, boolean isSyntheticSourceEnabled) { super(name); this.indexCreatedVersion = indexCreatedVersion; this.analyzers = new TextParams.Analyzers( @@ -107,10 +104,10 @@ public Builder( m -> builder(m).analyzers.positionIncrementGap.getValue(), indexCreatedVersion ); - this.isSyntheticSourceEnabledViaIndexMode = isSyntheticSourceEnabledViaIndexMode; + this.isSyntheticSourceEnabled = isSyntheticSourceEnabled; this.store = Parameter.storeParam( m -> builder(m).store.getValue(), - () -> isSyntheticSourceEnabledViaIndexMode && multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField() == false + () -> isSyntheticSourceEnabled && multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField() == false ); } @@ -171,7 +168,7 @@ public AnnotatedTextFieldMapper build(MapperBuilderContext context) { } public static TypeParser PARSER = new TypeParser( - (n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers(), c.getIndexSettings().getMode().isSyntheticSourceEnabled()) + (n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers(), SourceFieldMapper.isSynthetic(c.getIndexSettings())) ); /** @@ -559,12 +556,8 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder( - leafName(), - builder.indexCreatedVersion, - builder.analyzers.indexAnalyzers, - builder.isSyntheticSourceEnabledViaIndexMode - ).init(this); + return new Builder(leafName(), builder.indexCreatedVersion, builder.analyzers.indexAnalyzers, builder.isSyntheticSourceEnabled) + .init(this); } @Override @@ -582,7 +575,7 @@ protected void write(XContentBuilder b, Object value) throws IOException { var kwd = TextFieldMapper.SyntheticSourceHelper.getKeywordFieldMapperForSyntheticSource(this); if (kwd != null) { - return new SyntheticSourceSupport.Native(kwd.syntheticFieldLoader(leafName())); + return new SyntheticSourceSupport.Native(kwd.syntheticFieldLoader(fullPath(), leafName())); } return super.syntheticSourceSupport(); diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighter.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighter.java index 9b7c6afbb9f10..8b4a9d6544b75 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighter.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.annotatedtext; diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextPlugin.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextPlugin.java index 8c2e8c89a9835..1124b24943d4a 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextPlugin.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.annotatedtext; diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/Features.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/Features.java index 1c4bd22e88145..51a2d2bbe1d40 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/Features.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/Features.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.annotatedtext; diff --git a/plugins/mapper-annotated-text/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification b/plugins/mapper-annotated-text/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification index a19d9deb9c522..c9c9faed7ec77 100644 --- a/plugins/mapper-annotated-text/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification +++ b/plugins/mapper-annotated-text/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # org.elasticsearch.index.mapper.annotatedtext.Features diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java index a5319387a2b68..593d4b41df712 100644 --- a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java +++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.annotatedtext; diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java index 2a78699c8a4a9..fd248279b238d 100644 --- a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java +++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.annotatedtext; diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighterTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighterTests.java index 5a121ffa53658..61abd64e98a96 100644 --- a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighterTests.java +++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighterTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.annotatedtext; diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextParsingTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextParsingTests.java index 4b27a148acf2e..ad034aed2dcb4 100644 --- a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextParsingTests.java +++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextParsingTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.annotatedtext; diff --git a/plugins/mapper-annotated-text/src/yamlRestTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextClientYamlTestSuiteIT.java b/plugins/mapper-annotated-text/src/yamlRestTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextClientYamlTestSuiteIT.java index b5c46f06fa54e..68d141b6df840 100644 --- a/plugins/mapper-annotated-text/src/yamlRestTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextClientYamlTestSuiteIT.java +++ b/plugins/mapper-annotated-text/src/yamlRestTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.annotatedtext; @@ -11,8 +12,10 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; public class AnnotatedTextClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { @@ -24,4 +27,12 @@ public AnnotatedTextClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate public static Iterable parameters() throws Exception { return createParameters(); } + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().plugin("mapper-annotated-text").build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/plugins/mapper-annotated-text/src/yamlRestTest/resources/rest-api-spec/test/mapper_annotatedtext/20_synthetic_source.yml b/plugins/mapper-annotated-text/src/yamlRestTest/resources/rest-api-spec/test/mapper_annotatedtext/20_synthetic_source.yml index 4aac881700e15..ca3a9baa7b694 100644 --- a/plugins/mapper-annotated-text/src/yamlRestTest/resources/rest-api-spec/test/mapper_annotatedtext/20_synthetic_source.yml +++ b/plugins/mapper-annotated-text/src/yamlRestTest/resources/rest-api-spec/test/mapper_annotatedtext/20_synthetic_source.yml @@ -1,8 +1,8 @@ --- setup: - requires: - cluster_features: ["mapper.annotated_text.synthetic_source"] - reason: introduced in 8.15.0 + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" --- stored annotated_text field: @@ -10,9 +10,10 @@ stored annotated_text field: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: annotated_text: type: annotated_text @@ -40,9 +41,10 @@ annotated_text field with keyword multi-field: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: annotated_text: type: annotated_text @@ -72,9 +74,10 @@ multiple values in stored annotated_text field: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: annotated_text: type: annotated_text @@ -102,9 +105,10 @@ multiple values in annotated_text field with keyword multi-field: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: annotated_text: type: annotated_text @@ -135,9 +139,10 @@ multiple values in annotated_text field with stored keyword multi-field: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: annotated_text: type: annotated_text @@ -169,9 +174,10 @@ multiple values in stored annotated_text field with keyword multi-field: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: annotated_text: type: annotated_text @@ -202,9 +208,10 @@ fallback synthetic source: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: annotated_text: type: annotated_text diff --git a/plugins/mapper-murmur3/build.gradle b/plugins/mapper-murmur3/build.gradle index 7e93e6d295397..33a6cbadfd2c9 100644 --- a/plugins/mapper-murmur3/build.gradle +++ b/plugins/mapper-murmur3/build.gradle @@ -2,29 +2,25 @@ import org.elasticsearch.gradle.internal.info.BuildParams /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ -apply plugin: 'elasticsearch.legacy-yaml-rest-test' -apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { - description 'The Mapper Murmur3 plugin allows to compute hashes of a field\'s values at index-time and to store them in the index.' - classname 'org.elasticsearch.plugin.mapper.MapperMurmur3Plugin' + description = 'The Mapper Murmur3 plugin allows to compute hashes of a field\'s values at index-time and to store them in the index.' + classname ='org.elasticsearch.plugin.mapper.MapperMurmur3Plugin' extendedPlugins = ['lang-painless'] } dependencies { compileOnly project(':modules:lang-painless:spi') testImplementation project(':modules:lang-painless') -} - -if (BuildParams.isSnapshotBuild() == false) { - tasks.named("test").configure { - systemProperty 'es.index_mode_feature_flag_registered', 'true' - } + clusterModules project(':modules:lang-painless') } restResources { @@ -32,8 +28,3 @@ restResources { include '_common', 'indices', 'index', 'search' } } - -testClusters.configureEach { - testDistribution = 'DEFAULT' - setting 'xpack.security.enabled', 'false' -} diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index 979ca842ef346..2440668ff57be 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.murmur3; diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3Plugin.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3Plugin.java index 84d74d6f1b6e8..843516ab736e8 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3Plugin.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3Plugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.mapper; diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/script/field/murmur3/Murmur3DocValueField.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/script/field/murmur3/Murmur3DocValueField.java index 2f880c782f1d8..324fe910d36bb 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/script/field/murmur3/Murmur3DocValueField.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/script/field/murmur3/Murmur3DocValueField.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.field.murmur3; diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/script/field/murmur3/Murmur3PainlessExtension.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/script/field/murmur3/Murmur3PainlessExtension.java index 344642482947f..76a8cade454fc 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/script/field/murmur3/Murmur3PainlessExtension.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/script/field/murmur3/Murmur3PainlessExtension.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script.field.murmur3; diff --git a/plugins/mapper-murmur3/src/main/resources/org/elasticsearch/script/field/murmur3/org.elasticsearch.field.murmur3.txt b/plugins/mapper-murmur3/src/main/resources/org/elasticsearch/script/field/murmur3/org.elasticsearch.field.murmur3.txt index ae866177cba6e..14eff6776445b 100644 --- a/plugins/mapper-murmur3/src/main/resources/org/elasticsearch/script/field/murmur3/org.elasticsearch.field.murmur3.txt +++ b/plugins/mapper-murmur3/src/main/resources/org/elasticsearch/script/field/murmur3/org.elasticsearch.field.murmur3.txt @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # # subclass of AbstractLongDocValuesField diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java index eb17114ae9e6c..cd02c34a064b0 100644 --- a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java +++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.murmur3; diff --git a/plugins/mapper-murmur3/src/yamlRestTest/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3ClientYamlTestSuiteIT.java b/plugins/mapper-murmur3/src/yamlRestTest/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3ClientYamlTestSuiteIT.java index 82da6ca3f9efa..399b488a5d2f7 100644 --- a/plugins/mapper-murmur3/src/yamlRestTest/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3ClientYamlTestSuiteIT.java +++ b/plugins/mapper-murmur3/src/yamlRestTest/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3ClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.murmur3; @@ -11,8 +12,10 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; public class MapperMurmur3ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { @@ -24,4 +27,12 @@ public MapperMurmur3ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate public static Iterable parameters() throws Exception { return createParameters(); } + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("lang-painless").plugin("mapper-murmur3").build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } } diff --git a/plugins/mapper-murmur3/src/yamlRestTest/resources/rest-api-spec/test/mapper_murmur3/10_basic.yml b/plugins/mapper-murmur3/src/yamlRestTest/resources/rest-api-spec/test/mapper_murmur3/10_basic.yml index 12b23fb3b0395..4a2ca8544fe8d 100644 --- a/plugins/mapper-murmur3/src/yamlRestTest/resources/rest-api-spec/test/mapper_murmur3/10_basic.yml +++ b/plugins/mapper-murmur3/src/yamlRestTest/resources/rest-api-spec/test/mapper_murmur3/10_basic.yml @@ -129,14 +129,18 @@ setup: --- "Murmur3 docvalue_fields api with synthetic source": + - requires: + cluster_features: ["mapper.source.mode_from_index_setting"] + reason: "Source mode configured through index setting" - do: indices.create: index: test_synthetic_source body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: foo: type: keyword diff --git a/plugins/mapper-size/build.gradle b/plugins/mapper-size/build.gradle index ca4738335d0cc..0f9bf59767d90 100644 --- a/plugins/mapper-size/build.gradle +++ b/plugins/mapper-size/build.gradle @@ -1,17 +1,18 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.internal-yaml-rest-test' apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { - description 'The Mapper Size plugin allows document to record their uncompressed size at index time.' - classname 'org.elasticsearch.plugin.mapper.MapperSizePlugin' + description = 'The Mapper Size plugin allows document to record their uncompressed size at index time.' + classname ='org.elasticsearch.plugin.mapper.MapperSizePlugin' } restResources { diff --git a/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java b/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java index ee16153a98de1..c2251910c3122 100644 --- a/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java +++ b/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.size; @@ -24,6 +25,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponses; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.is; @@ -135,14 +137,11 @@ public void testWildCardWithFieldsWhenDisabled() throws Exception { assertAcked(prepareCreate("test").setMapping("_size", "enabled=false")); final String source = "{\"f\":\"" + randomAlphaOfLengthBetween(1, 100) + "\"}"; indexRandom(true, prepareIndex("test").setId("1").setSource(source, XContentType.JSON)); - assertResponse( + assertResponses( + response -> assertNull(response.getHits().getHits()[0].getFields().get("_size")), prepareSearch("test").addFetchField("_size"), - response -> assertNull(response.getHits().getHits()[0].getFields().get("_size")) - ); - - assertResponse( prepareSearch("test").addFetchField("*"), - response -> assertNull(response.getHits().getHits()[0].getFields().get("_size")) + prepareSearch("test").addStoredField("*") ); assertResponse( @@ -155,19 +154,11 @@ public void testWildCardWithFieldsWhenNotProvided() throws Exception { assertAcked(prepareCreate("test")); final String source = "{\"f\":\"" + randomAlphaOfLengthBetween(1, 100) + "\"}"; indexRandom(true, prepareIndex("test").setId("1").setSource(source, XContentType.JSON)); - assertResponse( + assertResponses( + response -> assertNull(response.getHits().getHits()[0].getFields().get("_size")), prepareSearch("test").addFetchField("_size"), - response -> assertNull(response.getHits().getHits()[0].getFields().get("_size")) - ); - - assertResponse( prepareSearch("test").addFetchField("*"), - response -> assertNull(response.getHits().getHits()[0].getFields().get("_size")) - ); - - assertResponse( - prepareSearch("test").addStoredField("*"), - response -> assertNull(response.getHits().getHits()[0].getFields().get("_size")) + prepareSearch("test").addStoredField("*") ); } } diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java index 6a4869b8c89b7..68b93db39646f 100644 --- a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java +++ b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.size; diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/plugin/mapper/MapperSizePlugin.java b/plugins/mapper-size/src/main/java/org/elasticsearch/plugin/mapper/MapperSizePlugin.java index 96ac258466ecf..a7bc7d6a21057 100644 --- a/plugins/mapper-size/src/main/java/org/elasticsearch/plugin/mapper/MapperSizePlugin.java +++ b/plugins/mapper-size/src/main/java/org/elasticsearch/plugin/mapper/MapperSizePlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.mapper; diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java index dbad4ec37a237..defa112a7834b 100644 --- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java +++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.size; diff --git a/plugins/mapper-size/src/yamlRestTest/java/org/elasticsearch/index/mapper/size/MapperSizeClientYamlTestSuiteIT.java b/plugins/mapper-size/src/yamlRestTest/java/org/elasticsearch/index/mapper/size/MapperSizeClientYamlTestSuiteIT.java index e9730418307e9..3938ff8f8cac8 100644 --- a/plugins/mapper-size/src/yamlRestTest/java/org/elasticsearch/index/mapper/size/MapperSizeClientYamlTestSuiteIT.java +++ b/plugins/mapper-size/src/yamlRestTest/java/org/elasticsearch/index/mapper/size/MapperSizeClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper.size; diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 2d32d1827cb13..b175f483fa724 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.internal.test.RestIntegTestTask @@ -14,12 +15,12 @@ apply plugin: 'elasticsearch.internal-java-rest-test' apply plugin: 'elasticsearch.internal-yaml-rest-test' esplugin { - description 'The HDFS repository plugin adds support for Hadoop Distributed File-System (HDFS) repositories.' - classname 'org.elasticsearch.repositories.hdfs.HdfsPlugin' + description = 'The HDFS repository plugin adds support for Hadoop Distributed File-System (HDFS) repositories.' + classname ='org.elasticsearch.repositories.hdfs.HdfsPlugin' } versions << [ - 'hadoop': '3.3.3' + 'hadoop': '3.4.1' ] configurations { @@ -28,7 +29,7 @@ configurations { } dependencies { - api project(path: 'hadoop-client-api', configuration: 'shadow') + api project(path: 'hadoop-client-api', configuration: 'default') if (isEclipse) { /* * Eclipse can't pick up the shadow dependency so we point it at *something* @@ -41,9 +42,9 @@ dependencies { api "com.google.protobuf:protobuf-java:${versions.protobuf}" api "commons-logging:commons-logging:${versions.commonslogging}" api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}" - api 'commons-cli:commons-cli:1.2' + api 'commons-cli:commons-cli:1.5.0' api "commons-codec:commons-codec:${versions.commonscodec}" - api 'commons-io:commons-io:2.8.0' + api 'commons-io:commons-io:2.16.1' api 'org.apache.commons:commons-lang3:3.11' api 'javax.servlet:javax.servlet-api:3.1.0' api "org.slf4j:slf4j-api:${versions.slf4j}" @@ -57,14 +58,14 @@ dependencies { javaRestTestImplementation project(':test:fixtures:krb5kdc-fixture') javaRestTestImplementation "org.slf4j:slf4j-api:${versions.slf4j}" javaRestTestRuntimeOnly "com.google.guava:guava:16.0.1" - javaRestTestRuntimeOnly "commons-cli:commons-cli:1.2" + javaRestTestRuntimeOnly "commons-cli:commons-cli:1.5.0" javaRestTestRuntimeOnly "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}" yamlRestTestCompileOnly(project(':test:fixtures:hdfs-fixture')) yamlRestTestImplementation project(':test:fixtures:krb5kdc-fixture') yamlRestTestImplementation "org.slf4j:slf4j-api:${versions.slf4j}" yamlRestTestRuntimeOnly "com.google.guava:guava:16.0.1" - yamlRestTestRuntimeOnly "commons-cli:commons-cli:1.2" + yamlRestTestRuntimeOnly "commons-cli:commons-cli:1.5.0" yamlRestTestRuntimeOnly "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}" hdfsFixture2 project(path: ':test:fixtures:hdfs-fixture', configuration: 'shadowedHdfs2') @@ -83,7 +84,7 @@ tasks.named("dependencyLicenses").configure { tasks.withType(RestIntegTestTask).configureEach { usesDefaultDistribution() - BuildParams.withFipsEnabledOnly(it) + buildParams.withFipsEnabledOnly(it) jvmArgs '--add-exports', 'java.security.jgss/sun.security.krb5=ALL-UNNAMED' } @@ -177,7 +178,6 @@ tasks.named("thirdPartyAudit").configure { 'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray', 'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1', 'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2', - 'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3', 'org.apache.hadoop.thirdparty.com.google.common.hash.Striped64', 'org.apache.hadoop.thirdparty.com.google.common.hash.Striped64$1', 'org.apache.hadoop.thirdparty.com.google.common.hash.Striped64$Cell', @@ -188,6 +188,9 @@ tasks.named("thirdPartyAudit").configure { 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil', 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$1', 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$JvmMemoryAccessor', - 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$MemoryAccessor' + 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$MemoryAccessor', + 'org.apache.hadoop.thirdparty.protobuf.MessageSchema', + 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$Android32MemoryAccessor', + 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$Android64MemoryAccessor' ) } diff --git a/plugins/repository-hdfs/hadoop-client-api/build.gradle b/plugins/repository-hdfs/hadoop-client-api/build.gradle index 4ac6f79530fcb..46b0d949cdee2 100644 --- a/plugins/repository-hdfs/hadoop-client-api/build.gradle +++ b/plugins/repository-hdfs/hadoop-client-api/build.gradle @@ -1,16 +1,54 @@ -apply plugin: 'elasticsearch.build' -apply plugin: 'com.github.johnrengelman.shadow' +import org.gradle.api.file.ArchiveOperations + +apply plugin: 'elasticsearch.java' + +sourceSets { + patcher +} + +configurations { + thejar { + canBeResolved = true + } +} dependencies { - implementation "org.apache.hadoop:hadoop-client-api:${project.parent.versions.hadoop}" + thejar("org.apache.hadoop:hadoop-client-api:${project.parent.versions.hadoop}") { + transitive = false + } + + patcherImplementation 'org.ow2.asm:asm:9.7.1' + patcherImplementation 'org.ow2.asm:asm-tree:9.7.1' +} + +def outputDir = layout.buildDirectory.dir("patched-classes") + +def patchTask = tasks.register("patchClasses", JavaExec) { + inputs.files(configurations.thejar).withPathSensitivity(PathSensitivity.RELATIVE) + inputs.files(sourceSets.patcher.output).withPathSensitivity(PathSensitivity.RELATIVE) + outputs.dir(outputDir) + classpath = sourceSets.patcher.runtimeClasspath + mainClass = 'org.elasticsearch.hdfs.patch.HdfsClassPatcher' + def thejar = configurations.thejar + doFirst { + args(thejar.singleFile, outputDir.get().asFile) + } } -tasks.named('shadowJar').configure { - exclude 'org/apache/hadoop/util/ShutdownHookManager$*.class' +interface InjectedArchiveOps { + @Inject ArchiveOperations getArchiveOperations() } -['jarHell', 'thirdPartyAudit', 'forbiddenApisMain', 'splitPackagesAudit'].each { - tasks.named(it).configure { - enabled = false +tasks.named('jar').configure { + dependsOn(configurations.thejar) + def injected = project.objects.newInstance(InjectedArchiveOps) + def thejar = configurations.thejar + from(patchTask) + from({ injected.getArchiveOperations().zipTree(thejar.singleFile) }) { + eachFile { + if (outputDir.get().file(it.relativePath.pathString).asFile.exists()) { + it.exclude() + } + } } } diff --git a/plugins/repository-hdfs/hadoop-client-api/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java b/plugins/repository-hdfs/hadoop-client-api/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java deleted file mode 100644 index 5e95d68d6638c..0000000000000 --- a/plugins/repository-hdfs/hadoop-client-api/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.apache.hadoop.util; - -import java.util.concurrent.TimeUnit; - -/** - * A replacement for the ShutdownHookManager from hadoop. - * - * This class does not actually add a shutdown hook. Hadoop's shutdown hook - * manager does not fail gracefully when it lacks security manager permissions - * to add shutdown hooks. This implements the same api as the hadoop class, but - * with no-ops. - */ -public class ShutdownHookManager { - private static final ShutdownHookManager MGR = new ShutdownHookManager(); - - public static ShutdownHookManager get() { - return MGR; - } - - private ShutdownHookManager() {} - - public void addShutdownHook(Runnable shutdownHook, int priority) {} - - public void addShutdownHook(Runnable shutdownHook, int priority, long timeout, TimeUnit unit) {} - - public boolean removeShutdownHook(Runnable shutdownHook) { - return false; - } - - public boolean hasShutdownHook(Runnable shutdownHook) { - return false; - } - - public boolean isShutdownInProgress() { - return false; - } - - public void clearShutdownHooks() {} -} diff --git a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/HdfsClassPatcher.java b/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/HdfsClassPatcher.java new file mode 100644 index 0000000000000..6636b39445964 --- /dev/null +++ b/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/HdfsClassPatcher.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.hdfs.patch; + +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.ClassWriter; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Map; +import java.util.function.Function; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; + +public class HdfsClassPatcher { + static final Map> patchers = Map.of( + "org/apache/hadoop/util/ShutdownHookManager.class", + ShutdownHookManagerPatcher::new, + "org/apache/hadoop/util/Shell.class", + ShellPatcher::new + ); + + public static void main(String[] args) throws Exception { + String jarPath = args[0]; + Path outputDir = Paths.get(args[1]); + + try (JarFile jarFile = new JarFile(new File(jarPath))) { + for (var patcher : patchers.entrySet()) { + JarEntry jarEntry = jarFile.getJarEntry(patcher.getKey()); + if (jarEntry == null) { + throw new IllegalArgumentException("path [" + patcher.getKey() + "] not found in [" + jarPath + "]"); + } + byte[] classToPatch = jarFile.getInputStream(jarEntry).readAllBytes(); + + ClassReader classReader = new ClassReader(classToPatch); + ClassWriter classWriter = new ClassWriter(classReader, 0); + classReader.accept(patcher.getValue().apply(classWriter), 0); + + Path outputFile = outputDir.resolve(patcher.getKey()); + Files.createDirectories(outputFile.getParent()); + Files.write(outputFile, classWriter.toByteArray()); + } + } + } +} diff --git a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/MethodReplacement.java b/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/MethodReplacement.java new file mode 100644 index 0000000000000..e07a32cc294a5 --- /dev/null +++ b/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/MethodReplacement.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.hdfs.patch; + +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; + +public class MethodReplacement extends MethodVisitor { + private final MethodVisitor delegate; + private final Runnable bodyWriter; + + MethodReplacement(MethodVisitor delegate, Runnable bodyWriter) { + super(Opcodes.ASM9); + this.delegate = delegate; + this.bodyWriter = bodyWriter; + } + + @Override + public void visitCode() { + // delegate.visitCode(); + bodyWriter.run(); + // delegate.visitEnd(); + } + + @Override + public void visitMaxs(int maxStack, int maxLocals) { + delegate.visitMaxs(maxStack, maxLocals); + } +} diff --git a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/ShellPatcher.java b/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/ShellPatcher.java new file mode 100644 index 0000000000000..397b63e434ba2 --- /dev/null +++ b/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/ShellPatcher.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.hdfs.patch; + +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; + +class ShellPatcher extends ClassVisitor { + + ShellPatcher(ClassWriter classWriter) { + super(Opcodes.ASM9, classWriter); + } + + @Override + public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { + MethodVisitor mv = super.visitMethod(access, name, descriptor, signature, exceptions); + if (name.equals("isSetsidSupported")) { + return new MethodReplacement(mv, () -> { + mv.visitInsn(Opcodes.ICONST_0); + mv.visitInsn(Opcodes.IRETURN); + }); + } + return mv; + } +} diff --git a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/ShutdownHookManagerPatcher.java b/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/ShutdownHookManagerPatcher.java new file mode 100644 index 0000000000000..1235b5af9002f --- /dev/null +++ b/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/ShutdownHookManagerPatcher.java @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.hdfs.patch; + +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.Type; + +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; + +class ShutdownHookManagerPatcher extends ClassVisitor { + private static final String CLASSNAME = "org/apache/hadoop/util/ShutdownHookManager"; + private static final Set VOID_METHODS = Set.of("addShutdownHook", "clearShutdownHooks"); + private static final Set BOOLEAN_METHODS = Set.of("removeShutdownHook", "hasShutdownHook", "isShutdownInProgress"); + + ShutdownHookManagerPatcher(ClassWriter classWriter) { + super(Opcodes.ASM9, classWriter); + } + + @Override + public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { + MethodVisitor mv = super.visitMethod(access, name, descriptor, signature, exceptions); + if (VOID_METHODS.contains(name)) { + // make void methods noops + return new MethodReplacement(mv, () -> { mv.visitInsn(Opcodes.RETURN); }); + } else if (BOOLEAN_METHODS.contains(name)) { + // make boolean methods always return false + return new MethodReplacement(mv, () -> { + mv.visitInsn(Opcodes.ICONST_0); + mv.visitInsn(Opcodes.IRETURN); + }); + } else if (name.equals("")) { + return new MethodReplacement(mv, () -> { + // just initialize the statics, don't actually get runtime to add shutdown hook + + var classType = Type.getObjectType(CLASSNAME); + mv.visitTypeInsn(Opcodes.NEW, CLASSNAME); + mv.visitInsn(Opcodes.DUP); + mv.visitMethodInsn(Opcodes.INVOKESPECIAL, CLASSNAME, "", "()V", false); + mv.visitFieldInsn(Opcodes.PUTSTATIC, CLASSNAME, "MGR", classType.getDescriptor()); + + var timeUnitType = Type.getType(TimeUnit.class); + mv.visitFieldInsn(Opcodes.GETSTATIC, timeUnitType.getInternalName(), "SECONDS", timeUnitType.getDescriptor()); + mv.visitFieldInsn(Opcodes.PUTSTATIC, CLASSNAME, "TIME_UNIT_DEFAULT", timeUnitType.getDescriptor()); + + var executorServiceType = Type.getType(ExecutorService.class); + mv.visitInsn(Opcodes.ACONST_NULL); + mv.visitFieldInsn(Opcodes.PUTSTATIC, CLASSNAME, "EXECUTOR", executorServiceType.getDescriptor()); + + mv.visitInsn(Opcodes.RETURN); + }); + } + return mv; + } +} diff --git a/plugins/repository-hdfs/src/javaRestTest/java/org/elasticsearch/repositories/hdfs/AbstractHaHdfsFailoverTestSuiteIT.java b/plugins/repository-hdfs/src/javaRestTest/java/org/elasticsearch/repositories/hdfs/AbstractHaHdfsFailoverTestSuiteIT.java index d14cff30caef3..3f76a4587b8f0 100644 --- a/plugins/repository-hdfs/src/javaRestTest/java/org/elasticsearch/repositories/hdfs/AbstractHaHdfsFailoverTestSuiteIT.java +++ b/plugins/repository-hdfs/src/javaRestTest/java/org/elasticsearch/repositories/hdfs/AbstractHaHdfsFailoverTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.hdfs; diff --git a/plugins/repository-hdfs/src/javaRestTest/java/org/elasticsearch/repositories/hdfs/HaHdfsFailoverTestSuiteIT.java b/plugins/repository-hdfs/src/javaRestTest/java/org/elasticsearch/repositories/hdfs/HaHdfsFailoverTestSuiteIT.java index 7bd15ad64582f..0da52664e6b2c 100644 --- a/plugins/repository-hdfs/src/javaRestTest/java/org/elasticsearch/repositories/hdfs/HaHdfsFailoverTestSuiteIT.java +++ b/plugins/repository-hdfs/src/javaRestTest/java/org/elasticsearch/repositories/hdfs/HaHdfsFailoverTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.hdfs; diff --git a/plugins/repository-hdfs/src/javaRestTest/java/org/elasticsearch/repositories/hdfs/SecureHaHdfsFailoverTestSuiteIT.java b/plugins/repository-hdfs/src/javaRestTest/java/org/elasticsearch/repositories/hdfs/SecureHaHdfsFailoverTestSuiteIT.java index 8ba27f703c419..0b23747cda4c1 100644 --- a/plugins/repository-hdfs/src/javaRestTest/java/org/elasticsearch/repositories/hdfs/SecureHaHdfsFailoverTestSuiteIT.java +++ b/plugins/repository-hdfs/src/javaRestTest/java/org/elasticsearch/repositories/hdfs/SecureHaHdfsFailoverTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.hdfs; diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java index fe74ba7a5b0ea..c8554c04ff23b 100644 --- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.hdfs; diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobStore.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobStore.java index a7c40980858ad..e817384d95c04 100644 --- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobStore.java +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobStore.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.hdfs; @@ -15,10 +16,8 @@ import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; -import org.elasticsearch.common.blobstore.OperationPurpose; import java.io.IOException; -import java.util.Iterator; final class HdfsBlobStore implements BlobStore { @@ -71,11 +70,6 @@ public BlobContainer blobContainer(BlobPath path) { return new HdfsBlobContainer(path, this, buildHdfsPath(path), bufferSize, securityContext, replicationFactor); } - @Override - public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobNames) throws IOException { - throw new UnsupportedOperationException("Bulk deletes are not supported in Hdfs repositories"); - } - private Path buildHdfsPath(BlobPath blobPath) { final Path path = translateToHdfsPath(blobPath); if (readOnly == false) { diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java index 426899643859b..ff63d3a1799e5 100644 --- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.hdfs; diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java index a855428c3252b..2a1847d65b70b 100644 --- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.hdfs; diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsSecurityContext.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsSecurityContext.java index 630afc8f18287..ce6acd79a0bb9 100644 --- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsSecurityContext.java +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsSecurityContext.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.hdfs; @@ -46,7 +47,8 @@ class HdfsSecurityContext { // 2) allow hadoop to add credentials to our Subject new AuthPermission("modifyPrivateCredentials"), // 3) RPC Engine requires this for re-establishing pooled connections over the lifetime of the client - new PrivateCredentialPermission("org.apache.hadoop.security.Credentials * \"*\"", "read") }; + new PrivateCredentialPermission("org.apache.hadoop.security.Credentials * \"*\"", "read"), + new RuntimePermission("getClassLoader") }; // If Security is enabled, we need all the following elevated permissions: KERBEROS_AUTH_PERMISSIONS = new Permission[] { diff --git a/plugins/repository-hdfs/src/main/plugin-metadata/plugin-security.policy b/plugins/repository-hdfs/src/main/plugin-metadata/plugin-security.policy index cd9f5e8d61490..db5a511267626 100644 --- a/plugins/repository-hdfs/src/main/plugin-metadata/plugin-security.policy +++ b/plugins/repository-hdfs/src/main/plugin-metadata/plugin-security.policy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ grant { diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java index ee1e54e8a3356..5808051026197 100644 --- a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java +++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.hdfs; diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreRepositoryTests.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreRepositoryTests.java index a52724496289a..3e1c112a4d9f7 100644 --- a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreRepositoryTests.java +++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreRepositoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.hdfs; @@ -45,11 +46,6 @@ public void testSnapshotAndRestore() throws Exception { testSnapshotAndRestore(false); } - @Override - public void testBlobStoreBulkDeletion() throws Exception { - // HDFS does not implement bulk deletion from different BlobContainers - } - @Override protected Collection> nodePlugins() { return Collections.singletonList(HdfsPlugin.class); diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsRepositoryTests.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsRepositoryTests.java index 6726929bdc91c..7961ca0257be8 100644 --- a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsRepositoryTests.java +++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsRepositoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.hdfs; diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsTests.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsTests.java index 39bc59012ed09..a95ba1b6fc7a7 100644 --- a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsTests.java +++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.hdfs; diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/TestingFs.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/TestingFs.java index 27aa8ebb507aa..4b26676c2b2a3 100644 --- a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/TestingFs.java +++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/TestingFs.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.hdfs; diff --git a/plugins/repository-hdfs/src/yamlRestTest/java/org/elasticsearch/repositories/hdfs/RepositoryHdfsClientYamlTestSuiteIT.java b/plugins/repository-hdfs/src/yamlRestTest/java/org/elasticsearch/repositories/hdfs/RepositoryHdfsClientYamlTestSuiteIT.java index a0a4d9379bc78..6a91cc51a564a 100644 --- a/plugins/repository-hdfs/src/yamlRestTest/java/org/elasticsearch/repositories/hdfs/RepositoryHdfsClientYamlTestSuiteIT.java +++ b/plugins/repository-hdfs/src/yamlRestTest/java/org/elasticsearch/repositories/hdfs/RepositoryHdfsClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.hdfs; diff --git a/plugins/repository-hdfs/src/yamlRestTest/java/org/elasticsearch/repositories/hdfs/SecureRepositoryHdfsClientYamlTestSuiteIT.java b/plugins/repository-hdfs/src/yamlRestTest/java/org/elasticsearch/repositories/hdfs/SecureRepositoryHdfsClientYamlTestSuiteIT.java index 45b992a35d731..f238a17993856 100644 --- a/plugins/repository-hdfs/src/yamlRestTest/java/org/elasticsearch/repositories/hdfs/SecureRepositoryHdfsClientYamlTestSuiteIT.java +++ b/plugins/repository-hdfs/src/yamlRestTest/java/org/elasticsearch/repositories/hdfs/SecureRepositoryHdfsClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.hdfs; diff --git a/plugins/store-smb/build.gradle b/plugins/store-smb/build.gradle index 7147c73027334..727f9ed588673 100644 --- a/plugins/store-smb/build.gradle +++ b/plugins/store-smb/build.gradle @@ -1,16 +1,17 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.legacy-yaml-rest-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { - description 'The Store SMB plugin adds support for SMB stores.' - classname 'org.elasticsearch.plugin.store.smb.SMBStorePlugin' + description = 'The Store SMB plugin adds support for SMB stores.' + classname ='org.elasticsearch.plugin.store.smb.SMBStorePlugin' } restResources { restApi { diff --git a/plugins/store-smb/src/internalClusterTest/java/org/elasticsearch/index/store/smb/AbstractAzureFsTestCase.java b/plugins/store-smb/src/internalClusterTest/java/org/elasticsearch/index/store/smb/AbstractAzureFsTestCase.java index 4a35779a42166..38c07ce918f11 100644 --- a/plugins/store-smb/src/internalClusterTest/java/org/elasticsearch/index/store/smb/AbstractAzureFsTestCase.java +++ b/plugins/store-smb/src/internalClusterTest/java/org/elasticsearch/index/store/smb/AbstractAzureFsTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.store.smb; diff --git a/plugins/store-smb/src/internalClusterTest/java/org/elasticsearch/index/store/smb/SmbMMapFsTests.java b/plugins/store-smb/src/internalClusterTest/java/org/elasticsearch/index/store/smb/SmbMMapFsTests.java index 565354a934272..dec0cc03c7786 100644 --- a/plugins/store-smb/src/internalClusterTest/java/org/elasticsearch/index/store/smb/SmbMMapFsTests.java +++ b/plugins/store-smb/src/internalClusterTest/java/org/elasticsearch/index/store/smb/SmbMMapFsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.store.smb; diff --git a/plugins/store-smb/src/internalClusterTest/java/org/elasticsearch/index/store/smb/SmbNIOFSTests.java b/plugins/store-smb/src/internalClusterTest/java/org/elasticsearch/index/store/smb/SmbNIOFSTests.java index 8468c33686b3b..2b9ff33034eb6 100644 --- a/plugins/store-smb/src/internalClusterTest/java/org/elasticsearch/index/store/smb/SmbNIOFSTests.java +++ b/plugins/store-smb/src/internalClusterTest/java/org/elasticsearch/index/store/smb/SmbNIOFSTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.store.smb; diff --git a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbDirectoryWrapper.java b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbDirectoryWrapper.java index 2e3d6eea69b93..5d3332d042258 100644 --- a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbDirectoryWrapper.java +++ b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbDirectoryWrapper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.store.smb; diff --git a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbMmapFsDirectoryFactory.java b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbMmapFsDirectoryFactory.java index ac95538553b92..4594e8d71c6fb 100644 --- a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbMmapFsDirectoryFactory.java +++ b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbMmapFsDirectoryFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.store.smb; diff --git a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbNIOFSDirectoryFactory.java b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbNIOFSDirectoryFactory.java index 22dc32a47770a..304cd9d732291 100644 --- a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbNIOFSDirectoryFactory.java +++ b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbNIOFSDirectoryFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.store.smb; diff --git a/plugins/store-smb/src/main/java/org/elasticsearch/plugin/store/smb/SMBStorePlugin.java b/plugins/store-smb/src/main/java/org/elasticsearch/plugin/store/smb/SMBStorePlugin.java index f4e1456c5aec4..a8fb5253290bb 100644 --- a/plugins/store-smb/src/main/java/org/elasticsearch/plugin/store/smb/SMBStorePlugin.java +++ b/plugins/store-smb/src/main/java/org/elasticsearch/plugin/store/smb/SMBStorePlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugin.store.smb; diff --git a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/smb/SmbMMapDirectoryTests.java b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/smb/SmbMMapDirectoryTests.java index 361c3e555a512..8e1283ca79833 100644 --- a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/smb/SmbMMapDirectoryTests.java +++ b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/smb/SmbMMapDirectoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.store.smb; diff --git a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/smb/SmbNIOFSDirectoryTests.java b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/smb/SmbNIOFSDirectoryTests.java index 77613a8e0b515..7971188901aa9 100644 --- a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/smb/SmbNIOFSDirectoryTests.java +++ b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/smb/SmbNIOFSDirectoryTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.store.smb; diff --git a/plugins/store-smb/src/yamlRestTest/java/org/elasticsearch/index/store/smb/StoreSmbClientYamlTestSuiteIT.java b/plugins/store-smb/src/yamlRestTest/java/org/elasticsearch/index/store/smb/StoreSmbClientYamlTestSuiteIT.java index d7317ec32034a..afb12c4c79967 100644 --- a/plugins/store-smb/src/yamlRestTest/java/org/elasticsearch/index/store/smb/StoreSmbClientYamlTestSuiteIT.java +++ b/plugins/store-smb/src/yamlRestTest/java/org/elasticsearch/index/store/smb/StoreSmbClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.store.smb; diff --git a/qa/ccs-common-rest/build.gradle b/qa/ccs-common-rest/build.gradle index e5e8c5a489d5b..6121f7dcd4f82 100644 --- a/qa/ccs-common-rest/build.gradle +++ b/qa/ccs-common-rest/build.gradle @@ -10,7 +10,7 @@ apply plugin: 'elasticsearch.internal-yaml-rest-test' restResources { restApi { - include '_common', 'bulk', 'count', 'cluster', 'field_caps', 'get', 'knn_search', 'index', 'indices', 'msearch', + include 'capabilities', '_common', 'bulk', 'count', 'cluster', 'field_caps', 'get', 'knn_search', 'index', 'indices', 'msearch', 'search', 'async_search', 'graph', '*_point_in_time', 'info', 'scroll', 'clear_scroll', 'search_mvt', 'eql', 'sql' } restTests { diff --git a/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java b/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java index 12c15a91be0e3..3a24427df24a3 100644 --- a/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java +++ b/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.test.rest.yaml; @@ -88,7 +89,8 @@ public class CcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase { .setting("xpack.security.enabled", "false") // geohex_grid requires gold license .setting("xpack.license.self_generated.type", "trial") - .feature(FeatureFlag.TIME_SERIES_MODE); + .feature(FeatureFlag.TIME_SERIES_MODE) + .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED); private static ElasticsearchCluster remoteCluster = ElasticsearchCluster.local() .name(REMOTE_CLUSTER_NAME) diff --git a/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java b/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java index d3c8e64b85066..5ada1e941266a 100644 --- a/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java +++ b/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.test.rest.yaml; @@ -90,6 +91,7 @@ public class RcsCcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase { .setting("xpack.security.remote_cluster_server.ssl.enabled", "false") .setting("xpack.security.remote_cluster_client.ssl.enabled", "false") .feature(FeatureFlag.TIME_SERIES_MODE) + .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED) .user("test_admin", "x-pack-test-password"); private static ElasticsearchCluster fulfillingCluster = ElasticsearchCluster.local() diff --git a/qa/ccs-rolling-upgrade-remote-cluster/build.gradle b/qa/ccs-rolling-upgrade-remote-cluster/build.gradle index b63522daa4b4c..ce5b840e6dc91 100644 --- a/qa/ccs-rolling-upgrade-remote-cluster/build.gradle +++ b/qa/ccs-rolling-upgrade-remote-cluster/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.Version @@ -15,7 +16,7 @@ apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.bwc-test' apply plugin: 'elasticsearch.rest-resources' -BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> +buildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> /** * We execute tests 3 times. @@ -51,7 +52,7 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> nonInputProperties.systemProperty('tests.rest.remote_cluster', remoteCluster.map(c -> c.allHttpSocketURI.join(","))) } - onlyIf("FIPS mode disabled") { BuildParams.inFipsJvm == false } + onlyIf("FIPS mode disabled") { buildParams.inFipsJvm == false } } tasks.register("${baseName}#oldClusterTest", StandaloneRestIntegTestTask) { diff --git a/qa/ccs-rolling-upgrade-remote-cluster/src/test/java/org/elasticsearch/upgrades/AggregationsIT.java b/qa/ccs-rolling-upgrade-remote-cluster/src/test/java/org/elasticsearch/upgrades/AggregationsIT.java index f752915d4a371..704bfb4552780 100644 --- a/qa/ccs-rolling-upgrade-remote-cluster/src/test/java/org/elasticsearch/upgrades/AggregationsIT.java +++ b/qa/ccs-rolling-upgrade-remote-cluster/src/test/java/org/elasticsearch/upgrades/AggregationsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; diff --git a/qa/ccs-rolling-upgrade-remote-cluster/src/test/java/org/elasticsearch/upgrades/SearchStatesIT.java b/qa/ccs-rolling-upgrade-remote-cluster/src/test/java/org/elasticsearch/upgrades/SearchStatesIT.java index df50ad785a47a..94e9a82e4a196 100644 --- a/qa/ccs-rolling-upgrade-remote-cluster/src/test/java/org/elasticsearch/upgrades/SearchStatesIT.java +++ b/qa/ccs-rolling-upgrade-remote-cluster/src/test/java/org/elasticsearch/upgrades/SearchStatesIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; diff --git a/qa/ccs-unavailable-clusters/build.gradle b/qa/ccs-unavailable-clusters/build.gradle index 40e583a641aa0..e013ccaf9341e 100644 --- a/qa/ccs-unavailable-clusters/build.gradle +++ b/qa/ccs-unavailable-clusters/build.gradle @@ -1,13 +1,17 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ -apply plugin: 'elasticsearch.legacy-java-rest-test' -testClusters.matching { it.name == "javaRestTest" }.configureEach { - setting 'xpack.security.enabled', 'true' - user username: 'admin', password: 'admin-password', role: 'superuser' +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask + +apply plugin: 'elasticsearch.internal-java-rest-test' + + +tasks.withType(StandaloneRestIntegTestTask).configureEach { + usesDefaultDistribution() } diff --git a/qa/ccs-unavailable-clusters/src/javaRestTest/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java b/qa/ccs-unavailable-clusters/src/javaRestTest/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java index d7ca24cd8c664..780f3994ce627 100644 --- a/qa/ccs-unavailable-clusters/src/javaRestTest/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java +++ b/qa/ccs-unavailable-clusters/src/javaRestTest/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search; @@ -37,6 +38,7 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.test.transport.MockTransportService; @@ -44,6 +46,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonXContent; +import org.junit.ClassRule; import java.io.IOException; import java.util.Collections; @@ -60,6 +63,14 @@ public class CrossClusterSearchUnavailableClusterIT extends ESRestTestCase { private final ThreadPool threadPool = new TestThreadPool(getClass().getName()); + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + @Override public void tearDown() throws Exception { super.tearDown(); @@ -143,7 +154,7 @@ public void testSearchSkipUnavailable() throws IOException { threadPool ) ) { - DiscoveryNode remoteNode = remoteTransport.getLocalDiscoNode(); + DiscoveryNode remoteNode = remoteTransport.getLocalNode(); updateRemoteClusterSettings(Collections.singletonMap("seeds", remoteNode.getAddress().toString())); @@ -296,7 +307,7 @@ public void testSkipUnavailableDependsOnSeeds() throws IOException { threadPool ) ) { - DiscoveryNode remoteNode = remoteTransport.getLocalDiscoNode(); + DiscoveryNode remoteNode = remoteTransport.getLocalNode(); { // check that skip_unavailable alone cannot be set diff --git a/qa/custom-rest-controller/build.gradle b/qa/custom-rest-controller/build.gradle index 5fd5bc9baa335..6a2a72399b99d 100644 --- a/qa/custom-rest-controller/build.gradle +++ b/qa/custom-rest-controller/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.internal-java-rest-test' diff --git a/qa/custom-rest-controller/src/javaRestTest/java/co/elastic/elasticsearch/test/CustomRestPlugin.java b/qa/custom-rest-controller/src/javaRestTest/java/co/elastic/elasticsearch/test/CustomRestPlugin.java index e978e7f2a5c11..9499f0971997a 100644 --- a/qa/custom-rest-controller/src/javaRestTest/java/co/elastic/elasticsearch/test/CustomRestPlugin.java +++ b/qa/custom-rest-controller/src/javaRestTest/java/co/elastic/elasticsearch/test/CustomRestPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package co.elastic.elasticsearch.test; diff --git a/qa/custom-rest-controller/src/javaRestTest/java/org/elasticsearch/plugins/interceptor/CustomRestPluginIT.java b/qa/custom-rest-controller/src/javaRestTest/java/org/elasticsearch/plugins/interceptor/CustomRestPluginIT.java index 45a88fcfa4d5d..6f237f117c334 100644 --- a/qa/custom-rest-controller/src/javaRestTest/java/org/elasticsearch/plugins/interceptor/CustomRestPluginIT.java +++ b/qa/custom-rest-controller/src/javaRestTest/java/org/elasticsearch/plugins/interceptor/CustomRestPluginIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.interceptor; diff --git a/qa/evil-tests/build.gradle b/qa/evil-tests/build.gradle index ef040b002cd39..242fb381e176d 100644 --- a/qa/evil-tests/build.gradle +++ b/qa/evil-tests/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /* diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/ESPolicyUnitTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/ESPolicyUnitTests.java index 86a5e6734c701..2510208a496a9 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/ESPolicyUnitTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/ESPolicyUnitTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.bootstrap; diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilBootstrapChecksTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilBootstrapChecksTests.java index 58bf1760551d4..e7eacca83d850 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilBootstrapChecksTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilBootstrapChecksTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.bootstrap; diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java index acee204f571ba..08d7e3b45702b 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.bootstrap; diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/PolicyUtilTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/PolicyUtilTests.java index 57c79c7d2e411..d8352669083d1 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/PolicyUtilTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/PolicyUtilTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.bootstrap; diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/cli/EvilEnvironmentAwareCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/cli/EvilEnvironmentAwareCommandTests.java index 6e0a248804283..c0c3af8df3e5c 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/cli/EvilEnvironmentAwareCommandTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/cli/EvilEnvironmentAwareCommandTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cli; diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/cluster/metadata/EvilSystemPropertyTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/cluster/metadata/EvilSystemPropertyTests.java index 799752d5399e2..91ee6d584327d 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/cluster/metadata/EvilSystemPropertyTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/cluster/metadata/EvilSystemPropertyTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerConfigurationTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerConfigurationTests.java index db531026dbad5..8d23e3ed4bd10 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerConfigurationTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerConfigurationTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.logging; diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java index 956d3c0e104ae..992bebe57e561 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.logging; diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/common/scheduler/EvilSchedulerEngineTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/common/scheduler/EvilSchedulerEngineTests.java index 7d6f543cb85a8..3276b92b1d716 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/common/scheduler/EvilSchedulerEngineTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/common/scheduler/EvilSchedulerEngineTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.scheduler; diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/common/settings/EvilKeyStoreWrapperTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/common/settings/EvilKeyStoreWrapperTests.java index 46af7f7a55e72..b153cf7ed1c5a 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/common/settings/EvilKeyStoreWrapperTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/common/settings/EvilKeyStoreWrapperTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.settings; diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/env/NodeEnvironmentEvilTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/env/NodeEnvironmentEvilTests.java index 3d94b97ada45e..880fa35710af6 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/env/NodeEnvironmentEvilTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/env/NodeEnvironmentEvilTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.env; diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/index/engine/EvilInternalEngineTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/index/engine/EvilInternalEngineTests.java index 089f3de3bf257..62083ff9811e1 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/index/engine/EvilInternalEngineTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/index/engine/EvilInternalEngineTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.engine; diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/index/store/LuceneFilesExtensionsTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/index/store/LuceneFilesExtensionsTests.java index 25483ec06aa3e..9f547a47305e9 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/index/store/LuceneFilesExtensionsTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/index/store/LuceneFilesExtensionsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.store; diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/monitor/os/EvilOsProbeTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/monitor/os/EvilOsProbeTests.java index 90c3a3e0cabb3..6561734b12a7f 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/monitor/os/EvilOsProbeTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/monitor/os/EvilOsProbeTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.monitor.os; diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/cli/PluginSecurityTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/cli/PluginSecurityTests.java index 26088c0b7fec3..f40c0707db7fd 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/cli/PluginSecurityTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/cli/PluginSecurityTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.cli; diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/threadpool/EvilThreadPoolTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/threadpool/EvilThreadPoolTests.java index 8afc423a6bd74..7b3df344c2190 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/threadpool/EvilThreadPoolTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/threadpool/EvilThreadPoolTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.threadpool; diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/plugins/cli/complex-plugin-security.policy b/qa/evil-tests/src/test/resources/org/elasticsearch/plugins/cli/complex-plugin-security.policy index 47d7d3b044666..da4792e587d05 100644 --- a/qa/evil-tests/src/test/resources/org/elasticsearch/plugins/cli/complex-plugin-security.policy +++ b/qa/evil-tests/src/test/resources/org/elasticsearch/plugins/cli/complex-plugin-security.policy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ grant { diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/plugins/cli/simple-plugin-security.policy b/qa/evil-tests/src/test/resources/org/elasticsearch/plugins/cli/simple-plugin-security.policy index 9c4d366d6aa98..f554bc62d7311 100644 --- a/qa/evil-tests/src/test/resources/org/elasticsearch/plugins/cli/simple-plugin-security.policy +++ b/qa/evil-tests/src/test/resources/org/elasticsearch/plugins/cli/simple-plugin-security.policy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ grant { diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/plugins/cli/unresolved-plugin-security.policy b/qa/evil-tests/src/test/resources/org/elasticsearch/plugins/cli/unresolved-plugin-security.policy index 7ee290a5e1010..fe1c607f2f196 100644 --- a/qa/evil-tests/src/test/resources/org/elasticsearch/plugins/cli/unresolved-plugin-security.policy +++ b/qa/evil-tests/src/test/resources/org/elasticsearch/plugins/cli/unresolved-plugin-security.policy @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ grant { diff --git a/qa/full-cluster-restart/build.gradle b/qa/full-cluster-restart/build.gradle index 2692cf0f0b05f..5e68c4d1ad26b 100644 --- a/qa/full-cluster-restart/build.gradle +++ b/qa/full-cluster-restart/build.gradle @@ -1,19 +1,19 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ -import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask apply plugin: 'elasticsearch.internal-java-rest-test' apply plugin: 'elasticsearch.internal-test-artifact' apply plugin: 'elasticsearch.bwc-test' -BuildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName -> +buildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName -> tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) { usesBwcDistribution(bwcVersion) systemProperty("tests.old_cluster_version", bwcVersion) diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartDownsampleIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartDownsampleIT.java index b171c6e6f0358..7145f4b1d2d84 100644 --- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartDownsampleIT.java +++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartDownsampleIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; @@ -263,7 +264,7 @@ private String getRollupIndexName() throws IOException { if (asMap.size() == 1) { return (String) asMap.keySet().toArray()[0]; } - logger.warn("--> No matching rollup name for path [%s]", endpoint); + logger.warn("--> No matching rollup name for path [{}]", endpoint); return null; } diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 20c13ca92f5c1..53574ad46716f 100644 --- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; @@ -81,6 +82,7 @@ import static org.elasticsearch.transport.RemoteClusterService.REMOTE_CLUSTER_COMPRESS; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -90,6 +92,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.startsWith; /** * Tests to run before and after a full cluster restart. This is run twice, @@ -1284,12 +1287,16 @@ private void checkSnapshot(String snapshotName, int count, String tookOnVersion, assertEquals(singletonList(snapshotName), XContentMapValues.extractValue("snapshots.snapshot", snapResponse)); assertEquals(singletonList("SUCCESS"), XContentMapValues.extractValue("snapshots.state", snapResponse)); // the format can change depending on the ES node version running & this test code running + // and if there's an in-progress release that hasn't been published yet, + // which could affect the top range of the index release version + String firstReleaseVersion = tookOnIndexVersion.toReleaseVersion().split("-")[0]; assertThat( - XContentMapValues.extractValue("snapshots.version", snapResponse), + (Iterable) XContentMapValues.extractValue("snapshots.version", snapResponse), anyOf( - equalTo(List.of(tookOnVersion)), - equalTo(List.of(tookOnIndexVersion.toString())), - equalTo(List.of(tookOnIndexVersion.toReleaseVersion())) + contains(tookOnVersion), + contains(tookOnIndexVersion.toString()), + contains(firstReleaseVersion), + contains(startsWith(firstReleaseVersion + "-")) ) ); diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartTestOrdering.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartTestOrdering.java index 232619ee93bb9..56e379722a1b2 100644 --- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartTestOrdering.java +++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartTestOrdering.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartUpgradeStatus.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartUpgradeStatus.java index 06048d020e2a0..2d735abdaa699 100644 --- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartUpgradeStatus.java +++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartUpgradeStatus.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeFullClusterRestartIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeFullClusterRestartIT.java index 739b4e302bb54..04a50939c8eb1 100644 --- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeFullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeFullClusterRestartIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; @@ -16,7 +17,6 @@ import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.FormatNames; -import org.elasticsearch.test.MapMatcher; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.rest.RestTestLegacyFeatures; @@ -30,9 +30,6 @@ import java.util.Map; import java.util.function.Supplier; -import static org.elasticsearch.test.MapMatcher.assertMap; -import static org.elasticsearch.test.MapMatcher.matchesMap; - public class LogsIndexModeFullClusterRestartIT extends ParameterizedFullClusterRestartTestCase { @ClassRule @@ -171,22 +168,16 @@ public void testLogsIndexing() throws IOException { assertOK(bulkIndexResponse); assertThat(entityAsMap(bulkIndexResponse).get("errors"), Matchers.is(false)); - assertIndexMappingsAndSettings(0, Matchers.nullValue(), matchesMap().extraOk()); - assertIndexMappingsAndSettings( - 1, - Matchers.equalTo("logsdb"), - matchesMap().extraOk().entry("_source", Map.of("mode", "synthetic")) - ); + assertIndexSettings(0, Matchers.nullValue()); + assertIndexSettings(1, Matchers.equalTo("logsdb")); } } - private void assertIndexMappingsAndSettings(int backingIndex, final Matcher indexModeMatcher, final MapMatcher mappingsMatcher) - throws IOException { + private void assertIndexSettings(int backingIndex, final Matcher indexModeMatcher) throws IOException { assertThat( getSettings(client(), getWriteBackingIndex(client(), "logs-apache-production", backingIndex)).get("index.mode"), indexModeMatcher ); - assertMap(getIndexMappingAsMap(getWriteBackingIndex(client(), "logs-apache-production", backingIndex)), mappingsMatcher); } private static Request createDataStream(final String dataStreamName) { diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedFullClusterRestartTestCase.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedFullClusterRestartTestCase.java index c57112d0455c6..ed5ac80604480 100644 --- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedFullClusterRestartTestCase.java +++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedFullClusterRestartTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java index 66c2a5c472ac2..42a0a3d6da055 100644 --- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java +++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; @@ -153,29 +154,29 @@ public QueryBuilderBWCIT(@Name("cluster") FullClusterRestartUpgradeStatus upgrad ); addCandidate( """ - "span_near": {"clauses": [{ "span_term": { "keyword_field": "value1" }}, \ - { "span_term": { "keyword_field": "value2" }}]} + "span_near": {"clauses": [{ "span_term": { "text_field": "value1" }}, \ + { "span_term": { "text_field": "value2" }}]} """, - new SpanNearQueryBuilder(new SpanTermQueryBuilder("keyword_field", "value1"), 0).addClause( - new SpanTermQueryBuilder("keyword_field", "value2") + new SpanNearQueryBuilder(new SpanTermQueryBuilder("text_field", "value1"), 0).addClause( + new SpanTermQueryBuilder("text_field", "value2") ) ); addCandidate( """ - "span_near": {"clauses": [{ "span_term": { "keyword_field": "value1" }}, \ - { "span_term": { "keyword_field": "value2" }}], "slop": 2} + "span_near": {"clauses": [{ "span_term": { "text_field": "value1" }}, \ + { "span_term": { "text_field": "value2" }}], "slop": 2} """, - new SpanNearQueryBuilder(new SpanTermQueryBuilder("keyword_field", "value1"), 2).addClause( - new SpanTermQueryBuilder("keyword_field", "value2") + new SpanNearQueryBuilder(new SpanTermQueryBuilder("text_field", "value1"), 2).addClause( + new SpanTermQueryBuilder("text_field", "value2") ) ); addCandidate( """ - "span_near": {"clauses": [{ "span_term": { "keyword_field": "value1" }}, \ - { "span_term": { "keyword_field": "value2" }}], "slop": 2, "in_order": false} + "span_near": {"clauses": [{ "span_term": { "text_field": "value1" }}, \ + { "span_term": { "text_field": "value2" }}], "slop": 2, "in_order": false} """, - new SpanNearQueryBuilder(new SpanTermQueryBuilder("keyword_field", "value1"), 2).addClause( - new SpanTermQueryBuilder("keyword_field", "value2") + new SpanNearQueryBuilder(new SpanTermQueryBuilder("text_field", "value1"), 2).addClause( + new SpanTermQueryBuilder("text_field", "value2") ).inOrder(false) ); } @@ -203,11 +204,6 @@ public void testQueryBuilderBWC() throws Exception { mappingsAndSettings.field("type", "percolator"); mappingsAndSettings.endObject(); } - { - mappingsAndSettings.startObject("keyword_field"); - mappingsAndSettings.field("type", "keyword"); - mappingsAndSettings.endObject(); - } { mappingsAndSettings.startObject("text_field"); mappingsAndSettings.field("type", "text"); diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/UpgradeWithOldIndexSettingsIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/UpgradeWithOldIndexSettingsIT.java index 3a2200f152768..acef190d39c16 100644 --- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/UpgradeWithOldIndexSettingsIT.java +++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/UpgradeWithOldIndexSettingsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; diff --git a/qa/logging-config/build.gradle b/qa/logging-config/build.gradle index a5d0ddf842253..255a513543c92 100644 --- a/qa/logging-config/build.gradle +++ b/qa/logging-config/build.gradle @@ -1,39 +1,20 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.build' -apply plugin: 'elasticsearch.legacy-java-rest-test' +apply plugin: 'elasticsearch.internal-java-rest-test' dependencies { - testImplementation project(":libs:elasticsearch-x-content") + testImplementation project(":libs:x-content") testImplementation project(":test:framework") } -testClusters.all { - setting 'xpack.security.enabled', 'false' -} - -testClusters.matching { it.name == "javaRestTest" }.configureEach { - /** - * Provide a custom log4j configuration where layout is an old style pattern and confirm that Elasticsearch - * can successfully startup. - */ - extraConfigFile 'log4j2.properties', file('es-v7-log4j2.properties') -} - -tasks.named("javaRestTest").configure { - nonInputProperties.systemProperty 'tests.logfile', - "${-> testClusters.javaRestTest.singleNode().getServerLog().absolutePath.replaceAll("_server.json", ".log")}" - - nonInputProperties.systemProperty 'tests.jsonLogfile', - "${-> testClusters.javaRestTest.singleNode().getServerLog()}" -} - tasks.named("test").configure { systemProperty 'tests.security.manager', 'false' } diff --git a/qa/logging-config/src/javaRestTest/java/org/elasticsearch/common/logging/CustomLoggingConfigIT.java b/qa/logging-config/src/javaRestTest/java/org/elasticsearch/common/logging/CustomLoggingConfigIT.java index 4ec12ed135d65..085fad8bcdd7f 100644 --- a/qa/logging-config/src/javaRestTest/java/org/elasticsearch/common/logging/CustomLoggingConfigIT.java +++ b/qa/logging-config/src/javaRestTest/java/org/elasticsearch/common/logging/CustomLoggingConfigIT.java @@ -1,24 +1,26 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.logging; -import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.LogType; +import org.elasticsearch.test.cluster.util.resource.Resource; import org.elasticsearch.test.rest.ESRestTestCase; import org.hamcrest.Matchers; +import org.junit.ClassRule; +import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.security.AccessController; -import java.security.PrivilegedAction; import java.util.List; import static org.hamcrest.Matchers.matchesRegex; @@ -33,53 +35,44 @@ public class CustomLoggingConfigIT extends ESRestTestCase { // [2020-03-20T14:51:59,989][INFO ][o.e.g.GatewayService ] [integTest-0] recovered [0] indices into cluster_state private static final String NODE_STARTED = ".*recovered.*cluster_state.*"; + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .setting("xpack.security.enabled", "false") + .configFile("log4j2.properties", Resource.fromClasspath("es-v7-log4j2.properties")) + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + public void testSuccessfulStartupWithCustomConfig() throws Exception { assertBusy(() -> { - List lines = readAllLines(getPlaintextLogFile()); + List lines = getPlaintextLog(); assertThat(lines, Matchers.hasItem(matchesRegex(NODE_STARTED))); }); } public void testParseAllV7JsonLines() throws Exception { assertBusy(() -> { - List lines = readAllLines(getJSONLogFile()); + List lines = getJSONLog(); assertThat(lines, Matchers.hasItem(matchesRegex(NODE_STARTED))); }); } - private List readAllLines(Path logFile) { - return AccessController.doPrivileged((PrivilegedAction>) () -> { - try { - return Files.readAllLines(logFile, StandardCharsets.UTF_8); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }); - } - - @SuppressForbidden(reason = "PathUtils doesn't have permission to read this file") - private Path getJSONLogFile() { - String logFileString = System.getProperty("tests.logfile"); - if (logFileString == null) { - fail( - "tests.logfile must be set to run this test. It is automatically " - + "set by gradle. If you must set it yourself then it should be the absolute path to the " - + "log file." - ); + private List getJSONLog() { + try (InputStream nodeLog = cluster.getNodeLog(0, LogType.SERVER_JSON)) { + return new BufferedReader(new InputStreamReader(nodeLog, StandardCharsets.UTF_8)).lines().toList(); + } catch (IOException e) { + throw new UncheckedIOException(e); } - return Paths.get(logFileString); } - @SuppressForbidden(reason = "PathUtils doesn't have permission to read this file") - private Path getPlaintextLogFile() { - String logFileString = System.getProperty("tests.logfile"); - if (logFileString == null) { - fail( - "tests.logfile must be set to run this test. It is automatically " - + "set by gradle. If you must set it yourself then it should be the absolute path to the " - + "log file." - ); + private List getPlaintextLog() { + try (InputStream nodeLog = cluster.getNodeLog(0, LogType.SERVER)) { + return new BufferedReader(new InputStreamReader(nodeLog, StandardCharsets.UTF_8)).lines().toList(); + } catch (IOException e) { + throw new UncheckedIOException(e); } - return Paths.get(logFileString); } } diff --git a/qa/logging-config/src/javaRestTest/java/org/elasticsearch/common/logging/ESJsonLogsConfigIT.java b/qa/logging-config/src/javaRestTest/java/org/elasticsearch/common/logging/ESJsonLogsConfigIT.java index 1f838d01517bb..fa3da02ffbf40 100644 --- a/qa/logging-config/src/javaRestTest/java/org/elasticsearch/common/logging/ESJsonLogsConfigIT.java +++ b/qa/logging-config/src/javaRestTest/java/org/elasticsearch/common/logging/ESJsonLogsConfigIT.java @@ -1,55 +1,52 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.logging; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.LogType; +import org.elasticsearch.test.cluster.util.resource.Resource; import org.elasticsearch.xcontent.ObjectParser; import org.hamcrest.Matcher; +import org.junit.ClassRule; -import java.io.BufferedReader; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.security.AccessController; -import java.security.PrivilegedAction; +import java.io.InputStream; import static org.hamcrest.Matchers.is; /** * Test to verify ES JSON log format. Used in ES v7. Some users might decide to keep that format. + * Provide a custom log4j configuration where layout is an old style pattern and confirm that Elasticsearch can successfully startup. */ public class ESJsonLogsConfigIT extends JsonLogsIntegTestCase { + + private static final String NODE_NAME = "test-node-0"; + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .setting("xpack.security.enabled", "false") + .withNode(localNodeSpecBuilder -> localNodeSpecBuilder.name(NODE_NAME)) + .configFile("log4j2.properties", Resource.fromClasspath("es-v7-log4j2.properties")) + .build(); + @Override - protected Matcher nodeNameMatcher() { - return is("javaRestTest-0"); + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); } @Override - protected BufferedReader openReader(Path logFile) { - assumeFalse( - "Skipping test because it is being run against an external cluster.", - logFile.getFileName().toString().equals("--external--") - ); - - return AccessController.doPrivileged((PrivilegedAction) () -> { - try { - String temp = Files.readString(logFile); - - return Files.newBufferedReader(logFile, StandardCharsets.UTF_8); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); + protected InputStream openLogsStream() { + return cluster.getNodeLog(0, LogType.SERVER_JSON); } @Override - protected String getLogFileName() { - return System.getProperty("tests.jsonLogfile"); + protected Matcher nodeNameMatcher() { + return is(NODE_NAME); } @Override diff --git a/qa/logging-config/es-v7-log4j2.properties b/qa/logging-config/src/javaRestTest/resources/es-v7-log4j2.properties similarity index 96% rename from qa/logging-config/es-v7-log4j2.properties rename to qa/logging-config/src/javaRestTest/resources/es-v7-log4j2.properties index 3e7719ecf9bcd..e83a9636b631e 100644 --- a/qa/logging-config/es-v7-log4j2.properties +++ b/qa/logging-config/src/javaRestTest/resources/es-v7-log4j2.properties @@ -1,9 +1,10 @@ # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". # status = error diff --git a/qa/logging-config/src/test/java/org/elasticsearch/common/logging/ESJsonLayoutTests.java b/qa/logging-config/src/test/java/org/elasticsearch/common/logging/ESJsonLayoutTests.java index 9f0b94a914616..8ad1c63290a65 100644 --- a/qa/logging-config/src/test/java/org/elasticsearch/common/logging/ESJsonLayoutTests.java +++ b/qa/logging-config/src/test/java/org/elasticsearch/common/logging/ESJsonLayoutTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.logging; diff --git a/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java b/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java index 8d518051a9583..1066bf1360e41 100644 --- a/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java +++ b/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.logging; diff --git a/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLogsTestSetup.java b/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLogsTestSetup.java index e28fef91168e9..8c710cb4bc5b0 100644 --- a/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLogsTestSetup.java +++ b/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLogsTestSetup.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.logging; diff --git a/qa/logging-spi/build.gradle b/qa/logging-spi/build.gradle index 04f09b9638bfe..d6b707eb24dd5 100644 --- a/qa/logging-spi/build.gradle +++ b/qa/logging-spi/build.gradle @@ -3,9 +3,9 @@ apply plugin: 'elasticsearch.base-internal-es-plugin' apply plugin: 'elasticsearch.internal-java-rest-test' esplugin { - name 'logging-spi-test' - description 'An test plugin to test the SPI behaviour of ES logging' - classname 'org.elasticsearch.test.logging.plugin.TestLoggingPlugin' + name = 'logging-spi-test' + description = 'An test plugin to test the SPI behaviour of ES logging' + classname ='org.elasticsearch.test.logging.plugin.TestLoggingPlugin' } dependencies { diff --git a/qa/logging-spi/src/javaRestTest/java/org/elasticsearch/common/logging/DynamicContextDataProviderIT.java b/qa/logging-spi/src/javaRestTest/java/org/elasticsearch/common/logging/DynamicContextDataProviderIT.java index 2fb2dca823c57..e8c12a8703da5 100644 --- a/qa/logging-spi/src/javaRestTest/java/org/elasticsearch/common/logging/DynamicContextDataProviderIT.java +++ b/qa/logging-spi/src/javaRestTest/java/org/elasticsearch/common/logging/DynamicContextDataProviderIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.logging; diff --git a/qa/logging-spi/src/main/java/module-info.java b/qa/logging-spi/src/main/java/module-info.java index f2e050d943f1c..e65792cb6c35d 100644 --- a/qa/logging-spi/src/main/java/module-info.java +++ b/qa/logging-spi/src/main/java/module-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ module org.elasticsearch.internal.security { diff --git a/qa/logging-spi/src/main/java/org/elasticsearch/test/logging/plugin/CustomDataProvider.java b/qa/logging-spi/src/main/java/org/elasticsearch/test/logging/plugin/CustomDataProvider.java index 6a1e6a4108d2d..96d50b3ec6e0e 100644 --- a/qa/logging-spi/src/main/java/org/elasticsearch/test/logging/plugin/CustomDataProvider.java +++ b/qa/logging-spi/src/main/java/org/elasticsearch/test/logging/plugin/CustomDataProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.test.logging.plugin; diff --git a/qa/logging-spi/src/main/java/org/elasticsearch/test/logging/plugin/TestLoggingPlugin.java b/qa/logging-spi/src/main/java/org/elasticsearch/test/logging/plugin/TestLoggingPlugin.java index a0e1c59a22fd1..efef804a69c7c 100644 --- a/qa/logging-spi/src/main/java/org/elasticsearch/test/logging/plugin/TestLoggingPlugin.java +++ b/qa/logging-spi/src/main/java/org/elasticsearch/test/logging/plugin/TestLoggingPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.test.logging.plugin; diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle index 3953237a0e8d9..4123b337ae260 100644 --- a/qa/mixed-cluster/build.gradle +++ b/qa/mixed-cluster/build.gradle @@ -1,16 +1,20 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.VersionProperties -import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask +import org.elasticsearch.gradle.testclusters.TestClusterValueSource +import org.elasticsearch.gradle.testclusters.TestClustersRegistry +import org.elasticsearch.gradle.util.GradleUtils +import org.elasticsearch.gradle.testclusters.TestClustersPlugin apply plugin: 'elasticsearch.internal-testclusters' apply plugin: 'elasticsearch.standalone-rest-test' @@ -57,8 +61,15 @@ excludeList.add('cluster.desired_nodes/11_old_format/Test node version must have excludeList.add('cluster.desired_nodes/11_old_format/Test node version can not be null') excludeList.add('cluster.desired_nodes/20_dry_run/Test validation works for dry run updates') -BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> +// Excluded because they create dot-prefixed indices on older versions +excludeList.add('indices.resolve_index/20_resolve_system_index/*') +// Can't work until auto-expand replicas is 0-1 for synonyms index +excludeList.add("synonyms/90_synonyms_reloading_for_synset/Reload analyzers for specific synonym set") + +def clusterPath = getPath() + +buildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> if (bwcVersion != VersionProperties.getElasticsearchVersion()) { /* This project runs the core REST tests against a 4 node cluster where two of the nodes has a different minor. */ @@ -67,6 +78,7 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> numberOfNodes = 4 setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" setting 'xpack.security.enabled', 'false' + setting "xpack.license.self_generated.type", "trial" /* There is a chance we have more master changes than "normal", so to avoid this test from failing, we increase the threshold (as this purpose of this test isn't to test that specific indicator). */ if (bwcVersion.onOrAfter(Version.fromString("8.4.0"))) { @@ -78,18 +90,42 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> tasks.register("${baseName}#mixedClusterTest", StandaloneRestIntegTestTask) { useCluster baseCluster mustRunAfter("precommit") + Provider serviceProvider = GradleUtils.getBuildService( + project.gradle.sharedServices, + TestClustersPlugin.REGISTRY_SERVICE_NAME + ) + + def baseInfo = project.getProviders().of(TestClusterValueSource.class) { + it.parameters.path.set(clusterPath) + it.parameters.clusterName.set(baseName) + it.parameters.service = serviceProvider + }.map { it.getAllHttpSocketURI() } + + def baseInfoAfterOneNodeUpdate = project.getProviders().of(TestClusterValueSource.class) { + it.parameters.path.set(clusterPath) + it.parameters.clusterName.set(baseName) + it.parameters.service = serviceProvider + }.map { it.getAllHttpSocketURI() } + + def baseInfoAfterTwoNodesUpdate = project.getProviders().of(TestClusterValueSource.class) { + it.parameters.path.set(clusterPath) + it.parameters.clusterName.set(baseName) + it.parameters.service = serviceProvider + }.map { it.getAllHttpSocketURI() } + def nonInputProps = nonInputProperties + def sharedRepoFolder = new File(buildDir, "cluster/shared/repo/${baseName}") doFirst { - delete("${buildDir}/cluster/shared/repo/${baseName}") + delete(sharedRepoFolder) // Getting the endpoints causes a wait for the cluster - println "Test cluster endpoints are: ${-> baseCluster.get().allHttpSocketURI.join(",")}" + println "Test cluster endpoints are: ${-> baseInfo.get().join(",")}" println "Upgrading one node to create a mixed cluster" baseCluster.get().nextNodeToNextVersion() // Getting the endpoints causes a wait for the cluster - println "Upgrade complete, endpoints are: ${-> baseCluster.get().allHttpSocketURI.join(",")}" + println "Upgrade complete, endpoints are: ${-> baseInfoAfterOneNodeUpdate.get().join(",")}" println "Upgrading another node to create a mixed cluster" baseCluster.get().nextNodeToNextVersion() - nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(","))) - nonInputProperties.systemProperty('tests.clustername', baseName) + nonInputProps.systemProperty('tests.rest.cluster', baseInfoAfterTwoNodesUpdate.map(c -> c.join(","))) + nonInputProps.systemProperty('tests.clustername', baseName) if (excludeList.isEmpty() == false) { systemProperty 'tests.rest.blacklist', excludeList.join(',') } @@ -97,7 +133,7 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> systemProperty 'tests.path.repo', "${buildDir}/cluster/shared/repo/${baseName}" systemProperty 'tests.bwc_nodes_version', bwcVersion.toString().replace('-SNAPSHOT', '') systemProperty 'tests.new_nodes_version', project.version.toString().replace('-SNAPSHOT', '') - onlyIf("BWC tests disabled") { project.bwc_tests_enabled } +// onlyIf("BWC tests disabled") { project.bwc_tests_enabled } } tasks.register(bwcTaskName(bwcVersion)) { diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/HotThreadsIT.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/HotThreadsIT.java index 04c59e1ce9214..9df522f4d9111 100644 --- a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/HotThreadsIT.java +++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/HotThreadsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.backwards; diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java index 825a866cdf2f8..dc3e527d656cb 100644 --- a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java +++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.backwards; diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterClientYamlTestSuiteIT.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterClientYamlTestSuiteIT.java index f7caf4805be15..a61d773bdba0f 100644 --- a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterClientYamlTestSuiteIT.java +++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.backwards; diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterTestNode.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterTestNode.java index c9cdf9dd41735..293493603c185 100644 --- a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterTestNode.java +++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterTestNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.backwards; diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterTestNodes.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterTestNodes.java index d4bfea19fd1ff..778d5a40bec44 100644 --- a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterTestNodes.java +++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterTestNodes.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.backwards; diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/RareTermsIT.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/RareTermsIT.java index f2ca41e5ef8bc..2ee769b54a09d 100644 --- a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/RareTermsIT.java +++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/RareTermsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.backwards; diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/SearchWithMinCompatibleSearchNodeIT.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/SearchWithMinCompatibleSearchNodeIT.java index 808ebb764768f..a391ee5a3bd7b 100644 --- a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/SearchWithMinCompatibleSearchNodeIT.java +++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/SearchWithMinCompatibleSearchNodeIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.backwards; diff --git a/qa/multi-cluster-search/build.gradle b/qa/multi-cluster-search/build.gradle index d0cbc208f4d8e..906a49134bb51 100644 --- a/qa/multi-cluster-search/build.gradle +++ b/qa/multi-cluster-search/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /* @@ -34,7 +35,7 @@ def ccsSupportedVersion = bwcVersion -> { return currentVersion.minor == 0 || (currentVersion.major == bwcVersion.major && currentVersion.minor - bwcVersion.minor <= 1) } -BuildParams.bwcVersions.withWireCompatible(ccsSupportedVersion) { bwcVersion, baseName -> +buildParams.bwcVersions.withWireCompatible(ccsSupportedVersion) { bwcVersion, baseName -> def remoteCluster = testClusters.register("${baseName}-remote") { numberOfNodes = 2 diff --git a/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java b/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java index b1d300134120b..79cdc1047aec9 100644 --- a/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java +++ b/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search; @@ -42,6 +43,7 @@ import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; +import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; @@ -579,13 +581,14 @@ public void testSortByField() throws Exception { public void testSortByFieldOneClusterHasNoResults() throws Exception { assumeMultiClusterSetup(); - // set to a value greater than the number of shards to avoid differences due to the skipping of shards + // setting aggs to avoid differences due to the skipping of shards when matching none SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); boolean onlyRemote = randomBoolean(); sourceBuilder.query(new TermQueryBuilder("_index", onlyRemote ? REMOTE_INDEX_NAME : INDEX_NAME)); sourceBuilder.sort("type.keyword", SortOrder.ASC); sourceBuilder.sort("creationDate", SortOrder.DESC); sourceBuilder.sort("user.keyword", SortOrder.ASC); + sourceBuilder.aggregation(AggregationBuilders.max("max").field("creationDate")); CheckedConsumer responseChecker = response -> { assertHits(response); int size = response.evaluateArraySize("hits.hits"); diff --git a/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/MultiClusterSearchYamlTestSuiteIT.java b/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/MultiClusterSearchYamlTestSuiteIT.java index 0ebd36ec50f1a..47bac5c51afa1 100644 --- a/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/MultiClusterSearchYamlTestSuiteIT.java +++ b/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/MultiClusterSearchYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search; diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/70_skip_shards.yml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/70_skip_shards.yml index 92ae11c712b25..f392ae6d09413 100644 --- a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/70_skip_shards.yml +++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/70_skip_shards.yml @@ -166,8 +166,7 @@ - match: { hits.total.value: 0 } - match: { _shards.total: 2 } - match: { _shards.successful: 2 } - # When all shards are skipped current logic returns 1 to produce a valid search result - - match: { _shards.skipped : 1} + - match: { _shards.skipped : 2} - match: { _shards.failed: 0 } # check that skipped when we don't match the alias with a terms query @@ -183,8 +182,7 @@ - match: { hits.total.value: 0 } - match: { _shards.total: 2 } - match: { _shards.successful: 2 } - # When all shards are skipped current logic returns 1 to produce a valid search result - - match: { _shards.skipped : 1} + - match: { _shards.skipped : 2} - match: { _shards.failed: 0 } # check that skipped when we don't match the alias with a prefix query @@ -200,8 +198,7 @@ - match: { hits.total.value: 0 } - match: { _shards.total: 2 } - match: { _shards.successful: 2 } - # When all shards are skipped current logic returns 1 to produce a valid search result - - match: { _shards.skipped : 1} + - match: { _shards.skipped : 2} - match: { _shards.failed: 0 } # check that skipped when we don't match the alias with a wildcard query @@ -217,7 +214,6 @@ - match: { hits.total.value: 0 } - match: { _shards.total: 2 } - match: { _shards.successful: 2 } - # When all shards are skipped current logic returns 1 to produce a valid search result - - match: { _shards.skipped : 1} + - match: { _shards.skipped : 2} - match: { _shards.failed: 0 } diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/90_index_name_query.yml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/90_index_name_query.yml index a60a1b0d812ee..be2ce033b123c 100644 --- a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/90_index_name_query.yml +++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/90_index_name_query.yml @@ -81,7 +81,7 @@ teardown: - match: { hits.total.value: 0 } - match: { _shards.total: 2 } - match: { _shards.successful: 2 } - - match: { _shards.skipped : 1} + - match: { _shards.skipped : 2} - match: { _shards.failed: 0 } - do: @@ -98,5 +98,5 @@ teardown: - match: { hits.total.value: 0 } - match: { _shards.total: 2 } - match: { _shards.successful: 2 } - - match: { _shards.skipped : 1} + - match: { _shards.skipped : 2} - match: { _shards.failed: 0 } diff --git a/qa/no-bootstrap-tests/build.gradle b/qa/no-bootstrap-tests/build.gradle index bc3a735fcc14a..d3a8eebcc0ae9 100644 --- a/qa/no-bootstrap-tests/build.gradle +++ b/qa/no-bootstrap-tests/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /* diff --git a/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java b/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java index 6369e02e1f605..d47ba685b9834 100644 --- a/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java +++ b/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.bootstrap; @@ -12,7 +13,6 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.core.LogEvent; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.lucene.util.Constants; import org.elasticsearch.Version; @@ -24,6 +24,7 @@ import org.elasticsearch.plugins.Platforms; import org.elasticsearch.plugins.PluginTestUtil; import org.elasticsearch.test.GraalVMThreadsFilter; +import org.elasticsearch.test.JnaCleanerThreadsFilter; import org.elasticsearch.test.MockLog; import java.io.IOException; @@ -35,7 +36,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.function.Function; @@ -51,7 +51,7 @@ * that prevents the Spawner class from doing its job. Also needs to run in a separate JVM to other * tests that extend ESTestCase for the same reason. */ -@ThreadLeakFilters(filters = { GraalVMThreadsFilter.class }) +@ThreadLeakFilters(filters = { GraalVMThreadsFilter.class, JnaCleanerThreadsFilter.class }) public class SpawnerNoBootstrapTests extends LuceneTestCase { private static final String CONTROLLER_SOURCE = """ @@ -69,34 +69,6 @@ public class SpawnerNoBootstrapTests extends LuceneTestCase { MockLog.init(); } - static class ExpectedStreamMessage implements MockLog.LoggingExpectation { - final String expectedLogger; - final String expectedMessage; - final CountDownLatch matched; - volatile boolean saw; - - ExpectedStreamMessage(String logger, String message, CountDownLatch matched) { - this.expectedLogger = logger; - this.expectedMessage = message; - this.matched = matched; - } - - @Override - public void match(LogEvent event) { - if (event.getLoggerName().equals(expectedLogger) - && event.getLevel().equals(Level.WARN) - && event.getMessage().getFormattedMessage().equals(expectedMessage)) { - saw = true; - matched.countDown(); - } - } - - @Override - public void assertMatched() { - assertTrue("Expected to see message [" + expectedMessage + "] on logger [" + expectedLogger + "]", saw); - } - } - /** * Simplest case: a module with no controller daemon. */ @@ -208,32 +180,32 @@ private void assertControllerSpawns(final Function pluginsDir String stderrLoggerName = "test_plugin-controller-stderr"; Loggers.setLevel(LogManager.getLogger(stdoutLoggerName), Level.TRACE); Loggers.setLevel(LogManager.getLogger(stderrLoggerName), Level.TRACE); - CountDownLatch messagesLoggedLatch = new CountDownLatch(2); - try (var mockLog = MockLog.capture(stdoutLoggerName, stderrLoggerName)) { - if (expectSpawn) { - mockLog.addExpectation(new ExpectedStreamMessage(stdoutLoggerName, "I am alive", messagesLoggedLatch)); - mockLog.addExpectation(new ExpectedStreamMessage(stderrLoggerName, "I am an error", messagesLoggedLatch)); + if (expectSpawn) { + final Process process; + try (var mockLog = MockLog.capture(stdoutLoggerName, stderrLoggerName)) { + mockLog.addExpectation(new MockLog.SeenEventExpectation("stdout", stdoutLoggerName, Level.WARN, "I am alive")); + mockLog.addExpectation(new MockLog.SeenEventExpectation("stderr", stderrLoggerName, Level.WARN, "I am an error")); + + try (var spawner = new Spawner()) { + spawner.spawnNativeControllers(environment); + List processes = spawner.getProcesses(); + + // as there should only be a reference in the list for the module that had the controller daemon, we expect one here + assertThat(processes, hasSize(1)); + process = processes.get(0); + // fail if we don't get the expected log messages soonish + mockLog.awaitAllExpectationsMatched(); + } } - Spawner spawner = new Spawner(); - spawner.spawnNativeControllers(environment); - - List processes = spawner.getProcesses(); - - if (expectSpawn) { - // as there should only be a reference in the list for the module that had the controller daemon, we expect one here - assertThat(processes, hasSize(1)); - Process process = processes.get(0); - // fail if we don't get the expected log messages within one second; usually it will be even quicker - assertTrue(messagesLoggedLatch.await(1, TimeUnit.SECONDS)); - spawner.close(); - // fail if the process does not die within one second; usually it will be even quicker but it depends on OS scheduling - assertTrue(process.waitFor(1, TimeUnit.SECONDS)); - } else { - assertThat(processes, is(empty())); + // fail if the process does not die within one second; usually it will be even quicker but it depends on OS scheduling + assertTrue(process.waitFor(1, TimeUnit.SECONDS)); + } else { + try (var spawner = new Spawner()) { + spawner.spawnNativeControllers(environment); + assertThat(spawner.getProcesses(), is(empty())); } - mockLog.assertAllExpectationsMatched(); } } diff --git a/qa/packaging/build.gradle b/qa/packaging/build.gradle index 758dfe6661766..f9a903223c88a 100644 --- a/qa/packaging/build.gradle +++ b/qa/packaging/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ plugins { @@ -12,7 +13,7 @@ plugins { dependencies { testImplementation project(':server') - testImplementation project(':libs:elasticsearch-core') + testImplementation project(':libs:core') testImplementation(testArtifact(project(':x-pack:plugin:core'))) testImplementation "junit:junit:${versions.junit}" testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}" diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/ArchiveGenerateInitialCredentialsTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/ArchiveGenerateInitialCredentialsTests.java index 169c8f0c0aca3..f155b05d9552e 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/ArchiveGenerateInitialCredentialsTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/ArchiveGenerateInitialCredentialsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.test; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java index 07012eee9ce44..64fbc6176a31c 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.test; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/BootstrapCheckTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/BootstrapCheckTests.java index 28f9fdb577dc9..9d4eb66d0ee49 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/BootstrapCheckTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/BootstrapCheckTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.test; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/CertGenCliTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/CertGenCliTests.java index 03b820de58668..9f2abf2ef957a 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/CertGenCliTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/CertGenCliTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.test; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/ConfigurationTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/ConfigurationTests.java index 2ce9eef29d903..007a26822649a 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/ConfigurationTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/ConfigurationTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.test; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/CronEvalCliTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/CronEvalCliTests.java index 3f5090d42dcb4..60f4e5908dfb4 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/CronEvalCliTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/CronEvalCliTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.test; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DebMetadataTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DebMetadataTests.java index 3eff33982eed8..9f9aa78a4910b 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DebMetadataTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DebMetadataTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.test; @@ -11,18 +12,31 @@ import junit.framework.TestCase; import org.elasticsearch.packaging.util.Distribution; -import org.elasticsearch.packaging.util.FileUtils; +import org.elasticsearch.packaging.util.LintianResultParser; +import org.elasticsearch.packaging.util.LintianResultParser.Issue; +import org.elasticsearch.packaging.util.LintianResultParser.Result; import org.elasticsearch.packaging.util.Shell; import org.junit.BeforeClass; +import java.util.List; import java.util.Locale; import java.util.regex.Pattern; +import java.util.stream.Collectors; import static org.elasticsearch.packaging.util.FileUtils.getDistributionFile; import static org.junit.Assume.assumeTrue; public class DebMetadataTests extends PackagingTestCase { + private final LintianResultParser lintianParser = new LintianResultParser(); + private static final List IGNORED_TAGS = List.of( + // Override syntax changes between lintian versions in a non-backwards compatible way, so we have to tolerate these. + // Tag mismatched-override is a non-erasable tag which cannot be ignored with overrides, so we handle it here. + "mismatched-override", + // systemd-service-file-outside-lib has been incorrect and removed in the newer version on Lintian + "systemd-service-file-outside-lib" + ); + @BeforeClass public static void filterDistros() { assumeTrue("only deb", distribution.packaging == Distribution.Packaging.DEB); @@ -34,15 +48,26 @@ public void test05CheckLintian() { if (helpText.contains("--fail-on-warnings")) { extraArgs = "--fail-on-warnings"; } else if (helpText.contains("--fail-on error")) { - extraArgs = "--fail-on warning"; - // Recent lintian versions are picky about malformed or mismatched overrides. - // Unfortunately override syntax changes between lintian versions in a non-backwards compatible - // way, so we have to tolerate these (or maintain separate override files per lintian version). - if (helpText.contains("--suppress-tags")) { - extraArgs += " --suppress-tags malformed-override,mismatched-override"; + extraArgs = "--fail-on error,warning"; + } + Shell.Result result = sh.runIgnoreExitCode( + String.format(Locale.ROOT, "lintian %s %s", extraArgs, getDistributionFile(distribution())) + ); + Result lintianResult = lintianParser.parse(result.stdout()); + // Unfortunately Lintian overrides syntax changes between Lintian versions in a non-backwards compatible + // way, so we have to manage some exclusions outside the overrides file. + if (lintianResult.isSuccess() == false) { + List importantIssues = lintianResult.issues() + .stream() + .filter(issue -> IGNORED_TAGS.contains(issue.tag()) == false) + .toList(); + if (importantIssues.isEmpty() == false) { + fail( + "Issues for DEB package found by Lintian:\n" + + importantIssues.stream().map(Record::toString).collect(Collectors.joining("\n")) + ); } } - sh.run(String.format(Locale.ROOT, "lintian %s %s", extraArgs, FileUtils.getDistributionFile(distribution()))); } public void test06Dependencies() { diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DebPreservationTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DebPreservationTests.java index 0901a20c984e1..30a6f5ca4db31 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DebPreservationTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DebPreservationTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.test; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java index 18668b842b2d3..8cb8354eb5d71 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.test; @@ -98,6 +99,7 @@ *
  • The default image with a custom, small base image
  • *
  • A UBI-based image
  • *
  • Another UBI image for Iron Bank
  • + *
  • A WOLFI-based image
  • *
  • Images for Cloud
  • * */ @@ -167,10 +169,7 @@ public void test012SecurityCanBeDisabled() throws Exception { * Checks that no plugins are initially active. */ public void test020PluginsListWithNoPlugins() { - assumeTrue( - "Only applies to non-Cloud images", - distribution.packaging != Packaging.DOCKER_CLOUD && distribution().packaging != Packaging.DOCKER_CLOUD_ESS - ); + assumeTrue("Only applies to non-Cloud images", distribution().packaging != Packaging.DOCKER_CLOUD_ESS); final Installation.Executables bin = installation.executables(); final Result r = sh.run(bin.pluginTool + " list"); @@ -205,7 +204,9 @@ public void test022InstallPluginsFromLocalArchive() { final String plugin = "analysis-icu"; final Installation.Executables bin = installation.executables(); + listPluginArchive().forEach(System.out::println); assertThat("Expected " + plugin + " to not be installed", listPlugins(), not(hasItems(plugin))); + assertThat("Expected " + plugin + " available in archive", listPluginArchive(), hasSize(16)); // Stuff the proxy settings with garbage, so any attempt to go out to the internet would fail sh.getEnv() @@ -385,6 +386,9 @@ public void test040JavaUsesTheOsProvidedKeystore() { if (distribution.packaging == Packaging.DOCKER_UBI || distribution.packaging == Packaging.DOCKER_IRON_BANK) { // In these images, the `cacerts` file ought to be a symlink here assertThat(path, equalTo("/etc/pki/ca-trust/extracted/java/cacerts")); + } else if (distribution.packaging == Packaging.DOCKER_WOLFI || distribution.packaging == Packaging.DOCKER_CLOUD_ESS) { + // In these images, the `cacerts` file ought to be a symlink here + assertThat(path, equalTo("/etc/ssl/certs/java/cacerts")); } else { // Whereas on other images, it's a real file so the real path is the same assertThat(path, equalTo("/usr/share/elasticsearch/jdk/lib/security/cacerts")); @@ -1109,8 +1113,8 @@ public void test170DefaultShellIsBash() { */ public void test171AdditionalCliOptionsAreForwarded() throws Exception { assumeTrue( - "Does not apply to Cloud images, because they don't use the default entrypoint", - distribution.packaging != Packaging.DOCKER_CLOUD && distribution().packaging != Packaging.DOCKER_CLOUD_ESS + "Does not apply to Cloud ESS images, because they don't use the default entrypoint", + distribution().packaging != Packaging.DOCKER_CLOUD_ESS ); runContainer(distribution(), builder().runArgs("bin/elasticsearch", "-Ecluster.name=kimchy").envVar("ELASTIC_PASSWORD", PASSWORD)); @@ -1197,7 +1201,7 @@ public void test310IronBankImageHasNoAdditionalLabels() throws Exception { * Check that the Cloud image contains the required Beats */ public void test400CloudImageBundlesBeats() { - assumeTrue(distribution.packaging == Packaging.DOCKER_CLOUD || distribution.packaging == Packaging.DOCKER_CLOUD_ESS); + assumeTrue(distribution.packaging == Packaging.DOCKER_CLOUD_ESS); final List contents = listContents("/opt"); assertThat("Expected beats in /opt", contents, hasItems("filebeat", "metricbeat")); @@ -1215,6 +1219,10 @@ private List listPlugins() { return sh.run(bin.pluginTool + " list").stdout().lines().collect(Collectors.toList()); } + private List listPluginArchive() { + return sh.run("ls -lh /opt/plugins/archive").stdout().lines().collect(Collectors.toList()); + } + /** * Check that readiness listener works */ diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/EnrollNodeToClusterTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/EnrollNodeToClusterTests.java index eac86cf7aac23..10d25efcfac13 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/EnrollNodeToClusterTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/EnrollNodeToClusterTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.test; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/EnrollmentProcessTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/EnrollmentProcessTests.java index 3598c7f8212fc..0f0a599423092 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/EnrollmentProcessTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/EnrollmentProcessTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.test; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/HttpClientThreadsFilter.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/HttpClientThreadsFilter.java index e23206d5d8ad1..f4be8fd001e4a 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/HttpClientThreadsFilter.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/HttpClientThreadsFilter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.test; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java index 9d9bb3f2f368a..02e1ce35764cf 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.test; @@ -435,7 +436,7 @@ private void verifyKeystorePermissions() { switch (distribution.packaging) { case TAR, ZIP -> assertThat(keystore, file(File, ARCHIVE_OWNER, ARCHIVE_OWNER, p660)); case DEB, RPM -> assertThat(keystore, file(File, "root", "elasticsearch", p660)); - case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS -> assertThat(keystore, DockerFileMatcher.file(p660)); + case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> assertThat(keystore, DockerFileMatcher.file(p660)); default -> throw new IllegalStateException("Unknown Elasticsearch packaging type."); } } diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/MemoryLockingTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/MemoryLockingTests.java index 82a17c54b6d69..53994e0f6cc36 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/MemoryLockingTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/MemoryLockingTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.test; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackageTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackageTests.java index bff48ba74e8fc..65cb241a69d77 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackageTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackageTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.test; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackageUpgradeTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackageUpgradeTests.java index 7346558e63c7f..235fc5e817d81 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackageUpgradeTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackageUpgradeTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.test; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagesSecurityAutoConfigurationTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagesSecurityAutoConfigurationTests.java index 9dab7e5eb8d16..d32e3083351a6 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagesSecurityAutoConfigurationTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagesSecurityAutoConfigurationTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.test; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java index 36fac5fe89ab5..b4a00ca56924a 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.test; @@ -244,7 +245,7 @@ protected static void install() throws Exception { installation = Packages.installPackage(sh, distribution); Packages.verifyPackageInstallation(installation, distribution, sh); } - case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS -> { + case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> { installation = Docker.runContainer(distribution); Docker.verifyContainerInstallation(installation); } @@ -334,8 +335,8 @@ public Shell.Result runElasticsearchStartCommand(String password, boolean daemon case DOCKER: case DOCKER_UBI: case DOCKER_IRON_BANK: - case DOCKER_CLOUD: case DOCKER_CLOUD_ESS: + case DOCKER_WOLFI: // nothing, "installing" docker image is running it return Shell.NO_OP; default: @@ -356,8 +357,8 @@ public void stopElasticsearch() throws Exception { case DOCKER: case DOCKER_UBI: case DOCKER_IRON_BANK: - case DOCKER_CLOUD: case DOCKER_CLOUD_ESS: + case DOCKER_WOLFI: // nothing, "installing" docker image is running it break; default: @@ -370,7 +371,7 @@ public void awaitElasticsearchStartup(Shell.Result result) throws Exception { switch (distribution.packaging) { case TAR, ZIP -> Archives.assertElasticsearchStarted(installation); case DEB, RPM -> Packages.assertElasticsearchStarted(sh, installation); - case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS -> Docker.waitForElasticsearchToStart(); + case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> Docker.waitForElasticsearchToStart(); default -> throw new IllegalStateException("Unknown Elasticsearch packaging type."); } } diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PasswordToolsTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PasswordToolsTests.java index 623cf34ce8a46..16795abf1b931 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PasswordToolsTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PasswordToolsTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.test; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PluginCliTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PluginCliTests.java index 2e9fb1256adc7..d0d962c91a848 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PluginCliTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PluginCliTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.test; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/RpmMetadataTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/RpmMetadataTests.java index b09a185e50ce7..72eaf520553c1 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/RpmMetadataTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/RpmMetadataTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.test; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/RpmPreservationTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/RpmPreservationTests.java index 18537c744f8b2..6ebb8b01ef30b 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/RpmPreservationTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/RpmPreservationTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.test; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/SqlCliTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/SqlCliTests.java index b01f525a04fb5..07ec6272e943e 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/SqlCliTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/SqlCliTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.test; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/TemporaryDirectoryConfigTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/TemporaryDirectoryConfigTests.java index b61fa72273dba..4b0cabac031f9 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/TemporaryDirectoryConfigTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/TemporaryDirectoryConfigTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.test; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/WindowsServiceTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/WindowsServiceTests.java index cca5bd702485a..36325bb156374 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/WindowsServiceTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/WindowsServiceTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.test; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Archives.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Archives.java index 787069eb2605c..69932ea34de5f 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Archives.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Archives.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.util; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Cleanup.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Cleanup.java index 4a0b5cd1a9a75..9dcbf76fa0ef6 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Cleanup.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Cleanup.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.util; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java index 6600514c67f9e..11b8324384631 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.util; @@ -32,10 +33,10 @@ public Distribution(Path path) { this.packaging = Packaging.DOCKER_UBI; } else if (filename.endsWith(".ironbank.tar")) { this.packaging = Packaging.DOCKER_IRON_BANK; - } else if (filename.endsWith(".cloud.tar")) { - this.packaging = Packaging.DOCKER_CLOUD; } else if (filename.endsWith(".cloud-ess.tar")) { this.packaging = Packaging.DOCKER_CLOUD_ESS; + } else if (filename.endsWith(".wolfi.tar")) { + this.packaging = Packaging.DOCKER_WOLFI; } else { int lastDot = filename.lastIndexOf('.'); this.packaging = Packaging.valueOf(filename.substring(lastDot + 1).toUpperCase(Locale.ROOT)); @@ -60,7 +61,7 @@ public boolean isPackage() { */ public boolean isDocker() { return switch (packaging) { - case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS -> true; + case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> true; default -> false; }; } @@ -74,8 +75,8 @@ public enum Packaging { DOCKER(".docker.tar", Platforms.isDocker()), DOCKER_UBI(".ubi.tar", Platforms.isDocker()), DOCKER_IRON_BANK(".ironbank.tar", Platforms.isDocker()), - DOCKER_CLOUD(".cloud.tar", Platforms.isDocker()), - DOCKER_CLOUD_ESS(".cloud-ess.tar", Platforms.isDocker()); + DOCKER_CLOUD_ESS(".cloud-ess.tar", Platforms.isDocker()), + DOCKER_WOLFI(".wolfi.tar", Platforms.isDocker()); /** The extension of this distribution's file */ public final String extension; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/FileExistenceMatchers.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/FileExistenceMatchers.java index 17dfed2fbce0d..62e80ad78ccb0 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/FileExistenceMatchers.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/FileExistenceMatchers.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.util; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/FileMatcher.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/FileMatcher.java index c9ff515ea0966..d98f7b4ebd4b9 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/FileMatcher.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/FileMatcher.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.util; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/FileUtils.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/FileUtils.java index 9c8cc8a7245cf..76140da810c2b 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/FileUtils.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/FileUtils.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.util; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Installation.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Installation.java index d1568bcf08a4f..88dbd2428db9c 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Installation.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Installation.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.util; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/LintianResultParser.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/LintianResultParser.java new file mode 100644 index 0000000000000..511080427ea77 --- /dev/null +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/LintianResultParser.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.packaging.util; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class LintianResultParser { + + private static final Logger logger = LogManager.getLogger(LintianResultParser.class); + private static final Pattern RESULT_PATTERN = Pattern.compile("(?[EW]): (?\\S+): (?\\S+) (?.+)"); + + public Result parse(String output) { + String[] lines = output.split("\n"); + List issues = Arrays.stream(lines).map(line -> { + Matcher matcher = RESULT_PATTERN.matcher(line); + if (matcher.matches() == false) { + logger.info("Lintian output not matching expected pattern: {}", line); + return null; + } + Severity severity = switch (matcher.group("severity")) { + case "E" -> Severity.ERROR; + case "W" -> Severity.WARNING; + default -> Severity.UNKNOWN; + }; + return new Issue(severity, matcher.group("tag"), matcher.group("message")); + }).filter(Objects::nonNull).toList(); + + return new Result(issues.stream().noneMatch(it -> it.severity == Severity.ERROR || it.severity == Severity.WARNING), issues); + } + + public record Result(boolean isSuccess, List issues) {} + + public record Issue(Severity severity, String tag, String message) {} + + enum Severity { + ERROR, + WARNING, + UNKNOWN + } +} diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Packages.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Packages.java index 54f82b2366d14..9b98c632106ca 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Packages.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Packages.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.util; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Platforms.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Platforms.java index 9b924e8508827..08255313231ac 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Platforms.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Platforms.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.util; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/ProcessInfo.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/ProcessInfo.java index 4080a0d7e76e2..3dfa1b6026261 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/ProcessInfo.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/ProcessInfo.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.util; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java index e6ed9e3f79e9e..f03e8ed205b48 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.util; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Shell.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Shell.java index d06110deb9ef0..42348b1fe2644 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Shell.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Shell.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.util; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java index f6fed3346139a..808aec92fb35d 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.util.docker; @@ -205,15 +206,34 @@ public static void waitForElasticsearchToStart() { ps output: %s - stdout(): + Stdout: %s Stderr: + %s + + Thread dump: %s\ - """, psOutput, dockerLogs.stdout(), dockerLogs.stderr())); + """, psOutput, dockerLogs.stdout(), dockerLogs.stderr(), getThreadDump())); } } + /** + * @return output of jstack for currently running Java process + */ + private static String getThreadDump() { + try { + String pid = dockerShell.run("/usr/share/elasticsearch/jdk/bin/jps | grep -v 'Jps' | awk '{print $1}'").stdout(); + if (pid.isEmpty() == false) { + return dockerShell.run("/usr/share/elasticsearch/jdk/bin/jstack " + Integer.parseInt(pid)).stdout(); + } + } catch (Exception e) { + logger.error("Failed to get thread dump", e); + } + + return ""; + } + /** * Waits for the Elasticsearch container to exit. */ @@ -485,9 +505,9 @@ public static void verifyContainerInstallation(Installation es) { // Ensure the `elasticsearch` user and group exist. // These lines will both throw an exception if the command fails dockerShell.run("id elasticsearch"); - dockerShell.run("getent group elasticsearch"); + dockerShell.run("grep -E '^elasticsearch:' /etc/group"); - final Shell.Result passwdResult = dockerShell.run("getent passwd elasticsearch"); + final Shell.Result passwdResult = dockerShell.run("grep -E '^elasticsearch:' /etc/passwd"); final String homeDir = passwdResult.stdout().trim().split(":")[5]; assertThat("elasticsearch user's home directory is incorrect", homeDir, equalTo("/usr/share/elasticsearch")); @@ -531,7 +551,7 @@ public static void verifyContainerInstallation(Installation es) { ) ); - if (es.distribution.packaging == Packaging.DOCKER_CLOUD || es.distribution.packaging == Packaging.DOCKER_CLOUD_ESS) { + if (es.distribution.packaging == Packaging.DOCKER_CLOUD_ESS) { verifyCloudContainerInstallation(es); } } diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerFileAttributes.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerFileAttributes.java index 64b5e166c028f..32fd36aaf4da5 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerFileAttributes.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerFileAttributes.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.util.docker; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerFileMatcher.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerFileMatcher.java index d9dca46b9c24a..7f371e3d5ec12 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerFileMatcher.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerFileMatcher.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.util.docker; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java index 41c61edde02be..6df970fed16c1 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.util.docker; @@ -28,6 +29,9 @@ */ public class DockerRun { + // Use less secure entropy source to avoid hanging when generating certificates + private static final String DEFAULT_JAVA_OPTS = "-Djava.security.egd=file:/dev/urandom"; + private Distribution distribution; private final Map envVars = new HashMap<>(); private final Map volumes = new HashMap<>(); @@ -111,6 +115,11 @@ String build() { // Limit container memory cmd.add("--memory " + memory); + // Add default java opts + for (String envVar : List.of("CLI_JAVA_OPTS", "ES_JAVA_OPTS")) { + this.envVars.put(envVar, this.envVars.getOrDefault(envVar, "") + " " + DEFAULT_JAVA_OPTS); + } + this.envVars.forEach((key, value) -> cmd.add("--env " + key + "=\"" + value + "\"")); // Map ports in the container to the host, so that we can send requests @@ -164,8 +173,8 @@ public static String getImageName(Distribution distribution) { case DOCKER -> ""; case DOCKER_UBI -> "-ubi8"; case DOCKER_IRON_BANK -> "-ironbank"; - case DOCKER_CLOUD -> "-cloud"; case DOCKER_CLOUD_ESS -> "-cloud-ess"; + case DOCKER_WOLFI -> "-wolfi"; default -> throw new IllegalStateException("Unexpected distribution packaging type: " + distribution.packaging); }; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerShell.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerShell.java index 0847302f1c40f..2f6848cc4b25f 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerShell.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerShell.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.util.docker; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/MockServer.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/MockServer.java index d2c43e478df8f..d87ba9ed7d4c5 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/MockServer.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/MockServer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.packaging.util.docker; diff --git a/qa/remote-clusters/build.gradle b/qa/remote-clusters/build.gradle index 67f62c0fee04d..e3d42850375d9 100644 --- a/qa/remote-clusters/build.gradle +++ b/qa/remote-clusters/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.Architecture @@ -38,14 +39,22 @@ elasticsearch_distributions { } } + +interface Injected { + @Inject + FileSystemOperations getFs() +} + tasks.named("preProcessFixture").configure { dependsOn "copyNodeKeyMaterial", elasticsearch_distributions.docker + def injected = project.objects.newInstance(Injected) + doLast { // tests expect to have an empty repo - project.delete( - "${testFixturesDir}/repo", - "${testFixturesDir}/oss-repo" - ) + injected.fs.delete { + it.delete("${testFixturesDir}/repo") + it.delete("${testFixturesDir}/oss-repo") + } createAndSetWritable( "${testFixturesDir}/repo", "${testFixturesDir}/oss-repo", @@ -68,7 +77,8 @@ dockerCompose { def createAndSetWritable(Object... locations) { locations.each { location -> - File file = file(location) + println "location = $location" + File file = new File(location) file.mkdirs() file.setWritable(true, false) } diff --git a/qa/remote-clusters/output.log b/qa/remote-clusters/output.log new file mode 100644 index 0000000000000..11eebe6fb8fda --- /dev/null +++ b/qa/remote-clusters/output.log @@ -0,0 +1,125 @@ +Using gradle at '/Users/rene/dev/elastic/elasticsearch/gradlew' to run buildfile '/Users/rene/dev/elastic/elasticsearch/qa/remote-clusters/build.gradle': + +Starting a Gradle Daemon (subsequent builds will be faster) +Calculating task graph as no cached configuration is available for tasks: check + +> Task :build-conventions:compileJava +Note: Some input files use or override a deprecated API. +Note: Recompile with -Xlint:deprecation for details. +Note: /Users/rene/dev/elastic/elasticsearch/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/LicensingPlugin.java uses unchecked or unsafe operations. +Note: Recompile with -Xlint:unchecked for details. + +> Task :build-conventions:pluginDescriptors +> Task :build-conventions:processResources +> Task :build-conventions:classes +> Task :build-conventions:jar +> Task :build-tools:reaper:compileJava FROM-CACHE +> Task :build-tools:reaper:processResources NO-SOURCE +> Task :build-tools:reaper:classes UP-TO-DATE +> Task :build-tools-internal:extractPluginRequests FROM-CACHE +> Task :build-tools:reaper:jar +> Task :build-tools-internal:generatePluginAdapters FROM-CACHE +> Task :build-tools-internal:pluginDescriptors +> Task :build-tools:compileJava FROM-CACHE +> Task :build-tools:compileGroovy NO-SOURCE +> Task :build-tools:generateVersionProperties FROM-CACHE +> Task :build-tools:pluginDescriptors +> Task :build-tools:processResources +> Task :build-tools:classes +> Task :build-tools:jar +> Task :build-tools-internal:processResources + +> Task :build-tools-internal:compileJava +Note: Some input files use or override a deprecated API. +Note: Recompile with -Xlint:deprecation for details. +Note: Some input files use unchecked or unsafe operations. +Note: Recompile with -Xlint:unchecked for details. + +> Task :build-tools-internal:compileGroovy +> Task :build-tools-internal:compileGroovyPlugins +> Task :build-tools-internal:classes +> Task :build-tools-internal:jar +======================================= +Elasticsearch Build Hamster says Hello! + Gradle Version : 8.12 + OS Info : Mac OS X 15.2 (aarch64) + JDK Version : 21.0.5+9-LTS-239 (Oracle) + JAVA_HOME : /Users/rene/.sdkman/candidates/java/21.0.5-oracle + Random Testing Seed : 8D9A9CFD8E09C560 + In FIPS 140 mode : false +======================================= +> Task :qa:remote-clusters:processTestResources +> Task :server:generateModulesList +> Task :server:generatePluginsList +> Task :qa:remote-clusters:copyCheckstyleConf +> Task :qa:remote-clusters:filepermissions +> Task :client:sniffer:processResources NO-SOURCE +> Task :libs:core:processResources NO-SOURCE +> Task :libs:cli:processResources NO-SOURCE +> Task :libs:entitlement:processResources NO-SOURCE +> Task :libs:entitlement:bridge:processResources NO-SOURCE +> Task :libs:entitlement:asm-provider:processResources +> Task :libs:entitlement:processMain23Resources NO-SOURCE +> Task :libs:entitlement:bridge:processMain23Resources NO-SOURCE +> Task :libs:geo:processResources NO-SOURCE +> Task :libs:logging:processResources NO-SOURCE +> Task :libs:lz4:processResources NO-SOURCE +> Task :libs:native:processResources NO-SOURCE +> Task :libs:native:processMain22Resources NO-SOURCE +> Task :libs:plugin-analysis-api:processResources NO-SOURCE +> Task :libs:plugin-api:processResources NO-SOURCE +> Task :libs:secure-sm:processResources NO-SOURCE +> Task :libs:simdvec:processResources NO-SOURCE +> Task :libs:simdvec:processMain21Resources NO-SOURCE +> Task :libs:simdvec:processMain22Resources NO-SOURCE +> Task :client:rest:processResources +> Task :libs:tdigest:processResources NO-SOURCE +> Task :libs:x-content:impl:processResources +> Task :libs:x-content:processResources NO-SOURCE +> Task :modules:transport-netty4:processResources NO-SOURCE +> Task :server:processResources +> Task :libs:ssl-config:processResources NO-SOURCE +> Task :test:yaml-rest-runner:processResources NO-SOURCE +> Task :qa:remote-clusters:forbiddenApisResources +> Task :qa:remote-clusters:forbiddenPatterns +> Task :qa:remote-clusters:licenseHeaders +> Task :test:logger-usage:processResources NO-SOURCE +> Task :libs:grok:processResources +> Task :qa:remote-clusters:testingConventions UP-TO-DATE +> Task :qa:remote-clusters:copyNodeKeyMaterial FAILED +> Task :spotlessInternalRegisterDependencies +> Task :test:framework:processResources +> Task :libs:plugin-api:compileJava +> Task :libs:logging:compileJava +> Task :libs:entitlement:bridge:compileJava + +> Task :libs:secure-sm:compileJava +Note: Some input files use or override a deprecated API that is marked for removal. +Note: Recompile with -Xlint:removal for details. + +> Task :test:logger-usage:compileJava +> Task :qa:remote-clusters:spotlessJava +> Task :libs:grok:compileJava +> Task :libs:geo:compileJava +> Task :client:rest:compileJava +> Task :qa:remote-clusters:checkstyleTest + +[Incubating] Problems report is available at: file:///Users/rene/dev/elastic/elasticsearch/build/reports/problems/problems-report.html + +FAILURE: Build failed with an exception. + +* Where: +Build file '/Users/rene/dev/elastic/elasticsearch/qa/remote-clusters/build.gradle' line: 27 + +* What went wrong: +Execution failed for task ':qa:remote-clusters:copyNodeKeyMaterial'. +> Cannot reference a Gradle script object from a Groovy closure as these are not supported with the configuration cache. + +* Try: +> Run with --stacktrace option to get the stack trace. +> Run with --info or --debug option to get more log output. +> Get more help at https://help.gradle.org. + +BUILD FAILED in 1m 4s +45 actionable tasks: 40 executed, 5 from cache +Configuration cache entry stored. diff --git a/qa/remote-clusters/src/test/java/org/elasticsearch/cluster/remote/test/AbstractMultiClusterRemoteTestCase.java b/qa/remote-clusters/src/test/java/org/elasticsearch/cluster/remote/test/AbstractMultiClusterRemoteTestCase.java index d9be4045c37e0..de9def50a4677 100644 --- a/qa/remote-clusters/src/test/java/org/elasticsearch/cluster/remote/test/AbstractMultiClusterRemoteTestCase.java +++ b/qa/remote-clusters/src/test/java/org/elasticsearch/cluster/remote/test/AbstractMultiClusterRemoteTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.remote.test; diff --git a/qa/remote-clusters/src/test/java/org/elasticsearch/cluster/remote/test/RemoteClustersIT.java b/qa/remote-clusters/src/test/java/org/elasticsearch/cluster/remote/test/RemoteClustersIT.java index 78ffb9cb7b7b6..77986c4c52e5b 100644 --- a/qa/remote-clusters/src/test/java/org/elasticsearch/cluster/remote/test/RemoteClustersIT.java +++ b/qa/remote-clusters/src/test/java/org/elasticsearch/cluster/remote/test/RemoteClustersIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.remote.test; diff --git a/qa/repository-multi-version/build.gradle b/qa/repository-multi-version/build.gradle index c6c5d7212357f..79a8be4c1be24 100644 --- a/qa/repository-multi-version/build.gradle +++ b/qa/repository-multi-version/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.internal.info.BuildParams @@ -15,7 +16,7 @@ apply plugin: 'elasticsearch.internal-test-artifact' apply plugin: 'elasticsearch.bwc-test' -BuildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName -> +buildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName -> String oldClusterName = "${baseName}-old" String newClusterName = "${baseName}-new" diff --git a/qa/repository-multi-version/src/test/java/org/elasticsearch/upgrades/MultiVersionRepositoryAccessIT.java b/qa/repository-multi-version/src/test/java/org/elasticsearch/upgrades/MultiVersionRepositoryAccessIT.java index 9d330cd7e35eb..dc002ea1a44c1 100644 --- a/qa/repository-multi-version/src/test/java/org/elasticsearch/upgrades/MultiVersionRepositoryAccessIT.java +++ b/qa/repository-multi-version/src/test/java/org/elasticsearch/upgrades/MultiVersionRepositoryAccessIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; diff --git a/qa/restricted-loggers/build.gradle b/qa/restricted-loggers/build.gradle index f08f886d52888..2f890fd79c515 100644 --- a/qa/restricted-loggers/build.gradle +++ b/qa/restricted-loggers/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.standalone-test' diff --git a/qa/restricted-loggers/src/test/java/org/elasticsearch/common/logging/LoggersTests.java b/qa/restricted-loggers/src/test/java/org/elasticsearch/common/logging/LoggersTests.java index 5af036a9a0391..82535257dbc57 100644 --- a/qa/restricted-loggers/src/test/java/org/elasticsearch/common/logging/LoggersTests.java +++ b/qa/restricted-loggers/src/test/java/org/elasticsearch/common/logging/LoggersTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.logging; diff --git a/qa/rolling-upgrade-legacy/build.gradle b/qa/rolling-upgrade-legacy/build.gradle index 77dfc9724ce8f..e1c31fd50c0d4 100644 --- a/qa/rolling-upgrade-legacy/build.gradle +++ b/qa/rolling-upgrade-legacy/build.gradle @@ -1,15 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.Version -import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask apply plugin: 'elasticsearch.internal-testclusters' @@ -17,7 +17,7 @@ apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.bwc-test' apply plugin: 'elasticsearch.rest-resources' -BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> +buildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> /* * NOTE: This module is for the tests that were problematic when converting :qa:rolling-upgrade to the junit-based bwc test definition * Over time, these should be migrated into the :qa:rolling-upgrade module and fixed properly diff --git a/qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/AbstractRollingTestCase.java b/qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/AbstractRollingTestCase.java index 74a8eb7fd1988..e0ce4e47212c9 100644 --- a/qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/AbstractRollingTestCase.java +++ b/qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/AbstractRollingTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; diff --git a/qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java b/qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java index fe2236adc4904..b7391eb8a19c3 100644 --- a/qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java +++ b/qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; diff --git a/qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java b/qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java index 068747d5a4824..06889d1ed974f 100644 --- a/qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java +++ b/qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index d3af5d25b70ff..2f717f201f248 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.internal.info.BuildParams @@ -17,7 +18,7 @@ testArtifacts { registerTestArtifactFromSourceSet(sourceSets.javaRestTest) } -BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> +buildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) { usesBwcDistribution(bwcVersion) systemProperty("tests.old_cluster_version", bwcVersion) diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/AbstractRollingUpgradeTestCase.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/AbstractRollingUpgradeTestCase.java index 4837afbf6ccd2..89137b3ff9814 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/AbstractRollingUpgradeTestCase.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/AbstractRollingUpgradeTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ClusterFeatureMigrationIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ClusterFeatureMigrationIT.java index 73abb634dfd76..2b3e10acc15ed 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ClusterFeatureMigrationIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ClusterFeatureMigrationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DenseVectorMappingUpdateIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DenseVectorMappingUpdateIT.java index 0830c1b766a22..b8fc3b15636fb 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DenseVectorMappingUpdateIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DenseVectorMappingUpdateIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java index c7f99b3525f74..e0d1e7aafa637 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DownsampleIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DownsampleIT.java index 488cd966ed65e..f963bedf5d188 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DownsampleIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DownsampleIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; @@ -238,7 +239,7 @@ private String getRollupIndexName() throws IOException { if (asMap.size() == 1) { return (String) asMap.keySet().toArray()[0]; } - logger.warn("--> No matching rollup name for path [%s]", endpoint); + logger.warn("--> No matching rollup name for path [{}]", endpoint); return null; } diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java index fc77eef0ae8bb..9cc3fff5828c9 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FieldCapsIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FieldCapsIT.java index 306447d8cc2cd..c6f4588fb4cd2 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FieldCapsIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FieldCapsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; @@ -11,7 +12,6 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import org.apache.http.HttpHost; -import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.Build; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; import org.elasticsearch.client.Request; @@ -39,7 +39,6 @@ * In 8.2 we also added the ability to filter fields by type and metadata, with some post-hoc filtering applied on * the co-ordinating node if older nodes were included in the system */ -@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/103473") public class FieldCapsIT extends AbstractRollingUpgradeTestCase { public FieldCapsIT(@Name("upgradedNodes") int upgradedNodes) { diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsRoleMappingUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsRoleMappingUpgradeIT.java new file mode 100644 index 0000000000000..d4414f580887a --- /dev/null +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsRoleMappingUpgradeIT.java @@ -0,0 +1,139 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.upgrades; + +import com.carrotsearch.randomizedtesting.annotations.Name; + +import org.elasticsearch.client.Request; +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.test.XContentTestUtils; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.cluster.util.Version; +import org.elasticsearch.test.cluster.util.resource.Resource; +import org.elasticsearch.test.junit.RunnableTestRuleAdapter; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.TestRule; + +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; + +public class FileSettingsRoleMappingUpgradeIT extends ParameterizedRollingUpgradeTestCase { + + private static final int ROLE_MAPPINGS_CLEANUP_MIGRATION_VERSION = 2; + private static final String SETTING_JSON = """ + { + "metadata": { + "version": "1", + "compatibility": "8.4.0" + }, + "state": { + "role_mappings": { + "everyone_kibana": { + "enabled": true, + "roles": [ "kibana_user" ], + "rules": { "field": { "username": "*" } } + } + } + } + }"""; + + private static final TemporaryFolder repoDirectory = new TemporaryFolder(); + private static final ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .version(getOldClusterTestVersion()) + .nodes(NODE_NUM) + .setting("path.repo", new Supplier<>() { + @Override + @SuppressForbidden(reason = "TemporaryFolder only has io.File methods, not nio.File") + public String get() { + return repoDirectory.getRoot().getPath(); + } + }) + .setting("xpack.security.enabled", "true") + // workaround to avoid having to set up clients and authorization headers + .setting("xpack.security.authc.anonymous.roles", "superuser") + .configFile("operator/settings.json", Resource.fromString(SETTING_JSON)) + .build(); + private static final RunnableTestRuleAdapter versionLimit = new RunnableTestRuleAdapter( + () -> assumeTrue( + "Only relevant when upgrading from a version before role mappings were stored in cluster state", + getOldClusterTestVersion().after(new Version(8, 7, 0)) && getOldClusterTestVersion().before(new Version(8, 15, 0)) + ) + ); + + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(versionLimit).around(repoDirectory).around(cluster); + + public FileSettingsRoleMappingUpgradeIT(@Name("upgradedNodes") int upgradedNodes) { + super(upgradedNodes); + } + + @Override + protected ElasticsearchCluster getUpgradeCluster() { + return cluster; + } + + private static void waitForSecurityMigrationCompletionIfIndexExists() throws Exception { + final Request request = new Request("GET", "_cluster/state/metadata/.security-7"); + assertBusy(() -> { + Map indices = new XContentTestUtils.JsonMapView(entityAsMap(client().performRequest(request))).get( + "metadata.indices" + ); + assertNotNull(indices); + // If the security index exists, migration needs to happen. There is a bug in pre cluster state role mappings code that tries + // to write file based role mappings before security index manager state is recovered, this makes it look like the security + // index is outdated (isIndexUpToDate == false). Because we can't rely on the index being there for old versions, this check + // is needed. + if (indices.containsKey(".security-7")) { + // JsonMapView doesn't support . prefixed indices (splits on .) + @SuppressWarnings("unchecked") + String responseVersion = new XContentTestUtils.JsonMapView((Map) indices.get(".security-7")).get( + "migration_version.version" + ); + assertNotNull(responseVersion); + assertTrue(Integer.parseInt(responseVersion) >= ROLE_MAPPINGS_CLEANUP_MIGRATION_VERSION); + } + }); + } + + public void testRoleMappingsAppliedOnUpgrade() throws Exception { + if (isOldCluster()) { + Request clusterStateRequest = new Request("GET", "/_cluster/state/metadata"); + List roleMappings = new XContentTestUtils.JsonMapView(entityAsMap(client().performRequest(clusterStateRequest))).get( + "metadata.role_mappings.role_mappings" + ); + assertThat(roleMappings, is(nullValue())); + } else if (isUpgradedCluster()) { + // the nodes have all been upgraded. Check they re-processed the role mappings in the settings file on + // upgrade + Request clusterStateRequest = new Request("GET", "/_cluster/state/metadata"); + List clusterStateRoleMappings = new XContentTestUtils.JsonMapView( + entityAsMap(client().performRequest(clusterStateRequest)) + ).get("metadata.role_mappings.role_mappings"); + assertThat(clusterStateRoleMappings, is(not(nullValue()))); + assertThat(clusterStateRoleMappings.size(), equalTo(1)); + waitForSecurityMigrationCompletionIfIndexExists(); + assertThat( + entityAsMap(client().performRequest(new Request("GET", "/_security/role_mapping"))).keySet(), + contains("everyone_kibana-read-only-operator-mapping") + ); + } + } +} diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsUpgradeIT.java index 8ffaec5506f1d..0f48fb65b41f3 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsUpgradeIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsUpgradeIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/HealthNodeUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/HealthNodeUpgradeIT.java index 6647cb413c9f5..2ed1b7fe9e79b 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/HealthNodeUpgradeIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/HealthNodeUpgradeIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IgnoredMetaFieldRollingUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IgnoredMetaFieldRollingUpgradeIT.java index 1477e2b63cf03..9919fcaaefb4f 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IgnoredMetaFieldRollingUpgradeIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IgnoredMetaFieldRollingUpgradeIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java index 157e2293b69ae..b0309ac8e4a94 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; @@ -17,6 +18,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.test.ListMatcher; @@ -418,9 +420,15 @@ public void testSyntheticSource() throws IOException { if (isOldCluster()) { Request createIndex = new Request("PUT", "/synthetic"); XContentBuilder indexSpec = XContentBuilder.builder(XContentType.JSON.xContent()).startObject(); + boolean useIndexSetting = getOldClusterIndexVersion().onOrAfter(IndexVersions.DEPRECATE_SOURCE_MODE_MAPPER); + if (useIndexSetting) { + indexSpec.startObject("settings").field("index.mapping.source.mode", "synthetic").endObject(); + } indexSpec.startObject("mappings"); { - indexSpec.startObject("_source").field("mode", "synthetic").endObject(); + if (useIndexSetting == false) { + indexSpec.startObject("_source").field("mode", "synthetic").endObject(); + } indexSpec.startObject("properties").startObject("kwd").field("type", "keyword").endObject().endObject(); } indexSpec.endObject(); diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeRollingUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeRollingUpgradeIT.java index 1549789bcc44a..1eb7cbd3f70c2 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeRollingUpgradeIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeRollingUpgradeIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; @@ -16,7 +17,6 @@ import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.FormatNames; -import org.elasticsearch.test.MapMatcher; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.hamcrest.Matcher; @@ -29,9 +29,6 @@ import java.util.Map; import java.util.function.Supplier; -import static org.elasticsearch.test.MapMatcher.assertMap; -import static org.elasticsearch.test.MapMatcher.matchesMap; - public class LogsIndexModeRollingUpgradeIT extends AbstractRollingUpgradeTestCase { @ClassRule() @@ -44,7 +41,9 @@ public class LogsIndexModeRollingUpgradeIT extends AbstractRollingUpgradeTestCas .module("x-pack-stack") .setting("xpack.security.enabled", "false") .setting("xpack.license.self_generated.type", "trial") - .setting("cluster.logsdb.enabled", "true") + // We upgrade from standard to logsdb, so we need to start with logsdb disabled, + // then later cluster.logsdb.enabled gets set to true and next rollover data stream is in logsdb mode. + .setting("cluster.logsdb.enabled", "false") .setting("stack.templates.enabled", "false") .build(); @@ -90,39 +89,6 @@ protected String getTestRestCluster() { } }"""; - private static final String LOGS_TEMPLATE = """ - { - "index_patterns": [ "logs-*-*" ], - "data_stream": {}, - "priority": 500, - "template": { - "settings": { - "index": { - "mode": "logsdb" - } - }, - "mappings": { - "properties": { - "@timestamp" : { - "type": "date" - }, - "host.name": { - "type": "keyword" - }, - "method": { - "type": "keyword" - }, - "message": { - "type": "text" - }, - "ip.address": { - "type": "ip" - } - } - } - } - }"""; - public void testLogsIndexing() throws IOException { if (isOldCluster()) { assertOK(client().performRequest(putTemplate(client(), "logs-template", STANDARD_TEMPLATE))); @@ -168,7 +134,7 @@ public void testLogsIndexing() throws IOException { assertOK(bulkIndexResponse); assertThat(entityAsMap(bulkIndexResponse).get("errors"), Matchers.is(false)); } else if (isUpgradedCluster()) { - assertOK(client().performRequest(putTemplate(client(), "logs-template", LOGS_TEMPLATE))); + enableLogsdbByDefault(); assertOK(client().performRequest(rolloverDataStream(client(), "logs-apache-production"))); final Response bulkIndexResponse = client().performRequest(bulkIndex("logs-apache-production", () -> { final StringBuilder sb = new StringBuilder(); @@ -190,24 +156,30 @@ public void testLogsIndexing() throws IOException { assertOK(bulkIndexResponse); assertThat(entityAsMap(bulkIndexResponse).get("errors"), Matchers.is(false)); - assertIndexMappingsAndSettings(0, Matchers.nullValue(), matchesMap().extraOk()); - assertIndexMappingsAndSettings(1, Matchers.nullValue(), matchesMap().extraOk()); - assertIndexMappingsAndSettings(2, Matchers.nullValue(), matchesMap().extraOk()); - assertIndexMappingsAndSettings( - 3, - Matchers.equalTo("logsdb"), - matchesMap().extraOk().entry("_source", Map.of("mode", "synthetic")) - ); + assertIndexSettings(0, Matchers.nullValue()); + assertIndexSettings(1, Matchers.nullValue()); + assertIndexSettings(2, Matchers.nullValue()); + assertIndexSettings(3, Matchers.equalTo("logsdb")); } } - private void assertIndexMappingsAndSettings(int backingIndex, final Matcher indexModeMatcher, final MapMatcher mappingsMatcher) - throws IOException { + static void enableLogsdbByDefault() throws IOException { + var request = new Request("PUT", "/_cluster/settings"); + request.setJsonEntity(""" + { + "persistent": { + "cluster.logsdb.enabled": true + } + } + """); + assertOK(client().performRequest(request)); + } + + private void assertIndexSettings(int backingIndex, final Matcher indexModeMatcher) throws IOException { assertThat( getSettings(client(), getWriteBackingIndex(client(), "logs-apache-production", backingIndex)).get("index.mode"), indexModeMatcher ); - assertMap(getIndexMappingAsMap(getWriteBackingIndex(client(), "logs-apache-production", backingIndex)), mappingsMatcher); } private static Request createDataStream(final String dataStreamName) { @@ -232,7 +204,7 @@ private static Request rolloverDataStream(final RestClient client, final String } @SuppressWarnings("unchecked") - private static String getWriteBackingIndex(final RestClient client, final String dataStreamName, int backingIndex) throws IOException { + static String getWriteBackingIndex(final RestClient client, final String dataStreamName, int backingIndex) throws IOException { final Request request = new Request("GET", "_data_stream/" + dataStreamName); final List dataStreams = (List) entityAsMap(client.performRequest(request)).get("data_streams"); final Map dataStream = (Map) dataStreams.get(0); diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsdbIndexingRollingUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsdbIndexingRollingUpgradeIT.java new file mode 100644 index 0000000000000..4be2ce986ba0c --- /dev/null +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsdbIndexingRollingUpgradeIT.java @@ -0,0 +1,263 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.upgrades; + +import com.carrotsearch.randomizedtesting.annotations.Name; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.test.rest.ObjectPath; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.io.InputStream; +import java.time.Instant; +import java.util.Locale; +import java.util.Map; + +import static org.elasticsearch.upgrades.LogsIndexModeRollingUpgradeIT.enableLogsdbByDefault; +import static org.elasticsearch.upgrades.LogsIndexModeRollingUpgradeIT.getWriteBackingIndex; +import static org.elasticsearch.upgrades.TsdbIT.formatInstant; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.notNullValue; + +public class LogsdbIndexingRollingUpgradeIT extends AbstractRollingUpgradeTestCase { + + static String BULK_ITEM_TEMPLATE = + """ + {"@timestamp": "$now", "host.name": "$host", "method": "$method", "ip": "$ip", "message": "$message", "length": $length, "factor": $factor} + """; + + private static final String TEMPLATE = """ + { + "mappings": { + "properties": { + "@timestamp" : { + "type": "date" + }, + "method": { + "type": "keyword" + }, + "message": { + "type": "text" + }, + "ip": { + "type": "ip" + }, + "length": { + "type": "long" + }, + "factor": { + "type": "double" + } + } + } + }"""; + + public LogsdbIndexingRollingUpgradeIT(@Name("upgradedNodes") int upgradedNodes) { + super(upgradedNodes); + } + + public void testIndexing() throws Exception { + assumeTrue("test relies on index.mapping.source.mode setting", getOldClusterTestVersion().onOrAfter("8.16.0")); + String dataStreamName = "logs-bwc-test"; + if (isOldCluster()) { + startTrial(); + enableLogsdbByDefault(); + createTemplate(dataStreamName, getClass().getSimpleName().toLowerCase(Locale.ROOT), TEMPLATE); + + Instant startTime = Instant.now().minusSeconds(60 * 60); + bulkIndex(dataStreamName, 4, 1024, startTime); + + String firstBackingIndex = getWriteBackingIndex(client(), dataStreamName, 0); + var settings = (Map) getIndexSettingsWithDefaults(firstBackingIndex).get(firstBackingIndex); + assertThat(((Map) settings.get("settings")).get("index.mode"), equalTo("logsdb")); + assertThat(((Map) settings.get("defaults")).get("index.mapping.source.mode"), equalTo("SYNTHETIC")); + + ensureGreen(dataStreamName); + search(dataStreamName); + query(dataStreamName); + } else if (isMixedCluster()) { + Instant startTime = Instant.now().minusSeconds(60 * 30); + bulkIndex(dataStreamName, 4, 1024, startTime); + + ensureGreen(dataStreamName); + search(dataStreamName); + query(dataStreamName); + } else if (isUpgradedCluster()) { + ensureGreen(dataStreamName); + Instant startTime = Instant.now(); + bulkIndex(dataStreamName, 4, 1024, startTime); + search(dataStreamName); + query(dataStreamName); + + var forceMergeRequest = new Request("POST", "/" + dataStreamName + "/_forcemerge"); + forceMergeRequest.addParameter("max_num_segments", "1"); + assertOK(client().performRequest(forceMergeRequest)); + + ensureGreen(dataStreamName); + search(dataStreamName); + query(dataStreamName); + } + } + + static void createTemplate(String dataStreamName, String id, String template) throws IOException { + final String INDEX_TEMPLATE = """ + { + "index_patterns": ["$DATASTREAM"], + "template": $TEMPLATE, + "data_stream": { + } + }"""; + var putIndexTemplateRequest = new Request("POST", "/_index_template/" + id); + putIndexTemplateRequest.setJsonEntity(INDEX_TEMPLATE.replace("$TEMPLATE", template).replace("$DATASTREAM", dataStreamName)); + assertOK(client().performRequest(putIndexTemplateRequest)); + } + + static void bulkIndex(String dataStreamName, int numRequest, int numDocs, Instant startTime) throws Exception { + for (int i = 0; i < numRequest; i++) { + var bulkRequest = new Request("POST", "/" + dataStreamName + "/_bulk"); + StringBuilder requestBody = new StringBuilder(); + for (int j = 0; j < numDocs; j++) { + String hostName = "host" + j % 50; // Not realistic, but makes asserting search / query response easier. + String methodName = "method" + j % 5; + String ip = NetworkAddress.format(randomIp(true)); + String message = randomAlphaOfLength(128); + long length = randomLong(); + double factor = randomDouble(); + + requestBody.append("{\"create\": {}}"); + requestBody.append('\n'); + requestBody.append( + BULK_ITEM_TEMPLATE.replace("$now", formatInstant(startTime)) + .replace("$host", hostName) + .replace("$method", methodName) + .replace("$ip", ip) + .replace("$message", message) + .replace("$length", Long.toString(length)) + .replace("$factor", Double.toString(factor)) + ); + requestBody.append('\n'); + + startTime = startTime.plusMillis(1); + } + bulkRequest.setJsonEntity(requestBody.toString()); + bulkRequest.addParameter("refresh", "true"); + var response = client().performRequest(bulkRequest); + assertOK(response); + var responseBody = entityAsMap(response); + assertThat("errors in response:\n " + responseBody, responseBody.get("errors"), equalTo(false)); + } + } + + void search(String dataStreamName) throws Exception { + var searchRequest = new Request("POST", "/" + dataStreamName + "/_search"); + searchRequest.addParameter("pretty", "true"); + searchRequest.setJsonEntity(""" + { + "size": 0, + "aggs": { + "host_name": { + "terms": { + "field": "host.name", + "order": { "_key": "asc" } + }, + "aggs": { + "max_length": { + "max": { + "field": "length" + } + }, + "max_factor": { + "max": { + "field": "factor" + } + } + } + } + } + } + """); + var response = client().performRequest(searchRequest); + assertOK(response); + var responseBody = entityAsMap(response); + + Integer totalCount = ObjectPath.evaluate(responseBody, "hits.total.value"); + assertThat(totalCount, greaterThanOrEqualTo(4096)); + String key = ObjectPath.evaluate(responseBody, "aggregations.host_name.buckets.0.key"); + assertThat(key, equalTo("host0")); + Integer docCount = ObjectPath.evaluate(responseBody, "aggregations.host_name.buckets.0.doc_count"); + assertThat(docCount, greaterThan(0)); + Double maxTx = ObjectPath.evaluate(responseBody, "aggregations.host_name.buckets.0.max_length.value"); + assertThat(maxTx, notNullValue()); + Double maxRx = ObjectPath.evaluate(responseBody, "aggregations.host_name.buckets.0.max_factor.value"); + assertThat(maxRx, notNullValue()); + } + + void query(String dataStreamName) throws Exception { + var queryRequest = new Request("POST", "/_query"); + queryRequest.addParameter("pretty", "true"); + queryRequest.setJsonEntity(""" + { + "query": "FROM $ds | STATS max(length), max(factor) BY host.name | SORT host.name | LIMIT 5" + } + """.replace("$ds", dataStreamName)); + var response = client().performRequest(queryRequest); + assertOK(response); + var responseBody = entityAsMap(response); + + String column1 = ObjectPath.evaluate(responseBody, "columns.0.name"); + String column2 = ObjectPath.evaluate(responseBody, "columns.1.name"); + String column3 = ObjectPath.evaluate(responseBody, "columns.2.name"); + assertThat(column1, equalTo("max(length)")); + assertThat(column2, equalTo("max(factor)")); + assertThat(column3, equalTo("host.name")); + + String key = ObjectPath.evaluate(responseBody, "values.0.2"); + assertThat(key, equalTo("host0")); + Long maxRx = ObjectPath.evaluate(responseBody, "values.0.0"); + assertThat(maxRx, notNullValue()); + Double maxTx = ObjectPath.evaluate(responseBody, "values.0.1"); + assertThat(maxTx, notNullValue()); + } + + protected static void startTrial() throws IOException { + Request startTrial = new Request("POST", "/_license/start_trial"); + startTrial.addParameter("acknowledge", "true"); + try { + assertOK(client().performRequest(startTrial)); + } catch (ResponseException e) { + var responseBody = entityAsMap(e.getResponse()); + String error = ObjectPath.evaluate(responseBody, "error_message"); + assertThat(error, containsString("Trial was already activated.")); + } + } + + static Map getIndexSettingsWithDefaults(String index) throws IOException { + Request request = new Request("GET", "/" + index + "/_settings"); + request.addParameter("flat_settings", "true"); + request.addParameter("include_defaults", "true"); + Response response = client().performRequest(request); + try (InputStream is = response.getEntity().getContent()) { + return XContentHelper.convertToMap( + XContentType.fromMediaType(response.getEntity().getContentType().getValue()).xContent(), + is, + true + ); + } + } + +} diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/NodesCapabilitiesUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/NodesCapabilitiesUpgradeIT.java index 2acaf33c2130c..6ddb59a653716 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/NodesCapabilitiesUpgradeIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/NodesCapabilitiesUpgradeIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedRollingUpgradeTestCase.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedRollingUpgradeTestCase.java index d5f645c387d61..a20981a119d8f 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedRollingUpgradeTestCase.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedRollingUpgradeTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; @@ -54,14 +55,13 @@ public static Iterable parameters() { protected abstract ElasticsearchCluster getUpgradeCluster(); @Before - public void extractOldClusterFeatures() { + public void upgradeNode() throws Exception { + // extract old cluster features if (isOldCluster() && oldClusterTestFeatureService == null) { oldClusterTestFeatureService = testFeatureService; } - } - @Before - public void extractOldIndexVersion() throws Exception { + // extract old index version if (oldIndexVersion == null && upgradedNodes.isEmpty()) { IndexVersion indexVersion = null; // these should all be the same version @@ -92,13 +92,11 @@ public void extractOldIndexVersion() throws Exception { assertThat("Index version could not be read", indexVersion, notNullValue()); oldIndexVersion = indexVersion; } - } - @Before - public void upgradeNode() throws Exception { // Skip remaining tests if upgrade failed assumeFalse("Cluster upgrade failed", upgradeFailed); + // finally, upgrade node if (upgradedNodes.size() < requestedUpgradedNodes) { closeClients(); // we might be running a specific upgrade test by itself - check previous nodes too diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SnapshotBasedRecoveryIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SnapshotBasedRecoveryIT.java index 2ee668c0e3fe1..6f4c37f9e56a7 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SnapshotBasedRecoveryIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SnapshotBasedRecoveryIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SystemIndexMappingUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SystemIndexMappingUpgradeIT.java new file mode 100644 index 0000000000000..042d0cc9023bd --- /dev/null +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SystemIndexMappingUpgradeIT.java @@ -0,0 +1,168 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.upgrades; + +import org.elasticsearch.Build; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.core.UpdateForV9; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.cluster.util.Version; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ObjectPath; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.TestRule; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +import static org.elasticsearch.test.LambdaMatchers.transformedMatch; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.anEmptyMap; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.everyItem; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.not; + +public class SystemIndexMappingUpgradeIT extends ESRestTestCase { + + private static final String OLD_CLUSTER_VERSION = System.getProperty("tests.old_cluster_version"); + private static final TemporaryFolder repoDirectory = new TemporaryFolder(); + + private static final ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .version(Version.fromString(OLD_CLUSTER_VERSION)) + .nodes(3) + .setting("path.repo", new Supplier<>() { + @Override + @SuppressForbidden(reason = "TemporaryFolder only has io.File methods, not nio.File") + public String get() { + return repoDirectory.getRoot().getPath(); + } + }) + .setting("xpack.security.enabled", "false") + .build(); + + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(repoDirectory).around(cluster); + + private String testRestCluster = cluster.getHttpAddresses(); + + @Override + protected String getTestRestCluster() { + return testRestCluster; + } + + @Override + protected final Settings restClientSettings() { + return Settings.builder() + .put(super.restClientSettings()) + // increase the timeout here to 90 seconds to handle long waits for a green + // cluster health. the waits for green need to be longer than a minute to + // account for delayed shards + .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "90s") + .build(); + } + + private static String getNodesAttribute(ObjectPath objectPath, String nodeId, String attribute) { + try { + return objectPath.evaluate("nodes." + nodeId + "." + attribute); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static List getUpgradedNodesAddresses() throws IOException { + Response response = client().performRequest(new Request("GET", "_nodes")); + ObjectPath objectPath = ObjectPath.createFromResponse(response); + Map nodesAsMap = objectPath.evaluate("nodes"); + return nodesAsMap.keySet() + .stream() + .filter(id -> getNodesAttribute(objectPath, id, "version").equals(Build.current().version())) // nodes on current version + .map(id -> getNodesAttribute(objectPath, id, "http.publish_address")) + .toList(); + } + + @SuppressWarnings("unchecked") + private List> fieldAsObjectList(Map objectMap, String field) { + return (List>) (objectMap.get(field)); + } + + @SuppressWarnings("unchecked") + private Map fieldAsObject(Map objectMap, String field) { + return (Map) (objectMap.get(field)); + } + + @UpdateForV9() + public void testGrowShrinkUpgradeUpdatesSystemIndexMapping() throws Exception { + /* + * From 8.11, CompatibilityVersions holds a map of system index names to their mappings versions, alongside the transport version. + * See https://github.com/elastic/elasticsearch/pull/99307 + * The problem was fixed in 8.16.1/8.17, so we want to test upgrade from a cluster pre-8.11 to a cluster post-8.16.0 + * For testing the first condition, we use a synthetic cluster feature. The second condition is implied, as the fix and this test + * are applied only to post-8.16.0 + */ + assumeFalse( + "Testing upgrades from before CompatibilityVersions held mapping versions in cluster state", + testFeatureService.clusterHasFeature("gte_v8.11.0") + ); + + // Upgrade node 0 and 1 to the current version, leave node 2 to the BwC version + logger.info("Upgrading node 0 to version {}", Version.CURRENT); + cluster.upgradeNodeToVersion(0, Version.CURRENT); + + logger.info("Upgrading node 1 to version {}", Version.CURRENT); + cluster.upgradeNodeToVersion(1, Version.CURRENT); + + // Query the nodes, ensure we do _not_ have node versions in the answer, or if we do, mappings are empty + Map nodesVersions = entityAsMap(client().performRequest(new Request("GET", "/_cluster/state"))); + assertThat( + nodesVersions, + anyOf( + not(hasKey("nodes_versions")), + transformedMatch( + x -> fieldAsObjectList(x, "nodes_versions"), + everyItem(transformedMatch(item -> fieldAsObject(item, "mappings_versions"), anEmptyMap())) + ) + ) + ); + + var upgradedNodes = getUpgradedNodesAddresses(); + + // Stop the last "old" node + cluster.stopNode(2, false); + + // Ensure we talk only to the 2 live, upgraded nodes + closeClients(); + testRestCluster = String.join(",", upgradedNodes); + initClient(); + + assertBusy(() -> { + var newNodesVersions = entityAsMap(client().performRequest(new Request("GET", "/_cluster/state"))); + assertThat( + newNodesVersions, + allOf( + hasKey("nodes_versions"), + transformedMatch( + x -> fieldAsObjectList(x, "nodes_versions"), + everyItem(transformedMatch(item -> fieldAsObject(item, "mappings_versions"), not(anEmptyMap()))) + ) + ) + ); + }); + } +} diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SystemIndicesUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SystemIndicesUpgradeIT.java index a2e3b03c9036f..dd8ecdd82ca7b 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SystemIndicesUpgradeIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SystemIndicesUpgradeIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TsdbIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TsdbIT.java index 2889885f83984..6744c84f29d0f 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TsdbIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TsdbIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; @@ -32,7 +33,7 @@ public TsdbIT(@Name("upgradedNodes") int upgradedNodes) { super(upgradedNodes); } - private static final String TEMPLATE = """ + static final String TEMPLATE = """ { "settings":{ "index": { @@ -288,7 +289,7 @@ private static void assertSearch(String dataStreamName, int expectedHitCount) th assertThat(ObjectPath.evaluate(responseBody, "hits.total.value"), equalTo(expectedHitCount)); } - private static String formatInstant(Instant instant) { + static String formatInstant(Instant instant) { return DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(instant); } diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TsdbIndexingRollingUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TsdbIndexingRollingUpgradeIT.java new file mode 100644 index 0000000000000..ac55f489a12aa --- /dev/null +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TsdbIndexingRollingUpgradeIT.java @@ -0,0 +1,198 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.upgrades; + +import com.carrotsearch.randomizedtesting.annotations.Name; + +import org.elasticsearch.client.Request; +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.test.rest.ObjectPath; + +import java.time.Instant; +import java.util.Locale; +import java.util.Map; + +import static org.elasticsearch.upgrades.LogsIndexModeRollingUpgradeIT.getWriteBackingIndex; +import static org.elasticsearch.upgrades.LogsdbIndexingRollingUpgradeIT.createTemplate; +import static org.elasticsearch.upgrades.LogsdbIndexingRollingUpgradeIT.getIndexSettingsWithDefaults; +import static org.elasticsearch.upgrades.LogsdbIndexingRollingUpgradeIT.startTrial; +import static org.elasticsearch.upgrades.TsdbIT.TEMPLATE; +import static org.elasticsearch.upgrades.TsdbIT.formatInstant; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.notNullValue; + +public class TsdbIndexingRollingUpgradeIT extends AbstractRollingUpgradeTestCase { + + static String BULK_ITEM_TEMPLATE = + """ + {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "$name", "uid":"$uid", "ip": "$ip", "network": {"tx": $tx, "rx": $rx}}}} + """; + + public TsdbIndexingRollingUpgradeIT(@Name("upgradedNodes") int upgradedNodes) { + super(upgradedNodes); + } + + public void testIndexing() throws Exception { + assumeTrue("test relies on index.mapping.source.mode setting", getOldClusterTestVersion().onOrAfter("8.16.0")); + String dataStreamName = "k9s"; + if (isOldCluster()) { + startTrial(); + createTemplate(dataStreamName, getClass().getSimpleName().toLowerCase(Locale.ROOT), TEMPLATE); + + Instant startTime = Instant.now().minusSeconds(60 * 60); + bulkIndex(dataStreamName, 4, 1024, startTime); + + String firstBackingIndex = getWriteBackingIndex(client(), dataStreamName, 0); + var settings = (Map) getIndexSettingsWithDefaults(firstBackingIndex).get(firstBackingIndex); + assertThat(((Map) settings.get("settings")).get("index.mode"), equalTo("time_series")); + assertThat(((Map) settings.get("defaults")).get("index.mapping.source.mode"), equalTo("SYNTHETIC")); + + var mapping = getIndexMappingAsMap(firstBackingIndex); + assertThat( + "incorrect k8s.pod.name field in mapping:" + mapping, + "keyword", + equalTo(ObjectPath.evaluate(mapping, "properties.k8s.properties.pod.properties.name.type")) + ); + + ensureGreen(dataStreamName); + search(dataStreamName); + query(dataStreamName); + } else if (isMixedCluster()) { + Instant startTime = Instant.now().minusSeconds(60 * 30); + bulkIndex(dataStreamName, 4, 1024, startTime); + + ensureGreen(dataStreamName); + search(dataStreamName); + query(dataStreamName); + } else if (isUpgradedCluster()) { + ensureGreen(dataStreamName); + Instant startTime = Instant.now(); + bulkIndex(dataStreamName, 4, 1024, startTime); + search(dataStreamName); + query(dataStreamName); + + var forceMergeRequest = new Request("POST", "/" + dataStreamName + "/_forcemerge"); + forceMergeRequest.addParameter("max_num_segments", "1"); + assertOK(client().performRequest(forceMergeRequest)); + + ensureGreen(dataStreamName); + search(dataStreamName); + query(dataStreamName); + } + } + + static void bulkIndex(String dataStreamName, int numRequest, int numDocs, Instant startTime) throws Exception { + for (int i = 0; i < numRequest; i++) { + var bulkRequest = new Request("POST", "/" + dataStreamName + "/_bulk"); + StringBuilder requestBody = new StringBuilder(); + for (int j = 0; j < numDocs; j++) { + String podName = "pod" + j % 5; // Not realistic, but makes asserting search / query response easier. + String podUid = randomUUID(); + String podIp = NetworkAddress.format(randomIp(true)); + long podTx = randomLong(); + long podRx = randomLong(); + + requestBody.append("{\"create\": {}}"); + requestBody.append('\n'); + requestBody.append( + BULK_ITEM_TEMPLATE.replace("$now", formatInstant(startTime)) + .replace("$name", podName) + .replace("$uid", podUid) + .replace("$ip", podIp) + .replace("$tx", Long.toString(podTx)) + .replace("$rx", Long.toString(podRx)) + ); + requestBody.append('\n'); + + startTime = startTime.plusMillis(1); + } + bulkRequest.setJsonEntity(requestBody.toString()); + bulkRequest.addParameter("refresh", "true"); + var response = client().performRequest(bulkRequest); + assertOK(response); + var responseBody = entityAsMap(response); + assertThat("errors in response:\n " + responseBody, responseBody.get("errors"), equalTo(false)); + } + } + + void search(String dataStreamName) throws Exception { + var searchRequest = new Request("POST", "/" + dataStreamName + "/_search"); + searchRequest.addParameter("pretty", "true"); + searchRequest.setJsonEntity(""" + { + "size": 0, + "aggs": { + "pod_name": { + "terms": { + "field": "k8s.pod.name", + "order": { "_key": "asc" } + }, + "aggs": { + "max_tx": { + "max": { + "field": "k8s.pod.network.tx" + } + }, + "max_rx": { + "max": { + "field": "k8s.pod.network.rx" + } + } + } + } + } + } + """); + var response = client().performRequest(searchRequest); + assertOK(response); + var responseBody = entityAsMap(response); + + Integer totalCount = ObjectPath.evaluate(responseBody, "hits.total.value"); + assertThat(totalCount, greaterThanOrEqualTo(4096)); + String key = ObjectPath.evaluate(responseBody, "aggregations.pod_name.buckets.0.key"); + assertThat(key, equalTo("pod0")); + Integer docCount = ObjectPath.evaluate(responseBody, "aggregations.pod_name.buckets.0.doc_count"); + assertThat(docCount, greaterThan(0)); + Double maxTx = ObjectPath.evaluate(responseBody, "aggregations.pod_name.buckets.0.max_tx.value"); + assertThat(maxTx, notNullValue()); + Double maxRx = ObjectPath.evaluate(responseBody, "aggregations.pod_name.buckets.0.max_rx.value"); + assertThat(maxRx, notNullValue()); + } + + void query(String dataStreamName) throws Exception { + var queryRequest = new Request("POST", "/_query"); + queryRequest.addParameter("pretty", "true"); + queryRequest.setJsonEntity(""" + { + "query": "FROM $ds | STATS max(k8s.pod.network.rx), max(k8s.pod.network.tx) BY k8s.pod.name | SORT k8s.pod.name | LIMIT 5" + } + """.replace("$ds", dataStreamName)); + var response = client().performRequest(queryRequest); + assertOK(response); + var responseBody = entityAsMap(response); + + String column1 = ObjectPath.evaluate(responseBody, "columns.0.name"); + String column2 = ObjectPath.evaluate(responseBody, "columns.1.name"); + String column3 = ObjectPath.evaluate(responseBody, "columns.2.name"); + assertThat(column1, equalTo("max(k8s.pod.network.rx)")); + assertThat(column2, equalTo("max(k8s.pod.network.tx)")); + assertThat(column3, equalTo("k8s.pod.name")); + + String key = ObjectPath.evaluate(responseBody, "values.0.2"); + assertThat(key, equalTo("pod0")); + Long maxRx = ObjectPath.evaluate(responseBody, "values.0.0"); + assertThat(maxRx, notNullValue()); + Long maxTx = ObjectPath.evaluate(responseBody, "values.0.1"); + assertThat(maxTx, notNullValue()); + } + +} diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/UpgradeWithOldIndexSettingsIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/UpgradeWithOldIndexSettingsIT.java index ae75069fa564d..0d35490e86721 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/UpgradeWithOldIndexSettingsIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/UpgradeWithOldIndexSettingsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/VectorSearchIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/VectorSearchIT.java index 544cd652741c8..df46ed0379980 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/VectorSearchIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/VectorSearchIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/XPackIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/XPackIT.java index 6379a8875dfb4..8f4f53848da8d 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/XPackIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/XPackIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.upgrades; diff --git a/qa/smoke-test-http/build.gradle b/qa/smoke-test-http/build.gradle index 6347e18c31e20..8331cb7c8b1e8 100644 --- a/qa/smoke-test-http/build.gradle +++ b/qa/smoke-test-http/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.legacy-java-rest-test' diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/action/support/tasks/RestListTasksCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/action/support/tasks/RestListTasksCancellationIT.java index d4d3f6143ff3c..d563e893f6ebc 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/action/support/tasks/RestListTasksCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/action/support/tasks/RestListTasksCancellationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.tasks; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/AutoCreateIndexIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/AutoCreateIndexIT.java index 0b33275431cdb..ce897664bd8fc 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/AutoCreateIndexIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/AutoCreateIndexIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BlockedSearcherRestCancellationTestCase.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BlockedSearcherRestCancellationTestCase.java index 250fe5a3a79fb..c7e3a5b1c9a77 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BlockedSearcherRestCancellationTestCase.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BlockedSearcherRestCancellationTestCase.java @@ -1,17 +1,20 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.Cancellable; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; @@ -28,6 +31,8 @@ import org.elasticsearch.index.shard.IndexShardTestCase; import org.elasticsearch.index.translog.TranslogStats; import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; import org.elasticsearch.plugins.EnginePlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.tasks.Task; @@ -140,6 +145,12 @@ public List> getSettings() { private static class SearcherBlockingEngine extends ReadOnlyEngine { + // using a specialized logger for this case because and "logger" means "Engine#logger" + // (relates investigation into https://github.com/elastic/elasticsearch/issues/88201) + private static final Logger blockedSearcherRestCancellationTestCaseLogger = LogManager.getLogger( + BlockedSearcherRestCancellationTestCase.class + ); + final Semaphore searcherBlock = new Semaphore(1); SearcherBlockingEngine(EngineConfig config) { @@ -148,12 +159,36 @@ private static class SearcherBlockingEngine extends ReadOnlyEngine { @Override public Searcher acquireSearcher(String source, SearcherScope scope, Function wrapper) throws EngineException { + if (blockedSearcherRestCancellationTestCaseLogger.isDebugEnabled()) { + blockedSearcherRestCancellationTestCaseLogger.debug( + Strings.format( + "in acquireSearcher for shard [%s] on thread [%s], availablePermits=%d", + config().getShardId(), + Thread.currentThread().getName(), + searcherBlock.availablePermits() + ), + new ElasticsearchException("stack trace") + ); + } + try { searcherBlock.acquire(); } catch (InterruptedException e) { throw new AssertionError(e); } searcherBlock.release(); + + if (blockedSearcherRestCancellationTestCaseLogger.isDebugEnabled()) { + blockedSearcherRestCancellationTestCaseLogger.debug( + Strings.format( + "continuing in acquireSearcher for shard [%s] on thread [%s], availablePermits=%d", + config().getShardId(), + Thread.currentThread().getName(), + searcherBlock.availablePermits() + ) + ); + } + return super.acquireSearcher(source, scope, wrapper); } } diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BulkRestIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BulkRestIT.java new file mode 100644 index 0000000000000..3faa88339f0a3 --- /dev/null +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BulkRestIT.java @@ -0,0 +1,224 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.http; + +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.entity.ContentType; +import org.elasticsearch.action.bulk.IncrementalBulkService; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xcontent.json.JsonXContent; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.rest.RestStatus.BAD_REQUEST; +import static org.elasticsearch.rest.RestStatus.OK; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 2, numClientNodes = 0) +public class BulkRestIT extends HttpSmokeTestCase { + + @Override + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal, otherSettings)) + .put(IncrementalBulkService.INCREMENTAL_BULK.getKey(), seventyFivePercentOfTheTime()) + .build(); + } + + private static boolean seventyFivePercentOfTheTime() { + return (randomBoolean() && randomBoolean()) == false; + } + + public void testBulkUriMatchingDoesNotMatchBulkCapabilitiesApi() throws IOException { + Request request = new Request("GET", "/_capabilities?method=GET&path=%2F_bulk&capabilities=failure_store_status&pretty"); + Response response = getRestClient().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + } + + public void testBulkMissingBody() throws IOException { + Request request = new Request(randomBoolean() ? "POST" : "PUT", "/_bulk"); + request.setJsonEntity(""); + ResponseException responseException = expectThrows(ResponseException.class, () -> getRestClient().performRequest(request)); + assertEquals(400, responseException.getResponse().getStatusLine().getStatusCode()); + assertThat(responseException.getMessage(), containsString("request body is required")); + } + + public void testBulkInvalidIndexNameString() throws IOException { + Request request = new Request("POST", "/_bulk"); + + byte[] bytes1 = "{\"create\":{\"_index\":\"".getBytes(StandardCharsets.UTF_8); + byte[] bytes2 = new byte[] { (byte) 0xfe, (byte) 0xfe, (byte) 0xff, (byte) 0xff }; + byte[] bytes3 = "\",\"_id\":\"1\"}}\n{\"field\":1}\n\r\n".getBytes(StandardCharsets.UTF_8); + byte[] bulkBody = new byte[bytes1.length + bytes2.length + bytes3.length]; + System.arraycopy(bytes1, 0, bulkBody, 0, bytes1.length); + System.arraycopy(bytes2, 0, bulkBody, bytes1.length, bytes2.length); + System.arraycopy(bytes3, 0, bulkBody, bytes1.length + bytes2.length, bytes3.length); + + request.setEntity(new ByteArrayEntity(bulkBody, ContentType.APPLICATION_JSON)); + + ResponseException responseException = expectThrows(ResponseException.class, () -> getRestClient().performRequest(request)); + assertThat(responseException.getResponse().getStatusLine().getStatusCode(), equalTo(BAD_REQUEST.getStatus())); + assertThat(responseException.getMessage(), containsString("x_content_parse_exception")); + assertThat(responseException.getMessage(), containsString("Invalid UTF-8")); + } + + public void testBulkRequestBodyImproperlyTerminated() throws IOException { + Request request = new Request(randomBoolean() ? "POST" : "PUT", "/_bulk"); + // missing final line of the bulk body. cannot process + request.setJsonEntity( + "{\"index\":{\"_index\":\"index_name\",\"_id\":\"1\"}}\n" + + "{\"field\":1}\n" + + "{\"index\":{\"_index\":\"index_name\",\"_id\":\"2\"}" + ); + ResponseException responseException = expectThrows(ResponseException.class, () -> getRestClient().performRequest(request)); + assertEquals(400, responseException.getResponse().getStatusLine().getStatusCode()); + assertThat(responseException.getMessage(), containsString("The bulk request must be terminated by a newline")); + } + + public void testBulkRequest() throws IOException { + Request createRequest = new Request("PUT", "/index_name"); + createRequest.setJsonEntity(""" + { + "settings": { + "index": { + "number_of_shards": 1, + "number_of_replicas": 1, + "write.wait_for_active_shards": 2 + } + } + }"""); + final Response indexCreatedResponse = getRestClient().performRequest(createRequest); + assertThat(indexCreatedResponse.getStatusLine().getStatusCode(), equalTo(OK.getStatus())); + + Request firstBulkRequest = new Request("POST", "/index_name/_bulk"); + + String bulkBody = "{\"index\":{\"_index\":\"index_name\",\"_id\":\"1\"}}\n" + + "{\"field\":1}\n" + + "{\"index\":{\"_index\":\"index_name\",\"_id\":\"2\"}}\n" + + "{\"field\":1}\n" + + "\r\n"; + + firstBulkRequest.setJsonEntity(bulkBody); + + final Response indexSuccessFul = getRestClient().performRequest(firstBulkRequest); + assertThat(indexSuccessFul.getStatusLine().getStatusCode(), equalTo(OK.getStatus())); + + sendLargeBulk(); + } + + public void testBulkWithIncrementalDisabled() throws IOException { + Request createRequest = new Request("PUT", "/index_name"); + createRequest.setJsonEntity(""" + { + "settings": { + "index": { + "number_of_shards": 1, + "number_of_replicas": 1, + "write.wait_for_active_shards": 2 + } + } + }"""); + final Response indexCreatedResponse = getRestClient().performRequest(createRequest); + assertThat(indexCreatedResponse.getStatusLine().getStatusCode(), equalTo(OK.getStatus())); + + Request firstBulkRequest = new Request("POST", "/index_name/_bulk"); + + String bulkBody = "{\"index\":{\"_index\":\"index_name\",\"_id\":\"1\"}}\n" + + "{\"field\":1}\n" + + "{\"index\":{\"_index\":\"index_name\",\"_id\":\"2\"}}\n" + + "{\"field\":1}\n" + + "\r\n"; + + firstBulkRequest.setJsonEntity(bulkBody); + + final Response indexSuccessFul = getRestClient().performRequest(firstBulkRequest); + assertThat(indexSuccessFul.getStatusLine().getStatusCode(), equalTo(OK.getStatus())); + + updateClusterSettings(Settings.builder().put(IncrementalBulkService.INCREMENTAL_BULK.getKey(), false)); + + internalCluster().getInstances(IncrementalBulkService.class).forEach(i -> i.setForTests(false)); + + try { + sendLargeBulk(); + } finally { + internalCluster().getInstances(IncrementalBulkService.class).forEach(i -> i.setForTests(true)); + updateClusterSettings(Settings.builder().put(IncrementalBulkService.INCREMENTAL_BULK.getKey(), (String) null)); + } + } + + public void testMalformedActionLineBulk() throws IOException { + Request createRequest = new Request("PUT", "/index_name"); + createRequest.setJsonEntity(""" + { + "settings": { + "index": { + "number_of_shards": 1, + "number_of_replicas": 1, + "write.wait_for_active_shards": 2 + } + } + }"""); + final Response indexCreatedResponse = getRestClient().performRequest(createRequest); + assertThat(indexCreatedResponse.getStatusLine().getStatusCode(), equalTo(OK.getStatus())); + + Request bulkRequest = new Request("POST", "/index_name/_bulk"); + + final StringBuilder bulk = new StringBuilder(); + bulk.append("{\"index\":{\"_index\":\"index_name\"}}\n"); + bulk.append("{\"field\":1}\n"); + bulk.append("{}\n"); + bulk.append("\r\n"); + + bulkRequest.setJsonEntity(bulk.toString()); + + expectThrows(ResponseException.class, () -> getRestClient().performRequest(bulkRequest)); + } + + @SuppressWarnings("unchecked") + private static void sendLargeBulk() throws IOException { + Request bulkRequest = new Request("POST", "/index_name/_bulk"); + + final StringBuilder bulk = new StringBuilder(); + bulk.append("{\"delete\":{\"_index\":\"index_name\",\"_id\":\"1\"}}\n"); + int updates = 0; + for (int i = 0; i < 1000; i++) { + bulk.append("{\"index\":{\"_index\":\"index_name\"}}\n"); + bulk.append("{\"field\":").append(i).append("}\n"); + if (randomBoolean() && randomBoolean() && randomBoolean() && randomBoolean()) { + ++updates; + bulk.append("{\"update\":{\"_index\":\"index_name\",\"_id\":\"2\"}}\n"); + bulk.append("{\"doc\":{\"field\":").append(i).append("}}\n"); + } + } + bulk.append("\r\n"); + + bulkRequest.setJsonEntity(bulk.toString()); + + final Response bulkResponse = getRestClient().performRequest(bulkRequest); + assertThat(bulkResponse.getStatusLine().getStatusCode(), equalTo(OK.getStatus())); + Map responseMap = XContentHelper.convertToMap( + JsonXContent.jsonXContent, + bulkResponse.getEntity().getContent(), + true + ); + + assertFalse((Boolean) responseMap.get("errors")); + assertThat(((List) responseMap.get("items")).size(), equalTo(1001 + updates)); + } +} diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterHealthRestCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterHealthRestCancellationIT.java index 5c3fb155588a7..c2a40143b6eb1 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterHealthRestCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterHealthRestCancellationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; @@ -18,25 +19,15 @@ import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; -import org.elasticsearch.test.junit.annotations.TestIssueLogging; import java.util.concurrent.CancellationException; import java.util.concurrent.CyclicBarrier; -import java.util.concurrent.TimeUnit; import static org.elasticsearch.action.support.ActionTestUtils.wrapAsRestResponseListener; import static org.elasticsearch.test.TaskAssertions.assertAllCancellableTasksAreCancelled; -import static org.elasticsearch.test.TaskAssertions.awaitTaskWithPrefixOnMaster; public class ClusterHealthRestCancellationIT extends HttpSmokeTestCase { - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/100062") - @TestIssueLogging( - issueUrl = "https://github.com/elastic/elasticsearch/issues/100062", - value = "org.elasticsearch.test.TaskAssertions:TRACE" - + ",org.elasticsearch.cluster.service.MasterService:TRACE" - + ",org.elasticsearch.tasks.TaskManager:TRACE" - ) public void testClusterHealthRestCancellation() throws Exception { final var barrier = new CyclicBarrier(2); @@ -46,18 +37,7 @@ public void testClusterHealthRestCancellation() throws Exception { @Override public ClusterState execute(ClusterState currentState) { safeAwait(barrier); - // safeAwait(barrier); - - // temporarily lengthen timeout on safeAwait while investigating #100062 - try { - barrier.await(60, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new AssertionError("unexpected", e); - } catch (Exception e) { - throw new AssertionError("unexpected", e); - } - + safeAwait(barrier); return currentState; } @@ -71,12 +51,23 @@ public void onFailure(Exception e) { clusterHealthRequest.addParameter("wait_for_events", Priority.LANGUID.toString()); final PlainActionFuture future = new PlainActionFuture<>(); - logger.info("--> sending cluster state request"); + logger.info("--> sending cluster health request"); final Cancellable cancellable = getRestClient().performRequestAsync(clusterHealthRequest, wrapAsRestResponseListener(future)); safeAwait(barrier); - awaitTaskWithPrefixOnMaster(TransportClusterHealthAction.NAME); + // wait until the health request is waiting on the (blocked) master service + assertBusy( + () -> assertTrue( + internalCluster().getCurrentMasterNodeInstance(ClusterService.class) + .getMasterService() + .pendingTasks() + .stream() + .anyMatch( + pendingClusterTask -> pendingClusterTask.source().string().equals("cluster_health (wait_for_events [LANGUID])") + ) + ) + ); logger.info("--> cancelling cluster health request"); cancellable.cancel(); diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterInfoRestCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterInfoRestCancellationIT.java index b6bd16e12c8ab..26487a2ef6a29 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterInfoRestCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterInfoRestCancellationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterStateRestCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterStateRestCancellationIT.java index d51ac416d2759..69a8dce5a2e99 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterStateRestCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterStateRestCancellationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterStatsRestCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterStatsRestCancellationIT.java index 53f4ae5c1bc37..673e2c384f66e 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterStatsRestCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterStatsRestCancellationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/CorsNotSetIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/CorsNotSetIT.java index 2cfa57439767a..5bc73bd67a9ef 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/CorsNotSetIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/CorsNotSetIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/CorsRegexIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/CorsRegexIT.java index 6ab236a89039b..d6dd9a51f3539 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/CorsRegexIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/CorsRegexIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/DanglingIndicesRestIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/DanglingIndicesRestIT.java index d04c8802635d3..dcb1b357c9211 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/DanglingIndicesRestIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/DanglingIndicesRestIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java index 93ca3876cd8df..6a640b3b7c0a3 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java index 25575fac50543..4fcbb2ef4c6a6 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HealthRestCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HealthRestCancellationIT.java index dc298fc616506..1a994df0e018a 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HealthRestCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HealthRestCancellationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpCompressionIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpCompressionIT.java index 2e41e311073a9..981190f22ce28 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpCompressionIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpCompressionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpSmokeTestCase.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpSmokeTestCase.java index 4536e2ee25fd6..4d9f25151a5e3 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpSmokeTestCase.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpSmokeTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpStatsIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpStatsIT.java index ac1bde443f703..5953055724556 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpStatsIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpStatsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IndexingPressureRestIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IndexingPressureRestIT.java index f7e3e98075726..a19ce67368dff 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IndexingPressureRestIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IndexingPressureRestIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; @@ -41,7 +42,8 @@ public class IndexingPressureRestIT extends HttpSmokeTestCase { protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { return Settings.builder() .put(super.nodeSettings(nodeOrdinal, otherSettings)) - .put(IndexingPressure.MAX_INDEXING_BYTES.getKey(), "1KB") + .put(IndexingPressure.MAX_COORDINATING_BYTES.getKey(), "1KB") + .put(IndexingPressure.MAX_PRIMARY_BYTES.getKey(), "1KB") .put(unboundedWriteQueue) .build(); } diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IndicesSegmentsRestCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IndicesSegmentsRestCancellationIT.java index d2ab3b5a01636..92fde6d7765cc 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IndicesSegmentsRestCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IndicesSegmentsRestCancellationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; @@ -11,12 +12,23 @@ import org.apache.http.client.methods.HttpGet; import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsAction; import org.elasticsearch.client.Request; +import org.elasticsearch.test.junit.annotations.TestIssueLogging; public class IndicesSegmentsRestCancellationIT extends BlockedSearcherRestCancellationTestCase { + @TestIssueLogging( + issueUrl = "https://github.com/elastic/elasticsearch/issues/88201", + value = "org.elasticsearch.http.BlockedSearcherRestCancellationTestCase:DEBUG" + + ",org.elasticsearch.transport.TransportService:TRACE" + ) public void testIndicesSegmentsRestCancellation() throws Exception { runTest(new Request(HttpGet.METHOD_NAME, "/_segments"), IndicesSegmentsAction.NAME); } + @TestIssueLogging( + issueUrl = "https://github.com/elastic/elasticsearch/issues/88201", + value = "org.elasticsearch.http.BlockedSearcherRestCancellationTestCase:DEBUG" + + ",org.elasticsearch.transport.TransportService:TRACE" + ) public void testCatSegmentsRestCancellation() throws Exception { runTest(new Request(HttpGet.METHOD_NAME, "/_cat/segments"), IndicesSegmentsAction.NAME); } diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IndicesStatsRestCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IndicesStatsRestCancellationIT.java index 53ef4816729a9..7bb54f0b760f5 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IndicesStatsRestCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IndicesStatsRestCancellationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/NoHandlerIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/NoHandlerIT.java index 386f49944b079..75195404c2969 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/NoHandlerIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/NoHandlerIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/NodeStatsRestCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/NodeStatsRestCancellationIT.java index 5800c60b3b8a4..ed1f084bbbf7c 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/NodeStatsRestCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/NodeStatsRestCancellationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/PointInTimeIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/PointInTimeIT.java index 5f6982b2f680e..36c0da24529f2 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/PointInTimeIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/PointInTimeIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/PrevalidateNodeRemovalRestIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/PrevalidateNodeRemovalRestIT.java index ae1764310a34d..82899946aa1fb 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/PrevalidateNodeRemovalRestIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/PrevalidateNodeRemovalRestIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ResponseHeaderPluginIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ResponseHeaderPluginIT.java index 62a67201886ed..d305ef26edaef 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ResponseHeaderPluginIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ResponseHeaderPluginIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java index acfd870576b21..d6254d091a868 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestClusterInfoActionCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestClusterInfoActionCancellationIT.java index 896da65fa83dd..322c055de140b 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestClusterInfoActionCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestClusterInfoActionCancellationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; @@ -18,7 +19,6 @@ import org.elasticsearch.client.Response; import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ack.AckedRequest; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.block.ClusterBlocks; @@ -101,22 +101,9 @@ private void runTest(String actionName, String endpoint) throws Exception { private void updateClusterState(Function transformationFn) { final TimeValue timeout = TimeValue.timeValueSeconds(10); - - final AckedRequest ackedRequest = new AckedRequest() { - @Override - public TimeValue ackTimeout() { - return timeout; - } - - @Override - public TimeValue masterNodeTimeout() { - return timeout; - } - }; - PlainActionFuture future = new PlainActionFuture<>(); internalCluster().getAnyMasterNodeInstance(ClusterService.class) - .submitUnbatchedStateUpdateTask("get_mappings_cancellation_test", new AckedClusterStateUpdateTask(ackedRequest, future) { + .submitUnbatchedStateUpdateTask("get_mappings_cancellation_test", new AckedClusterStateUpdateTask(timeout, timeout, future) { @Override public ClusterState execute(ClusterState currentState) throws Exception { return transformationFn.apply(currentState); diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestHttpResponseHeadersIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestHttpResponseHeadersIT.java index 693475e70d824..2ac07a3851ee5 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestHttpResponseHeadersIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestHttpResponseHeadersIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RolloverRestCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RolloverRestCancellationIT.java index 055a9a29519c3..cf5335c110dcf 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RolloverRestCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RolloverRestCancellationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SearchErrorTraceIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SearchErrorTraceIT.java new file mode 100644 index 0000000000000..6f9ab8ccdfdec --- /dev/null +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SearchErrorTraceIT.java @@ -0,0 +1,175 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.http; + +import org.apache.http.entity.ContentType; +import org.apache.http.nio.entity.NByteArrayEntity; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.search.MultiSearchRequest; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.client.Request; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.transport.TransportMessageListener; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xcontent.XContentType; +import org.junit.Before; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.elasticsearch.index.query.QueryBuilders.simpleQueryStringQuery; + +public class SearchErrorTraceIT extends HttpSmokeTestCase { + private AtomicBoolean hasStackTrace; + + @Before + private void setupMessageListener() { + internalCluster().getDataNodeInstances(TransportService.class).forEach(ts -> { + ts.addMessageListener(new TransportMessageListener() { + @Override + public void onResponseSent(long requestId, String action, Exception error) { + TransportMessageListener.super.onResponseSent(requestId, action, error); + if (action.startsWith("indices:data/read/search")) { + Optional throwable = ExceptionsHelper.unwrapCausesAndSuppressed( + error, + t -> t.getStackTrace().length > 0 + ); + hasStackTrace.set(throwable.isPresent()); + } + } + }); + }); + } + + private void setupIndexWithDocs() { + createIndex("test1", "test2"); + indexRandom( + true, + prepareIndex("test1").setId("1").setSource("field", "foo"), + prepareIndex("test2").setId("10").setSource("field", 5) + ); + refresh(); + } + + public void testSearchFailingQueryErrorTraceDefault() throws IOException { + hasStackTrace = new AtomicBoolean(); + setupIndexWithDocs(); + + Request searchRequest = new Request("POST", "/_search"); + searchRequest.setJsonEntity(""" + { + "query": { + "simple_query_string" : { + "query": "foo", + "fields": ["field"] + } + } + } + """); + getRestClient().performRequest(searchRequest); + assertFalse(hasStackTrace.get()); + } + + public void testSearchFailingQueryErrorTraceTrue() throws IOException { + hasStackTrace = new AtomicBoolean(); + setupIndexWithDocs(); + + Request searchRequest = new Request("POST", "/_search"); + searchRequest.setJsonEntity(""" + { + "query": { + "simple_query_string" : { + "query": "foo", + "fields": ["field"] + } + } + } + """); + searchRequest.addParameter("error_trace", "true"); + getRestClient().performRequest(searchRequest); + assertTrue(hasStackTrace.get()); + } + + public void testSearchFailingQueryErrorTraceFalse() throws IOException { + hasStackTrace = new AtomicBoolean(); + setupIndexWithDocs(); + + Request searchRequest = new Request("POST", "/_search"); + searchRequest.setJsonEntity(""" + { + "query": { + "simple_query_string" : { + "query": "foo", + "fields": ["field"] + } + } + } + """); + searchRequest.addParameter("error_trace", "false"); + getRestClient().performRequest(searchRequest); + assertFalse(hasStackTrace.get()); + } + + public void testMultiSearchFailingQueryErrorTraceDefault() throws IOException { + hasStackTrace = new AtomicBoolean(); + setupIndexWithDocs(); + + XContentType contentType = XContentType.JSON; + MultiSearchRequest multiSearchRequest = new MultiSearchRequest().add( + new SearchRequest("test*").source(new SearchSourceBuilder().query(simpleQueryStringQuery("foo").field("field"))) + ); + Request searchRequest = new Request("POST", "/_msearch"); + byte[] requestBody = MultiSearchRequest.writeMultiLineFormat(multiSearchRequest, contentType.xContent()); + searchRequest.setEntity( + new NByteArrayEntity(requestBody, ContentType.create(contentType.mediaTypeWithoutParameters(), (Charset) null)) + ); + getRestClient().performRequest(searchRequest); + assertFalse(hasStackTrace.get()); + } + + public void testMultiSearchFailingQueryErrorTraceTrue() throws IOException { + hasStackTrace = new AtomicBoolean(); + setupIndexWithDocs(); + + XContentType contentType = XContentType.JSON; + MultiSearchRequest multiSearchRequest = new MultiSearchRequest().add( + new SearchRequest("test*").source(new SearchSourceBuilder().query(simpleQueryStringQuery("foo").field("field"))) + ); + Request searchRequest = new Request("POST", "/_msearch"); + byte[] requestBody = MultiSearchRequest.writeMultiLineFormat(multiSearchRequest, contentType.xContent()); + searchRequest.setEntity( + new NByteArrayEntity(requestBody, ContentType.create(contentType.mediaTypeWithoutParameters(), (Charset) null)) + ); + searchRequest.addParameter("error_trace", "true"); + getRestClient().performRequest(searchRequest); + assertTrue(hasStackTrace.get()); + } + + public void testMultiSearchFailingQueryErrorTraceFalse() throws IOException { + hasStackTrace = new AtomicBoolean(); + setupIndexWithDocs(); + + XContentType contentType = XContentType.JSON; + MultiSearchRequest multiSearchRequest = new MultiSearchRequest().add( + new SearchRequest("test*").source(new SearchSourceBuilder().query(simpleQueryStringQuery("foo").field("field"))) + ); + Request searchRequest = new Request("POST", "/_msearch"); + byte[] requestBody = MultiSearchRequest.writeMultiLineFormat(multiSearchRequest, contentType.xContent()); + searchRequest.setEntity( + new NByteArrayEntity(requestBody, ContentType.create(contentType.mediaTypeWithoutParameters(), (Charset) null)) + ); + searchRequest.addParameter("error_trace", "false"); + getRestClient().performRequest(searchRequest); + + assertFalse(hasStackTrace.get()); + } +} diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SearchRestCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SearchRestCancellationIT.java index 73dd1525f8a08..a033582d5defa 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SearchRestCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SearchRestCancellationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SystemIndexRestIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SystemIndexRestIT.java index f130856265a32..2e48860e613ee 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SystemIndexRestIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SystemIndexRestIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/TestResponseHeaderPlugin.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/TestResponseHeaderPlugin.java index 4820202fc4d96..f6e533b12a774 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/TestResponseHeaderPlugin.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/TestResponseHeaderPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/TestResponseHeaderRestAction.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/TestResponseHeaderRestAction.java index 3ae90d71df4c4..a7b4e25ab91ed 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/TestResponseHeaderRestAction.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/TestResponseHeaderRestAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/AbstractSnapshotRestTestCase.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/AbstractSnapshotRestTestCase.java index 12cfdd0cd423b..d3d7695a10a38 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/AbstractSnapshotRestTestCase.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/AbstractSnapshotRestTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.snapshots; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestCatSnapshotsIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestCatSnapshotsIT.java index 681aa3a9e7740..5e3500314db59 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestCatSnapshotsIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestCatSnapshotsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.snapshots; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestGetSnapshotsCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestGetSnapshotsCancellationIT.java index fbaae0961dfcd..5a743a75291c8 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestGetSnapshotsCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestGetSnapshotsCancellationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.snapshots; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestGetSnapshotsIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestGetSnapshotsIT.java index 4c6774988d7ae..b1e28de1a5264 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestGetSnapshotsIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestGetSnapshotsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.snapshots; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestSnapshotsStatusCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestSnapshotsStatusCancellationIT.java index 3c842d67a78f6..dc10aa27b8a0f 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestSnapshotsStatusCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestSnapshotsStatusCancellationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.http.snapshots; diff --git a/qa/smoke-test-ingest-disabled/build.gradle b/qa/smoke-test-ingest-disabled/build.gradle index 5ffefed52b713..fe8904da7c1c1 100644 --- a/qa/smoke-test-ingest-disabled/build.gradle +++ b/qa/smoke-test-ingest-disabled/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.legacy-yaml-rest-test' diff --git a/qa/smoke-test-ingest-disabled/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestIngestDisabledClientYamlTestSuiteIT.java b/qa/smoke-test-ingest-disabled/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestIngestDisabledClientYamlTestSuiteIT.java index 9b9787a0421ed..4c050b776a16a 100644 --- a/qa/smoke-test-ingest-disabled/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestIngestDisabledClientYamlTestSuiteIT.java +++ b/qa/smoke-test-ingest-disabled/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestIngestDisabledClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.smoketest; diff --git a/qa/smoke-test-ingest-with-all-dependencies/build.gradle b/qa/smoke-test-ingest-with-all-dependencies/build.gradle index 6181548757c68..5482fa4f4df85 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/build.gradle +++ b/qa/smoke-test-ingest-with-all-dependencies/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.legacy-yaml-rest-test' diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/AbstractScriptTestCase.java b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/AbstractScriptTestCase.java index eca978353fb0d..8d877bd48c1e3 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/AbstractScriptTestCase.java +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/AbstractScriptTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest; @@ -30,7 +31,7 @@ public abstract class AbstractScriptTestCase extends ESTestCase { @Before public void init() throws Exception { - MustacheScriptEngine engine = new MustacheScriptEngine(); + MustacheScriptEngine engine = new MustacheScriptEngine(Settings.EMPTY); Map engines = Collections.singletonMap(engine.getType(), engine); scriptService = new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/IngestDocumentMustacheIT.java b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/IngestDocumentMustacheIT.java index f5a1839001e5c..0279061302cc7 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/IngestDocumentMustacheIT.java +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/IngestDocumentMustacheIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest; diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/ValueSourceMustacheIT.java b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/ValueSourceMustacheIT.java index df4c5827cebc1..59cc33ffc26fa 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/ValueSourceMustacheIT.java +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/ValueSourceMustacheIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest; diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestIngestWithAllDepsClientYamlTestSuiteIT.java b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestIngestWithAllDepsClientYamlTestSuiteIT.java index 350adaad35c62..9618381440d9d 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestIngestWithAllDepsClientYamlTestSuiteIT.java +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestIngestWithAllDepsClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.smoketest; diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/20_combine_processors.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/20_combine_processors.yml index 9a7444c4ffc6c..5b14efc7cce6a 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/20_combine_processors.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/20_combine_processors.yml @@ -1,6 +1,11 @@ +setup: + - requires: + test_runner_features: allowed_warnings_regex --- "Test with date processor": - do: + allowed_warnings_regex: + - 'Date format \[dd/MMM/yyyy:HH:mm:ss xx] contains textual field specifiers that could change in JDK 23.*' ingest.put_pipeline: id: "_id" body: > @@ -41,6 +46,8 @@ - match: { acknowledged: true } - do: + allowed_warnings_regex: + - 'Date format \[dd/MMM/yyyy:HH:mm:ss xx] contains textual field specifiers that could change in JDK 23.*' index: index: test id: "1" @@ -70,6 +77,8 @@ --- "Test with date processor and ECS-v1": - do: + allowed_warnings_regex: + - 'Date format \[dd/MMM/yyyy:HH:mm:ss xx] contains textual field specifiers that could change in JDK 23.*' ingest.put_pipeline: id: "_id" body: > @@ -99,6 +108,8 @@ - match: { acknowledged: true } - do: + allowed_warnings_regex: + - 'Date format \[dd/MMM/yyyy:HH:mm:ss xx] contains textual field specifiers that could change in JDK 23.*' index: index: test id: "1" diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml index 35ec9979c3250..d4843fb152888 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml @@ -183,6 +183,7 @@ setup: body: settings: default_pipeline: "my-pipeline" + - match: { acknowledged: true } - do: headers: @@ -303,3 +304,1475 @@ setup: - match: { docs.1.doc._index: "second-index" } - match: { docs.1.doc._source.bar: "foo" } - not_exists: docs.1.doc.error + +--- +"Test ingest simulate with template substitutions for component templates": + + - skip: + features: + - headers + - allowed_warnings + + - requires: + cluster_features: ["simulate.component.template.substitutions"] + reason: "ingest simulate component template substitutions added in 8.16" + + - do: + headers: + Content-Type: application/json + ingest.put_pipeline: + id: "foo-pipeline" + body: > + { + "processors": [ + { + "set": { + "field": "foo", + "value": true + } + } + ] + } + - match: { acknowledged: true } + + - do: + headers: + Content-Type: application/json + ingest.put_pipeline: + id: "bar-pipeline" + body: > + { + "processors": [ + { + "set": { + "field": "bar", + "value": true + } + } + ] + } + - match: { acknowledged: true } + + - do: + cluster.put_component_template: + name: mappings_template + body: + template: + mappings: + dynamic: strict + properties: + foo: + type: keyword + + - do: + cluster.put_component_template: + name: settings_template + body: + template: + settings: + index: + default_pipeline: "foo-pipeline" + + - do: + allowed_warnings: + - "index template [test-composable-1] has index patterns [foo*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-1] will take precedence during new index creation" + indices.put_index_template: + name: test-composable-1 + body: + index_patterns: + - foo* + composed_of: + - mappings_template + - settings_template + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": "FOO", + "other": "other" + } + } + ], + "component_template_substitutions": { + "mappings_template": { + "template": { + "mappings": { + "dynamic": "true", + "properties": { + "foo": { + "type": "keyword" + } + } + } + } + }, + "settings_template": { + "template": { + "settings": { + "index": { + "default_pipeline": "bar-pipeline" + } + } + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.other: "other" } + - match: { docs.0.doc._source.bar: true } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: ["bar-pipeline"] } + - not_exists: docs.0.doc.error + + - do: + indices.create: + index: foo-1 + - match: { acknowledged: true } + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": "FOO", + "other": "other" + } + } + ], + "component_template_substitutions": { + "mappings_template": { + "template": { + "mappings": { + "dynamic": "true", + "properties": { + "foo": { + "type": "keyword" + } + } + } + } + }, + "settings_template": { + "template": { + "settings": { + "index": { + "default_pipeline": "bar-pipeline" + } + } + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.other: "other" } + - match: { docs.0.doc._source.bar: true } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: ["bar-pipeline"] } + - not_exists: docs.0.doc.error + +--- +"Test ingest simulate with template substitutions for component templates removing pipelines": + + - skip: + features: + - headers + - allowed_warnings + + - requires: + cluster_features: ["simulate.component.template.substitutions"] + reason: "ingest simulate component template substitutions added in 8.16" + + - do: + headers: + Content-Type: application/json + ingest.put_pipeline: + id: "foo-pipeline" + body: > + { + "processors": [ + { + "set": { + "field": "foo", + "value": true + } + } + ] + } + - match: { acknowledged: true } + + - do: + cluster.put_component_template: + name: settings_template + body: + template: + settings: + index: + default_pipeline: "foo-pipeline" + + - do: + allowed_warnings: + - "index template [test-composable-1] has index patterns [foo*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-1] will take precedence during new index creation" + indices.put_index_template: + name: test-composable-1 + body: + index_patterns: + - foo* + composed_of: + - settings_template + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": "FOO" + } + } + ], + "component_template_substitutions": { + "settings_template": { + "template": { + "settings": { + "index": { + "default_pipeline": null + } + } + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: [] } + - not_exists: docs.0.doc.error + + - do: + indices.create: + index: foo-1 + - match: { acknowledged: true } + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": "FOO" + } + } + ], + "component_template_substitutions": { + "settings_template": { + "template": { + "settings": { + "index": { + "default_pipeline": null + } + } + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: [] } + - not_exists: docs.0.doc.error + +--- +"Test ingest simulate with component template substitutions for data streams": + # In this test, we make sure that when the index template is a data stream template, simulte ingest works the same whether the data stream + # has been created or not -- either way, we expect it to use the template rather than the data stream / index mappings and settings. + + - skip: + features: + - headers + - allowed_warnings + + - requires: + cluster_features: ["simulate.component.template.substitutions"] + reason: "ingest simulate component template substitutions added in 8.16" + + - do: + headers: + Content-Type: application/json + ingest.put_pipeline: + id: "foo-pipeline" + body: > + { + "processors": [ + { + "set": { + "field": "foo", + "value": true + } + } + ] + } + - match: { acknowledged: true } + + - do: + cluster.put_component_template: + name: mappings_template + body: + template: + mappings: + dynamic: strict + properties: + foo: + type: keyword + + - do: + cluster.put_component_template: + name: settings_template + body: + template: + settings: + index: + default_pipeline: "foo-pipeline" + + - do: + allowed_warnings: + - "index template [test-composable-1] has index patterns [foo*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-1] will take precedence during new index creation" + indices.put_index_template: + name: test-composable-1 + body: + index_patterns: + - foo* + composed_of: + - mappings_template + - settings_template + + - do: + allowed_warnings: + - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" + indices.put_index_template: + name: my-template1 + body: + index_patterns: [simple-data-stream1] + composed_of: + - mappings_template + - settings_template + data_stream: {} + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: simple-data-stream1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "@timestamp": 1234, + "foo": false + } + } + ], + "pipeline_substitutions": { + "foo-pipeline-2": { + "processors": [ + { + "set": { + "field": "foo", + "value": "FOO" + } + } + ] + } + }, + "component_template_substitutions": { + "settings_template": { + "template": { + "settings": { + "index": { + "default_pipeline": "foo-pipeline-2" + } + } + } + }, + "mappings_template": { + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "keyword" + } + } + } + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "simple-data-stream1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: ["foo-pipeline-2"] } + - not_exists: docs.0.doc.error + + - do: + indices.create_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + - do: + cluster.health: + wait_for_status: yellow + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: simple-data-stream1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "@timestamp": 1234, + "foo": false + } + } + ], + "pipeline_substitutions": { + "foo-pipeline-2": { + "processors": [ + { + "set": { + "field": "foo", + "value": "FOO" + } + } + ] + } + }, + "component_template_substitutions": { + "settings_template": { + "template": { + "settings": { + "index": { + "default_pipeline": "foo-pipeline-2" + } + } + } + }, + "mappings_template": { + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "keyword" + } + } + } + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "simple-data-stream1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: ["foo-pipeline-2"] } + - not_exists: docs.0.doc.error + +--- +"Test ingest simulate with index template substitutions": + + - skip: + features: + - headers + - allowed_warnings + + - requires: + cluster_features: ["simulate.index.template.substitutions"] + reason: "ingest simulate index template substitutions added in 8.16" + + - do: + headers: + Content-Type: application/json + ingest.put_pipeline: + id: "foo-pipeline" + body: > + { + "processors": [ + { + "set": { + "field": "foo", + "value": true + } + } + ] + } + - match: { acknowledged: true } + + - do: + cluster.put_component_template: + name: settings_template + body: + template: + settings: + index: + default_pipeline: "foo-pipeline" + + - do: + cluster.put_component_template: + name: mappings_template + body: + template: + mappings: + dynamic: strict + properties: + foo: + type: keyword + + - do: + allowed_warnings: + - "index template [test-composable-1] has index patterns [foo*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-1] will take precedence during new index creation" + indices.put_index_template: + name: test-composable-1 + body: + index_patterns: + - foo* + composed_of: + - mappings_template + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": "FOO" + } + } + ], + "component_template_substitutions": { + "settings_template": { + "template": { + "settings": { + "index": { + "default_pipeline": null + } + } + } + } + }, + "index_template_substitutions": { + "foo_index_template": { + "index_patterns":[ + "foo*" + ], + "composed_of": ["settings_template"] + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: [] } + - not_exists: docs.0.doc.error + + - do: + indices.create: + index: foo-1 + - match: { acknowledged: true } + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": "FOO" + } + } + ], + "component_template_substitutions": { + "settings_template": { + "template": { + "settings": { + "index": { + "default_pipeline": null + } + } + } + } + }, + "index_template_substitutions": { + "foo_index_template": { + "index_patterns":[ + "foo*" + ], + "composed_of": ["settings_template", "mappings_template"] + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: [] } + - not_exists: docs.0.doc.error + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": "FOO" + } + } + ], + "component_template_substitutions": { + "mappings_template": { + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "boolean" + } + } + } + } + } + }, + "index_template_substitutions": { + "foo_index_template": { + "index_patterns":[ + "foo*" + ], + "composed_of": ["settings_template", "mappings_template"] + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.foo: true } + - match: { docs.0.doc.executed_pipelines: ["foo-pipeline"] } + - not_exists: docs.0.doc.error + +--- +"Test ingest simulate with index template substitutions for data streams": + # In this test, we make sure that when the index template is a data stream template, simulate ingest works the same whether the data + # stream has been created or not -- either way, we expect it to use the template rather than the data stream / index mappings and settings. + + - skip: + features: + - headers + - allowed_warnings + + - requires: + cluster_features: ["simulate.index.template.substitutions"] + reason: "ingest simulate component template substitutions added in 8.16" + + - do: + headers: + Content-Type: application/json + ingest.put_pipeline: + id: "foo-pipeline" + body: > + { + "processors": [ + { + "set": { + "field": "foo", + "value": true + } + } + ] + } + - match: { acknowledged: true } + + - do: + cluster.put_component_template: + name: mappings_template + body: + template: + mappings: + dynamic: strict + properties: + foo: + type: boolean + + - do: + cluster.put_component_template: + name: settings_template + body: + template: + settings: + index: + default_pipeline: "foo-pipeline" + + - do: + allowed_warnings: + - "index template [test-composable-1] has index patterns [foo*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-1] will take precedence during new index creation" + indices.put_index_template: + name: test-composable-1 + body: + index_patterns: + - foo* + composed_of: + - mappings_template + - settings_template + + - do: + allowed_warnings: + - "index template [my-template-1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template-1] will take precedence during new index creation" + indices.put_index_template: + name: my-template-1 + body: + index_patterns: [simple-data-stream1] + composed_of: + - mappings_template + - settings_template + data_stream: {} + + # Here we replace my-template-1 with a substitute version that uses the settings_template_2 and mappings_template_2 templates defined in + # this request, and foo-pipeline-2 defined in this request. + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: simple-data-stream1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "@timestamp": 1234, + "foo": false + } + } + ], + "pipeline_substitutions": { + "foo-pipeline-2": { + "processors": [ + { + "set": { + "field": "foo", + "value": "FOO" + } + } + ] + } + }, + "component_template_substitutions": { + "settings_template_2": { + "template": { + "settings": { + "index": { + "default_pipeline": "foo-pipeline-2" + } + } + } + }, + "mappings_template_2": { + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "keyword" + } + } + } + } + } + }, + "index_template_substitutions": { + "my-template-1": { + "index_patterns": ["simple-data-stream1"], + "composed_of": ["settings_template_2", "mappings_template_2"], + "data_stream": {} + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "simple-data-stream1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: ["foo-pipeline-2"] } + - not_exists: docs.0.doc.error + + - do: + indices.create_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + - do: + cluster.health: + wait_for_status: yellow + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: simple-data-stream1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "@timestamp": 1234, + "foo": false + } + } + ], + "pipeline_substitutions": { + "foo-pipeline-2": { + "processors": [ + { + "set": { + "field": "foo", + "value": "FOO" + } + } + ] + } + }, + "component_template_substitutions": { + "settings_template_2": { + "template": { + "settings": { + "index": { + "default_pipeline": "foo-pipeline-2" + } + } + } + }, + "mappings_template_2": { + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "keyword" + } + } + } + } + } + }, + "index_template_substitutions": { + "my-template-1": { + "index_patterns": ["simple-data-stream1"], + "composed_of": ["settings_template_2", "mappings_template_2"], + "data_stream": {} + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "simple-data-stream1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: ["foo-pipeline-2"] } + - not_exists: docs.0.doc.error + +--- +"Test ingest simulate with mapping addition for data streams": + # In this test, we make sure that when the index template is a data stream template, simulate ingest works the same whether the data + # stream has been created or not -- either way, we expect it to use the template rather than the data stream / index mappings and settings. + + - skip: + features: + - headers + - allowed_warnings + + - requires: + cluster_features: ["simulate.mapping.addition"] + reason: "ingest simulate mapping addition added in 8.17" + + - do: + headers: + Content-Type: application/json + ingest.put_pipeline: + id: "foo-pipeline" + body: > + { + "processors": [ + { + "set": { + "field": "foo", + "value": true + } + } + ] + } + - match: { acknowledged: true } + + - do: + cluster.put_component_template: + name: mappings_template + body: + template: + mappings: + dynamic: strict + properties: + foo: + type: boolean + + - do: + cluster.put_component_template: + name: settings_template + body: + template: + settings: + index: + default_pipeline: "foo-pipeline" + + - do: + allowed_warnings: + - "index template [test-composable-1] has index patterns [foo*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-1] will take precedence during new index creation" + indices.put_index_template: + name: test-composable-1 + body: + index_patterns: + - foo* + composed_of: + - mappings_template + - settings_template + + - do: + allowed_warnings: + - "index template [my-template-1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template-1] will take precedence during new index creation" + indices.put_index_template: + name: my-template-1 + body: + index_patterns: [simple-data-stream1] + composed_of: + - mappings_template + - settings_template + data_stream: {} + + # Here we replace my-template-1 with a substitute version that uses the settings_template_2 and mappings_template_2 templates defined in + # this request, and foo-pipeline-2 defined in this request. + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: simple-data-stream1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "@timestamp": 1234, + "foo": false + } + } + ], + "pipeline_substitutions": { + "foo-pipeline-2": { + "processors": [ + { + "set": { + "field": "foo", + "value": "FOO" + } + } + ] + } + }, + "component_template_substitutions": { + "settings_template_2": { + "template": { + "settings": { + "index": { + "default_pipeline": "foo-pipeline-2" + } + } + } + }, + "mappings_template_2": { + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "integer" + } + } + } + } + } + }, + "index_template_substitutions": { + "my-template-1": { + "index_patterns": ["simple-data-stream1"], + "composed_of": ["settings_template_2", "mappings_template_2"], + "data_stream": {} + } + }, + "mapping_addition": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "keyword" + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "simple-data-stream1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: ["foo-pipeline-2"] } + - not_exists: docs.0.doc.error + + - do: + indices.create_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + - do: + cluster.health: + wait_for_status: yellow + + # Now that we have created a data stream, run the exact same simulate ingeset request to make sure we still get the same result, and that + # the substitutions and additions from the simulate ingest request are used instead of information from the data stream or its backing + # index. + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: simple-data-stream1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "@timestamp": 1234, + "foo": false + } + } + ], + "pipeline_substitutions": { + "foo-pipeline-2": { + "processors": [ + { + "set": { + "field": "foo", + "value": "FOO" + } + } + ] + } + }, + "component_template_substitutions": { + "settings_template_2": { + "template": { + "settings": { + "index": { + "default_pipeline": "foo-pipeline-2" + } + } + } + }, + "mappings_template_2": { + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "integer" + } + } + } + } + } + }, + "index_template_substitutions": { + "my-template-1": { + "index_patterns": ["simple-data-stream1"], + "composed_of": ["settings_template_2", "mappings_template_2"], + "data_stream": {} + } + }, + "mapping_addition": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "keyword" + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "simple-data-stream1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: ["foo-pipeline-2"] } + - not_exists: docs.0.doc.error + +--- +"Test mapping addition works with legacy templates": + # In this test, we make sure that when the index template is a data stream template, simulate ingest works the same whether the data + # stream has been created or not -- either way, we expect it to use the template rather than the data stream / index mappings and settings. + + - skip: + features: + - headers + - allowed_warnings + + - requires: + cluster_features: ["simulate.mapping.addition"] + reason: "ingest simulate mapping addition added in 8.17" + + - do: + indices.put_template: + name: my-legacy-template + body: + index_patterns: foo-* + settings: + number_of_replicas: 0 + mappings: + dynamic: strict + properties: + foo: + type: integer + bar: + type: boolean + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": 3, + "bar": "not a boolean" + } + } + ] + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.foo: 3 } + - match: { docs.0.doc._source.bar: "not a boolean" } + - match: { docs.0.doc.error.type: "document_parsing_exception" } + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": 3, + "bar": "not a boolean" + } + } + ], + "mapping_addition": { + "dynamic": "strict", + "properties": { + "bar": { + "type": "keyword" + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.foo: 3 } + - match: { docs.0.doc._source.bar: "not a boolean" } + - not_exists: docs.0.doc.error + + - do: + allowed_warnings: + - "index [foo-1] matches multiple legacy templates [global, my-legacy-template], composable templates will only match a single template" + indices.create: + index: foo-1 + - match: { acknowledged: true } + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": 3, + "bar": "not a boolean" + } + } + ], + "mapping_addition": { + "dynamic": "strict", + "properties": { + "bar": { + "type": "keyword" + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.foo: 3 } + - match: { docs.0.doc._source.bar: "not a boolean" } + - not_exists: docs.0.doc.error + +--- +"Test mapping addition works with indices without templates": + # In this test, we make sure that when we have an index that has mapping but was not built with a template, that the additional_mapping + # is merged in with that template. + + - skip: + features: + - headers + - allowed_warnings + + - requires: + cluster_features: ["simulate.support.non.template.mapping"] + reason: "ingest simulate support for indices with mappings that didn't come from templates added in 8.17" + + # A global match-everything legacy template is added to the cluster sometimes (rarely). We have to get rid of this template if it exists + # because this test is making sure we get correct behavior when an index matches *no* template: + - do: + indices.delete_template: + name: '*' + ignore: 404 + + # First, make sure that validation fails before we create the index (since we are only defining to bar field but trying to index a value + # for foo. + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": 3, + "bar": "some text value" + } + } + ], + "mapping_addition": { + "dynamic": "strict", + "properties": { + "bar": { + "type": "keyword" + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.foo: 3 } + - match: { docs.0.doc._source.bar: "some text value" } + - match: { docs.0.doc.error.type: "strict_dynamic_mapping_exception" } + + - do: + indices.create: + index: foo-1 + body: + mappings: + dynamic: strict + properties: + foo: + type: integer + - match: { acknowledged: true } + + # Now make sure that the mapping for the newly-created index is getting picked up. Validation fails because it only defined a mapping + # for foo, not for bar. + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": 3, + "bar": "some text value" + } + } + ] + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.foo: 3 } + - match: { docs.0.doc._source.bar: "some text value" } + - match: { docs.0.doc.error.type: "strict_dynamic_mapping_exception" } + + # Now we make sure that the index's mapping gets merged with the mapping_addition: + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": 3, + "bar": "some text value" + } + } + ], + "mapping_addition": { + "dynamic": "strict", + "properties": { + "bar": { + "type": "keyword" + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.foo: 3 } + - match: { docs.0.doc._source.bar: "some text value" } + - not_exists: docs.0.doc.error + + # This last call to simulate is just making sure that if there are no templates, no index mappings, no substitutions, and no mapping + # addition, then validation does not fail + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: nonexistent + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": 3, + "bar": "some text value" + } + } + ] + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "nonexistent" } + - match: { docs.0.doc._source.foo: 3 } + - match: { docs.0.doc._source.bar: "some text value" } + - not_exists: docs.0.doc.error + +--- +"Test ignored_fields": + - skip: + features: + - headers + - allowed_warnings + + - requires: + cluster_features: ["simulate.ignored.fields"] + reason: "ingest simulate ignored fields added in 8.18" + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: nonexistent + body: > + { + "docs": [ + { + "_index": "simulate-test", + "_id": "y9Es_JIBiw6_GgN-U0qy", + "_score": 1, + "_source": { + "abc": "sfdsfsfdsfsfdsfsfdsfsfdsfsfdsf" + } + } + ], + "index_template_substitutions": { + "ind_temp": { + "index_patterns": ["simulate-test"], + "composed_of": ["simulate-test"] + } + }, + "component_template_substitutions": { + "simulate-test": { + "template": { + "mappings": { + "dynamic": false, + "properties": { + "abc": { + "type": "keyword", + "ignore_above": 1 + } + } + } + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "simulate-test" } + - match: { docs.0.doc._source.abc: "sfdsfsfdsfsfdsfsfdsfsfdsfsfdsf" } + - match: { docs.0.doc.ignored_fields: [ {"field": "abc"} ] } + - not_exists: docs.0.doc.error diff --git a/qa/smoke-test-multinode/build.gradle b/qa/smoke-test-multinode/build.gradle index fa60e75ce067e..cc71a99562eb5 100644 --- a/qa/smoke-test-multinode/build.gradle +++ b/qa/smoke-test-multinode/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.internal-yaml-rest-test' diff --git a/qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java b/qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java index 63ada4c416f03..e53c0564be297 100644 --- a/qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java +++ b/qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.smoketest; @@ -34,6 +35,7 @@ public class SmokeTestMultiNodeClientYamlTestSuiteIT extends ESClientYamlSuiteTe // The first node does not have the ingest role so we're sure ingest requests are forwarded: .node(0, n -> n.setting("node.roles", "[master,data,ml,remote_cluster_client,transform]")) .feature(FeatureFlag.TIME_SERIES_MODE) + .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED) .build(); public SmokeTestMultiNodeClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { diff --git a/qa/smoke-test-plugins/build.gradle b/qa/smoke-test-plugins/build.gradle index 7ae46b8e0d683..c707c2b5e8c80 100644 --- a/qa/smoke-test-plugins/build.gradle +++ b/qa/smoke-test-plugins/build.gradle @@ -1,13 +1,13 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.apache.tools.ant.filters.ReplaceTokens -import org.elasticsearch.gradle.internal.info.BuildParams apply plugin: 'elasticsearch.legacy-yaml-rest-test' diff --git a/qa/smoke-test-plugins/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java b/qa/smoke-test-plugins/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java index b5367df0d1aaa..586f0bfcbc712 100644 --- a/qa/smoke-test-plugins/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java +++ b/qa/smoke-test-plugins/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.smoketest; diff --git a/qa/system-indices/build.gradle b/qa/system-indices/build.gradle index 4cb60860cca46..c619d4f02e527 100644 --- a/qa/system-indices/build.gradle +++ b/qa/system-indices/build.gradle @@ -1,20 +1,21 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ apply plugin: 'elasticsearch.base-internal-es-plugin' apply plugin: 'elasticsearch.legacy-java-rest-test' esplugin { - name 'system-indices-qa' - description 'Plugin for performing QA of system indices' - classname 'org.elasticsearch.system.indices.SystemIndicesQA' - licenseFile rootProject.file('licenses/SSPL-1.0+ELASTIC-LICENSE-2.0.txt') - noticeFile rootProject.file('NOTICE.txt') + name = 'system-indices-qa' + description = 'Plugin for performing QA of system indices' + classname ='org.elasticsearch.system.indices.SystemIndicesQA' + licenseFile = rootProject.file('licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt') + noticeFile = rootProject.file('NOTICE.txt') } testClusters.configureEach { diff --git a/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/FeatureUpgradeApiIT.java b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/FeatureUpgradeApiIT.java index 8c790fe1cde49..25cfbd7949611 100644 --- a/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/FeatureUpgradeApiIT.java +++ b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/FeatureUpgradeApiIT.java @@ -1,14 +1,16 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.system.indices; import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; @@ -42,13 +44,17 @@ protected Settings restClientSettings() { } public void testCreatingSystemIndex() throws Exception { - Response response = client().performRequest(new Request("PUT", "/_net_new_sys_index/_create")); + var request = new Request("PUT", "/_net_new_sys_index/_create"); + request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic")); + Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); } @SuppressWarnings("unchecked") public void testGetFeatureUpgradedStatuses() throws Exception { - client().performRequest(new Request("PUT", "/_net_new_sys_index/_create")); + var request = new Request("PUT", "/_net_new_sys_index/_create"); + request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic")); + client().performRequest(request); Response response = client().performRequest(new Request("GET", "/_migration/system_features")); assertThat(response.getStatusLine().getStatusCode(), is(200)); XContentTestUtils.JsonMapView view = XContentTestUtils.createJsonMapView(response.getEntity().getContent()); diff --git a/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/NetNewSystemIndicesIT.java b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/NetNewSystemIndicesIT.java index efa815756b582..d2c48d1e9b609 100644 --- a/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/NetNewSystemIndicesIT.java +++ b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/NetNewSystemIndicesIT.java @@ -1,16 +1,18 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.system.indices; import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.settings.SecureString; @@ -39,7 +41,9 @@ public void testCreatingSystemIndex() throws Exception { ); assertThat(EntityUtils.toString(e.getResponse().getEntity()), containsString("system")); - Response response = client().performRequest(new Request("PUT", "/_net_new_sys_index/_create")); + var request = new Request("PUT", "/_net_new_sys_index/_create"); + request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic")); + Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); } @@ -55,6 +59,7 @@ public void testIndexDoc() throws Exception { Request request = new Request("PUT", "/_net_new_sys_index/" + id); request.setJsonEntity("{}"); + request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic")); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); } @@ -73,6 +78,7 @@ public void testSearch() throws Exception { Request request = new Request("PUT", "/_net_new_sys_index/" + id); request.setJsonEntity("{}"); request.addParameter("refresh", "true"); + request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic")); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); diff --git a/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/SystemAliasIT.java b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/SystemAliasIT.java index 217b0640b6a7c..6109a1cf9dc31 100644 --- a/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/SystemAliasIT.java +++ b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/SystemAliasIT.java @@ -1,14 +1,16 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.system.indices; import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; @@ -41,6 +43,7 @@ public void testCreatingSystemIndexWithAlias() throws Exception { { Request request = new Request("PUT", "/.internal-unmanaged-index-8"); request.setJsonEntity("{\"aliases\": {\".internal-unmanaged-alias\": {}}}"); + request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic")); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); } @@ -67,6 +70,7 @@ public void testCreatingSystemIndexWithLegacyAlias() throws Exception { { Request request = new Request("PUT", "/.internal-unmanaged-index-8"); + request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic")); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); } @@ -78,6 +82,7 @@ public void testCreatingSystemIndexWithLegacyAlias() throws Exception { public void testCreatingSystemIndexWithIndexAliasEndpoint() throws Exception { { Request request = new Request("PUT", "/.internal-unmanaged-index-8"); + request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic")); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); } @@ -101,6 +106,7 @@ public void testCreatingSystemIndexWithIndexAliasEndpoint() throws Exception { public void testCreatingSystemIndexWithAliasEndpoint() throws Exception { { Request request = new Request("PUT", "/.internal-unmanaged-index-8"); + request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic")); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); } @@ -125,6 +131,7 @@ public void testCreatingSystemIndexWithAliasEndpoint() throws Exception { public void testCreatingSystemIndexWithAliasesEndpoint() throws Exception { { Request request = new Request("PUT", "/.internal-unmanaged-index-8"); + request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic")); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); } diff --git a/qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java b/qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java index 1d69ae5c1ee4a..4e16b61300b97 100644 --- a/qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java +++ b/qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java @@ -1,14 +1,16 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.system.indices; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.client.internal.node.NodeClient; @@ -178,12 +180,12 @@ public List routes() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + var content = request.requiredContent(); IndexRequest indexRequest = new IndexRequest(".net-new-system-index-primary"); - indexRequest.source(request.requiredContent(), request.getXContentType()); + indexRequest.source(content, request.getXContentType()); indexRequest.id(request.param("id")); indexRequest.setRefreshPolicy(request.param("refresh")); - - return channel -> client.index(indexRequest, new RestToXContentListener<>(channel)); + return channel -> client.index(indexRequest, ActionListener.withRef(new RestToXContentListener<>(channel), content)); } @Override diff --git a/qa/unconfigured-node-name/build.gradle b/qa/unconfigured-node-name/build.gradle index dd1dd0a2334fa..ecc088e5972ef 100644 --- a/qa/unconfigured-node-name/build.gradle +++ b/qa/unconfigured-node-name/build.gradle @@ -1,22 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ -apply plugin: 'elasticsearch.legacy-java-rest-test' - -testClusters.configureEach { - setting 'xpack.security.enabled', 'false' -} - -testClusters.matching { it.name == "javaRestTest" }.configureEach { - nameCustomization = { null } -} - -tasks.named("javaRestTest").configure { - nonInputProperties.systemProperty 'tests.logfile', - "${-> testClusters.javaRestTest.singleNode().getServerLog()}" -} +apply plugin: 'elasticsearch.internal-java-rest-test' diff --git a/qa/unconfigured-node-name/src/javaRestTest/java/org/elasticsearch/unconfigured_node_name/JsonLogsFormatAndParseIT.java b/qa/unconfigured-node-name/src/javaRestTest/java/org/elasticsearch/unconfigured_node_name/JsonLogsFormatAndParseIT.java index 562ded69bd07d..7ceda070adadb 100644 --- a/qa/unconfigured-node-name/src/javaRestTest/java/org/elasticsearch/unconfigured_node_name/JsonLogsFormatAndParseIT.java +++ b/qa/unconfigured-node-name/src/javaRestTest/java/org/elasticsearch/unconfigured_node_name/JsonLogsFormatAndParseIT.java @@ -1,23 +1,21 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.unconfigured_node_name; import org.elasticsearch.common.logging.JsonLogsIntegTestCase; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.LogType; import org.hamcrest.Matcher; +import org.junit.ClassRule; -import java.io.BufferedReader; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.security.AccessController; -import java.security.PrivilegedAction; +import java.io.InputStream; import static org.hamcrest.Matchers.equalTo; @@ -25,9 +23,24 @@ public class JsonLogsFormatAndParseIT extends JsonLogsIntegTestCase { private static final String OS_NAME = System.getProperty("os.name"); private static final boolean WINDOWS = OS_NAME.startsWith("Windows"); - // These match the values defined in org.elasticsearch.gradle.testclusters.ElasticsearchNode - private static final String COMPUTERNAME = "WindowsComputername"; - private static final String HOSTNAME = "LinuxDarwinHostname"; + private static final String COMPUTERNAME = "WindowsTestComputername"; + private static final String HOSTNAME = "LinuxDarwinTestHostname"; + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .setting("xpack.security.enabled", "false") + .setting("discovery.type", "single-node") + .withNode( + localNodeSpecBuilder -> localNodeSpecBuilder.withoutName() + .environment("HOSTNAME", HOSTNAME) + .environment("COMPUTERNAME", COMPUTERNAME) + ) + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } @Override protected Matcher nodeNameMatcher() { @@ -38,13 +51,7 @@ protected Matcher nodeNameMatcher() { } @Override - protected BufferedReader openReader(Path logFile) { - return AccessController.doPrivileged((PrivilegedAction) () -> { - try { - return Files.newBufferedReader(logFile, StandardCharsets.UTF_8); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); + protected InputStream openLogsStream() { + return cluster.getNodeLog(0, LogType.SERVER_JSON); } } diff --git a/qa/verify-version-constants/build.gradle b/qa/verify-version-constants/build.gradle index 54b452c8f1e52..ee29da53dc51b 100644 --- a/qa/verify-version-constants/build.gradle +++ b/qa/verify-version-constants/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.VersionProperties @@ -18,7 +19,7 @@ dependencies { testImplementation project(':modules:rest-root') } -BuildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName -> +buildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName -> def baseCluster = testClusters.register(baseName) { version = bwcVersion.toString() setting 'xpack.security.enabled', 'true' diff --git a/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java b/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java index e10709a5b078a..9b2aebb2e4c8d 100644 --- a/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java +++ b/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.qa.verify_version_constants; diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 089b7470e9a97..f0df33877a965 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -13,7 +13,7 @@ restResources { } // REST API specifications are published under the Apache 2.0 License -ext.projectLicenses.set(['The Apache Software License, Version 2.0': 'http://www.apache.org/licenses/LICENSE-2.0']) +ext.projectLicenses.set(['The Apache Software License, Version 2.0': providers.provider(() -> 'http://www.apache.org/licenses/LICENSE-2.0')]) licenseFile.set(rootProject.file('licenses/APACHE-LICENSE-2.0.txt')) configurations { @@ -47,6 +47,7 @@ tasks.named("yamlRestTestV7CompatTransform").configure { task -> task.skipTestsByFilePattern("**/indices.upgrade/*.yml", "upgrade api will only get a dummy endpoint returning an exception suggesting to use _reindex") task.skipTestsByFilePattern("**/indices.stats/60_field_usage/*/*.yml", "field usage results will be different between lucene versions") task.skipTestsByFilePattern("**/search.aggregation/*.yml", "run by the aggregation module") + task.skipTestsByFilePattern("**/indices.get_mapping/11_basic_with_types.yml", "Empty mapping creation has changed") task.skipTest("bulk/11_dynamic_templates/Dynamic templates", "Error message has changed") task.skipTest("index/80_date_nanos/date_nanos requires dates after 1970 and before 2262", "Error message has changed") @@ -84,6 +85,8 @@ tasks.named("yamlRestTestV7CompatTransform").configure { task -> task.skipTest("search/370_profile/fetch nested source", "profile output has changed") task.skipTest("search/240_date_nanos/doc value fields are working as expected across date and date_nanos fields", "Fetching docvalues field multiple times is no longer allowed") task.skipTest("search/110_field_collapsing/field collapsing and rescore", "#107779 Field collapsing is compatible with rescore in 8.15") + task.skipTest("indices.create/11_basic_with_types/Create index with mappings", "Empty mapping creation has changed") + task.skipTest("search/140_pre_filter_search_shards/pre_filter_shard_size with shards that have no hit", "#115314 we skipp all shards unless the query has aggs, therefore the _shard.skipped do not match") task.replaceValueInMatch("_type", "_doc") task.addAllowedWarningRegex("\\[types removal\\].*") @@ -241,3 +244,35 @@ tasks.register('enforceYamlTestConvention').configure { tasks.named("precommit").configure { dependsOn 'enforceYamlTestConvention' } + +tasks.named("yamlRestTestV7CompatTransform").configure({ task -> + task.skipTest("indices.sort/10_basic/Index Sort", "warning does not exist for compatibility") + task.skipTest("search/330_fetch_fields/Test search rewrite", "warning does not exist for compatibility") + task.skipTest("tsdb/20_mapping/stored source is supported", "no longer serialize source_mode") + task.skipTest("tsdb/20_mapping/Synthetic source", "no longer serialize source_mode") + task.skipTest("logsdb/10_settings/create logs index", "no longer serialize source_mode") + task.skipTest("logsdb/20_source_mapping/stored _source mode is supported", "no longer serialize source_mode") + task.skipTest("logsdb/20_source_mapping/include/exclude is supported with stored _source", "no longer serialize source_mode") + task.skipTest("logsdb/20_source_mapping/synthetic _source is default", "no longer serialize source_mode") + task.skipTest("search/520_fetch_fields/fetch _seq_no via fields", "error code is changed from 5xx to 400 in 9.0") + task.skipTest("search.vectors/41_knn_search_bbq_hnsw/Test knn search", "Scoring has changed in latest versions") + task.skipTest("search.vectors/42_knn_search_bbq_flat/Test knn search", "Scoring has changed in latest versions") + task.skipTest("synonyms/90_synonyms_reloading_for_synset/Reload analyzers for specific synonym set", "Can't work until auto-expand replicas is 0-1 for synonyms index") + task.skipTest("search/90_search_after/_shard_doc sort", "restriction has been lifted in latest versions") + task.skipTest("search.vectors/180_update_dense_vector_type/Test create and update dense vector mapping with bulk indexing", "waiting for #118774 backport") + task.skipTest("search.vectors/160_knn_query_missing_params/kNN query in a bool clause - missing num_candidates", "waiting for #118774 backport") + task.skipTest("search.vectors/110_knn_query_with_filter/Simple knn query", "waiting for #118774 backport") + task.skipTest("search.vectors/160_knn_query_missing_params/kNN search used in nested field - missing num_candidates", "waiting for #118774 backport") + task.skipTest("search.vectors/180_update_dense_vector_type/Test create and update dense vector mapping to int4 with per-doc indexing and flush", "waiting for #118774 backport") + task.skipTest("search.vectors/110_knn_query_with_filter/PRE_FILTER: knn query with internal filter as pre-filter", "waiting for #118774 backport") + task.skipTest("search.vectors/180_update_dense_vector_type/Index, update and merge", "waiting for #118774 backport") + task.skipTest("search.vectors/160_knn_query_missing_params/kNN query with missing num_candidates param - size provided", "waiting for #118774 backport") + task.skipTest("search.vectors/110_knn_query_with_filter/POST_FILTER: knn query with filter from a parent bool query as post-filter", "waiting for #118774 backport") + task.skipTest("search.vectors/120_knn_query_multiple_shards/Aggregations with collected number of docs depends on num_candidates", "waiting for #118774 backport") + task.skipTest("search.vectors/180_update_dense_vector_type/Test create and update dense vector mapping with per-doc indexing and flush", "waiting for #118774 backport") + task.skipTest("search.vectors/110_knn_query_with_filter/PRE_FILTER: knn query with alias filter as pre-filter", "waiting for #118774 backport") + task.skipTest("search.vectors/140_knn_query_with_other_queries/Function score query with knn query", "waiting for #118774 backport") + task.skipTest("search.vectors/130_knn_query_nested_search/nested kNN search inner_hits size > 1", "waiting for #118774 backport") + task.skipTest("search.vectors/110_knn_query_with_filter/PRE_FILTER: pre-filter across multiple aliases", "waiting for #118774 backport") + task.skipTest("search.vectors/160_knn_query_missing_params/kNN search in a dis_max query - missing num_candidates", "waiting for #118774 backport") +}) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/async_search.submit.json b/rest-api-spec/src/main/resources/rest-api-spec/api/async_search.submit.json index 5cd2b0e26459e..3de0dec85f547 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/async_search.submit.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/async_search.submit.json @@ -43,11 +43,6 @@ "description":"Control whether the response should be stored in the cluster if it completed within the provided [wait_for_completion] time (default: false)", "default":false }, - "keep_alive": { - "type": "time", - "description": "Update the time interval in which the results (partial or final) for this search will be available", - "default": "5d" - }, "batched_reduce_size":{ "type":"number", "description":"The number of shard results that should be reduced at once on the coordinating node. This value should be used as the granularity at which progress results will be made available.", @@ -65,6 +60,11 @@ "type":"boolean", "description":"Specify whether wildcard and prefix queries should be analyzed (default: false)" }, + "ccs_minimize_roundtrips":{ + "type":"boolean", + "default":false, + "description":"When doing a cross-cluster search, setting it to true may improve overall search latency, particularly when searching clusters with a large number of shards. However, when set to true, the progress of searches on the remote clusters will not be received until the search finishes on all clusters." + }, "default_operator":{ "type":"enum", "options":[ @@ -126,6 +126,11 @@ "type":"string", "description":"Specify the node or shard the operation should be performed on (default: random)" }, + "rest_total_hits_as_int":{ + "type":"boolean", + "description":"Indicates whether hits.total should be rendered as an integer or an object in the rest search response", + "default":false + }, "q":{ "type":"string", "description":"Query in the Lucene query string syntax" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.stats.json index 4a8ca46ceba8c..23f6ed4ec5b76 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.stats.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.stats.json @@ -32,9 +32,9 @@ ] }, "params":{ - "flat_settings":{ + "include_remotes":{ "type":"boolean", - "description":"Return settings in flat format (default: false)" + "description":"Include remote cluster data into the response (default: false)" }, "timeout":{ "type":"time", diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/eql.search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/eql.search.json index c854c44d9d761..0f9af508f4c16 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/eql.search.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/eql.search.json @@ -41,6 +41,16 @@ "type": "time", "description": "Update the time interval in which the results (partial or final) for this search will be available", "default": "5d" + }, + "allow_partial_search_results": { + "type":"boolean", + "description":"Control whether the query should keep running in case of shard failures, and return partial results", + "default":false + }, + "allow_partial_sequence_results": { + "type":"boolean", + "description":"Control whether a sequence query should return partial results or no results at all in case of shard failures. This option has effect only if [allow_partial_search_results] is true.", + "default":false } }, "body":{ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/esql.async_query_delete.json b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.async_query_delete.json new file mode 100644 index 0000000000000..a6339559afd72 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.async_query_delete.json @@ -0,0 +1,27 @@ +{ + "esql.async_query_delete": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/esql-async-query-delete-api.html", + "description": "Delete an async query request given its ID." + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": ["application/json"] + }, + "url": { + "paths": [ + { + "path": "/_query/async/{id}", + "methods": ["DELETE"], + "parts": { + "id": { + "type": "string", + "description": "The async query ID" + } + } + } + ] + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_data_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_data_lifecycle.json index a3106a982b809..92b3ce61b4603 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_data_lifecycle.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_data_lifecycle.json @@ -4,7 +4,7 @@ "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams-delete-lifecycle.html", "description":"Deletes the data stream lifecycle of the selected data streams." }, - "stability":"experimental", + "stability":"stable", "visibility":"public", "headers":{ "accept": [ "application/json"] diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.explain_data_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.explain_data_lifecycle.json index 3232407000b19..14e07ee28a80d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.explain_data_lifecycle.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.explain_data_lifecycle.json @@ -4,7 +4,7 @@ "url": "https://www.elastic.co/guide/en/elasticsearch/reference/current/data-streams-explain-lifecycle.html", "description": "Retrieves information about the index's current data stream lifecycle, such as any potential encountered error, time since creation etc." }, - "stability": "experimental", + "stability": "stable", "visibility": "public", "headers": { "accept": [ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_lifecycle.json index 6f05af1485f98..a8d2e7185db83 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_lifecycle.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_lifecycle.json @@ -4,7 +4,7 @@ "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams-get-lifecycle.html", "description":"Returns the data stream lifecycle of the selected data streams." }, - "stability":"experimental", + "stability":"stable", "visibility":"public", "headers":{ "accept": [ "application/json"] diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_lifecycle_stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_lifecycle_stats.json new file mode 100644 index 0000000000000..8c9e947903402 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_lifecycle_stats.json @@ -0,0 +1,21 @@ +{ + "indices.get_data_lifecycle_stats": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams-get-lifecycle-stats.html", + "description": "Get data stream lifecycle statistics." + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": ["application/json"] + }, + "url": { + "paths": [ + { + "path": "/_lifecycle/stats", + "methods": ["GET"] + } + ] + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_data_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_data_lifecycle.json index 591525f3d99ff..0a2f7b33498cf 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_data_lifecycle.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_data_lifecycle.json @@ -4,10 +4,11 @@ "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams-put-lifecycle.html", "description":"Updates the data stream lifecycle of the selected data streams." }, - "stability":"experimental", + "stability":"stable", "visibility":"public", "headers":{ - "accept": [ "application/json"] + "accept": [ "application/json"], + "content_type": ["application/json"] }, "url": { "paths": [ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.stream_inference.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.stream_inference.json new file mode 100644 index 0000000000000..32b4b2f311837 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.stream_inference.json @@ -0,0 +1,49 @@ +{ + "inference.stream_inference":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/post-stream-inference-api.html", + "description":"Perform streaming inference" + }, + "stability":"experimental", + "visibility":"public", + "headers":{ + "accept": [ "text/event-stream"], + "content_type": ["application/json"] + }, + "url":{ + "paths":[ + { + "path":"/_inference/{inference_id}/_stream", + "methods":[ + "POST" + ], + "parts":{ + "inference_id":{ + "type":"string", + "description":"The inference Id" + } + } + }, + { + "path":"/_inference/{task_type}/{inference_id}/_stream", + "methods":[ + "POST" + ], + "parts":{ + "task_type":{ + "type":"string", + "description":"The task type" + }, + "inference_id":{ + "type":"string", + "description":"The inference Id" + } + } + } + ] + }, + "body":{ + "description":"The inference payload" + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.update.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.update.json new file mode 100644 index 0000000000000..6c458ce080aa7 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.update.json @@ -0,0 +1,45 @@ +{ + "inference.update": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/update-inference-api.html", + "description": "Update inference" + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": ["application/json"], + "content_type": ["application/json"] + }, + "url": { + "paths": [ + { + "path": "/_inference/{inference_id}/_update", + "methods": ["POST"], + "parts": { + "inference_id": { + "type": "string", + "description": "The inference Id" + } + } + }, + { + "path": "/_inference/{task_type}/{inference_id}/_update", + "methods": ["POST"], + "parts": { + "task_type": { + "type": "string", + "description": "The task type" + }, + "inference_id": { + "type": "string", + "description": "The inference Id" + } + } + } + ] + }, + "body": { + "description": "The inference endpoint's task and service settings" + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_geoip_database.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_geoip_database.json index ef6dc94dd27a6..fe50da720a4da 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_geoip_database.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_geoip_database.json @@ -1,7 +1,7 @@ { "ingest.delete_geoip_database":{ "documentation":{ - "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/TODO.html", + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/delete-geoip-database-api.html", "description":"Deletes a geoip database configuration" }, "stability":"stable", diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_ip_location_database.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_ip_location_database.json new file mode 100644 index 0000000000000..e97d1da276906 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_ip_location_database.json @@ -0,0 +1,31 @@ +{ + "ingest.delete_ip_location_database":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/delete-ip-location-database-api.html", + "description":"Deletes an ip location database configuration" + }, + "stability":"stable", + "visibility":"public", + "headers":{ + "accept": [ "application/json"] + }, + "url":{ + "paths":[ + { + "path":"/_ingest/ip_location/database/{id}", + "methods":[ + "DELETE" + ], + "parts":{ + "id":{ + "type":"list", + "description":"A comma-separated list of ip location database configurations to delete" + } + } + } + ] + }, + "params":{ + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_geoip_database.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_geoip_database.json index 96f028e2e5251..5c59994d4b22e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_geoip_database.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_geoip_database.json @@ -1,7 +1,7 @@ { "ingest.get_geoip_database":{ "documentation":{ - "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/TODO.html", + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/get-geoip-database-api.html", "description":"Returns geoip database configuration." }, "stability":"stable", diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_ip_location_database.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_ip_location_database.json new file mode 100644 index 0000000000000..a2e42fe6c8e59 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_ip_location_database.json @@ -0,0 +1,37 @@ +{ + "ingest.get_ip_location_database":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/get-ip-location-database-api.html", + "description":"Returns the specified ip location database configuration" + }, + "stability":"stable", + "visibility":"public", + "headers":{ + "accept": [ "application/json"] + }, + "url":{ + "paths":[ + { + "path":"/_ingest/ip_location/database", + "methods":[ + "GET" + ] + }, + { + "path":"/_ingest/ip_location/database/{id}", + "methods":[ + "GET" + ], + "parts":{ + "id":{ + "type":"list", + "description":"A comma-separated list of ip location database configurations to get; use `*` to get all ip location database configurations" + } + } + } + ] + }, + "params":{ + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_geoip_database.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_geoip_database.json index 5447ea1e5a4e3..6d088e3f164f4 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_geoip_database.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_geoip_database.json @@ -1,7 +1,7 @@ { "ingest.put_geoip_database":{ "documentation":{ - "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/TODO.html", + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/put-geoip-database-api.html", "description":"Puts the configuration for a geoip database to be downloaded" }, "stability":"stable", diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_ip_location_database.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_ip_location_database.json new file mode 100644 index 0000000000000..18487969b1a90 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_ip_location_database.json @@ -0,0 +1,36 @@ +{ + "ingest.put_ip_location_database":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/put-ip-location-database-api.html", + "description":"Puts the configuration for a ip location database to be downloaded" + }, + "stability":"stable", + "visibility":"public", + "headers":{ + "accept": [ "application/json"], + "content_type": ["application/json"] + }, + "url":{ + "paths":[ + { + "path":"/_ingest/ip_location/database/{id}", + "methods":[ + "PUT" + ], + "parts":{ + "id":{ + "type":"string", + "description":"The id of the database configuration" + } + } + } + ] + }, + "params":{ + }, + "body":{ + "description":"The database configuration definition", + "required":true + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/license.post_start_trial.json b/rest-api-spec/src/main/resources/rest-api-spec/api/license.post_start_trial.json index 986040d69cb4f..9fb85807d611f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/license.post_start_trial.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/license.post_start_trial.json @@ -31,10 +31,6 @@ "master_timeout": { "type": "time", "description": "Timeout for processing on master node" - }, - "timeout": { - "type": "time", - "description": "Timeout for acknowledgement of update from all nodes in cluster" } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/migrate.cancel_reindex.json b/rest-api-spec/src/main/resources/rest-api-spec/api/migrate.cancel_reindex.json new file mode 100644 index 0000000000000..a034f204edbfb --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/migrate.cancel_reindex.json @@ -0,0 +1,30 @@ +{ + "migrate.cancel_reindex":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-stream-reindex.html", + "description":"This API returns the status of a migration reindex attempt for a data stream or index" + }, + "stability":"experimental", + "visibility":"private", + "headers":{ + "accept": [ "application/json"], + "content_type": ["application/json"] + }, + "url":{ + "paths":[ + { + "path":"/_migration/reindex/{index}/_cancel", + "methods":[ + "POST" + ], + "parts":{ + "index":{ + "type":"string", + "description":"The index or data stream name" + } + } + } + ] + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/migrate.get_reindex_status.json b/rest-api-spec/src/main/resources/rest-api-spec/api/migrate.get_reindex_status.json new file mode 100644 index 0000000000000..057269598a7d8 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/migrate.get_reindex_status.json @@ -0,0 +1,31 @@ +{ + "migrate.get_reindex_status":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-stream-reindex.html", + "description":"This API returns the status of a migration reindex attempt for a data stream or index" + }, + "stability":"experimental", + "visibility":"private", + "headers":{ + "accept": [ "application/json"], + "content_type": ["application/json"] + }, + "url":{ + "paths":[ + { + "path":"/_migration/reindex/{index}/_status", + "methods":[ + "GET" + ], + "parts":{ + "index":{ + "type":"string", + "description":"The index or data stream name" + } + } + } + ] + } + } +} + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/migrate.reindex.json b/rest-api-spec/src/main/resources/rest-api-spec/api/migrate.reindex.json new file mode 100644 index 0000000000000..149a90bc198b0 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/migrate.reindex.json @@ -0,0 +1,29 @@ +{ + "migrate.reindex":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-stream-reindex.html", + "description":"This API reindexes all legacy backing indices for a data stream. It does this in a persistent task. The persistent task id is returned immediately, and the reindexing work is completed in that task" + }, + "stability":"experimental", + "visibility":"private", + "headers":{ + "accept": [ "application/json"], + "content_type": ["application/json"] + }, + "url":{ + "paths":[ + { + "path":"/_migration/reindex", + "methods":[ + "POST" + ] + } + ] + }, + "body":{ + "description":"The body contains the fields `mode` and `source.index, where the only mode currently supported is `upgrade`, and the `source.index` must be a data stream name", + "required":true + } + } +} + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/open_point_in_time.json b/rest-api-spec/src/main/resources/rest-api-spec/api/open_point_in_time.json index bce8dfd794dca..6f3d09c15c081 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/open_point_in_time.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/open_point_in_time.json @@ -55,6 +55,10 @@ "type": "string", "description": "Specific the time to live for the point in time", "required": true + }, + "allow_partial_search_results": { + "type": "boolean", + "description": "Specify whether to tolerate shards missing when creating the point-in-time, or otherwise throw an exception. (default: false)" } }, "body":{ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/query_rules.test.json b/rest-api-spec/src/main/resources/rest-api-spec/api/query_rules.test.json new file mode 100644 index 0000000000000..c82b45771ac7f --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/query_rules.test.json @@ -0,0 +1,38 @@ +{ + "query_rules.test": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/test-query-ruleset.html", + "description": "Tests a query ruleset to identify the rules that would match input criteria" + }, + "stability": "experimental", + "visibility": "public", + "headers": { + "accept": [ + "application/json" + ], + "content_type": [ + "application/json" + ] + }, + "url": { + "paths": [ + { + "path": "/_query_rules/{ruleset_id}/_test", + "methods": [ + "POST" + ], + "parts": { + "ruleset_id": { + "type": "string", + "description": "The unique identifier of the ruleset to test." + } + } + } + ] + }, + "body": { + "description": "The match criteria to test against the ruleset", + "required": true + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/security.delegate_pki.json b/rest-api-spec/src/main/resources/rest-api-spec/api/security.delegate_pki.json new file mode 100644 index 0000000000000..752ea35028b4f --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/security.delegate_pki.json @@ -0,0 +1,26 @@ +{ + "security.delegate_pki": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/security-api-delegate-pki-authentication.html", + "description": "Delegate PKI authentication." + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": ["application/json"] + }, + "url": { + "paths": [ + { + "path": "/_security/delegate_pki", + "methods": ["POST"] + } + ] + }, + "params": {}, + "body": { + "description":"The X509Certificate chain.", + "required":true + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/shutdown.delete_node.json b/rest-api-spec/src/main/resources/rest-api-spec/api/shutdown.delete_node.json index d990d1da1f144..6f1ec484e94d0 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/shutdown.delete_node.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/shutdown.delete_node.json @@ -26,6 +26,15 @@ } ] }, - "params":{} + "params":{ + "master_timeout":{ + "type":"time", + "description":"Explicit operation timeout for connection to master node" + }, + "timeout":{ + "type":"time", + "description":"Explicit operation timeout" + } + } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/shutdown.put_node.json b/rest-api-spec/src/main/resources/rest-api-spec/api/shutdown.put_node.json index bf20cf3b70bac..90b19557f5fb2 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/shutdown.put_node.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/shutdown.put_node.json @@ -26,7 +26,16 @@ } ] }, - "params":{}, + "params":{ + "master_timeout":{ + "type":"time", + "description":"Explicit operation timeout for connection to master node" + }, + "timeout":{ + "type":"time", + "description":"Explicit operation timeout" + } + }, "body":{ "description":"The shutdown type definition to register", "required": true diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.delete_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.delete_lifecycle.json index 12202a7a2a7b1..1d66312f053c7 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.delete_lifecycle.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.delete_lifecycle.json @@ -25,6 +25,15 @@ } ] }, - "params":{} + "params":{ + "master_timeout":{ + "type":"time", + "description":"Explicit operation timeout for connection to master node" + }, + "timeout":{ + "type":"time", + "description":"Explicit operation timeout" + } + } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.execute_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.execute_lifecycle.json index 1395a3d3275ae..71f1727a8638b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.execute_lifecycle.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.execute_lifecycle.json @@ -25,6 +25,15 @@ } ] }, - "params":{} + "params":{ + "master_timeout":{ + "type":"time", + "description":"Explicit operation timeout for connection to master node" + }, + "timeout":{ + "type":"time", + "description":"Explicit operation timeout" + } + } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.execute_retention.json b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.execute_retention.json index f6ce3e75cc379..4166122d5bf1d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.execute_retention.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.execute_retention.json @@ -19,6 +19,15 @@ } ] }, - "params":{} + "params":{ + "master_timeout":{ + "type":"time", + "description":"Explicit operation timeout for connection to master node" + }, + "timeout":{ + "type":"time", + "description":"Explicit operation timeout" + } + } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_lifecycle.json index 94d0772a405da..406fee6015522 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_lifecycle.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_lifecycle.json @@ -31,6 +31,15 @@ } ] }, - "params":{} + "params":{ + "master_timeout":{ + "type":"time", + "description":"Explicit operation timeout for connection to master node" + }, + "timeout":{ + "type":"time", + "description":"Explicit operation timeout" + } + } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_stats.json index aa693ad31711c..05281ff46cb8d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_stats.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_stats.json @@ -19,6 +19,15 @@ } ] }, - "params":{} + "params":{ + "master_timeout":{ + "type":"time", + "description":"Explicit operation timeout for connection to master node" + }, + "timeout":{ + "type":"time", + "description":"Explicit operation timeout" + } + } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_status.json b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_status.json index 92ba1b4c321e6..404f92f55921f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_status.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.get_status.json @@ -19,6 +19,15 @@ } ] }, - "params":{} + "params":{ + "master_timeout":{ + "type":"time", + "description":"Explicit operation timeout for connection to master node" + }, + "timeout":{ + "type":"time", + "description":"Explicit operation timeout" + } + } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.put_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.put_lifecycle.json index 7e7babb987c79..621ed870ffdbe 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/slm.put_lifecycle.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/slm.put_lifecycle.json @@ -26,7 +26,16 @@ } ] }, - "params":{}, + "params":{ + "master_timeout":{ + "type":"time", + "description":"Explicit operation timeout for connection to master node" + }, + "timeout":{ + "type":"time", + "description":"Explicit operation timeout" + } + }, "body":{ "description":"The snapshot lifecycle policy definition to register" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/xpack.info.json b/rest-api-spec/src/main/resources/rest-api-spec/api/xpack.info.json index 68b2a5d2c2c8b..35895f0ddb581 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/xpack.info.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/xpack.info.json @@ -20,6 +20,12 @@ ] }, "params":{ + "human":{ + "type":"boolean", + "required":false, + "description":"Defines whether additional human-readable information is included in the response. In particular, it adds descriptions and a tag line. The default value is true.", + "default":true + }, "categories":{ "type":"list", "description":"Comma-separated list of info categories. Can be any of: build, license, features" diff --git a/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java b/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java index 5017ee06c7a8a..a92f5df53aeec 100644 --- a/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java +++ b/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.test.rest; @@ -35,6 +36,7 @@ public class ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { .module("health-shards-availability") .module("data-streams") .feature(FeatureFlag.TIME_SERIES_MODE) + .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED) .build(); public ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.indices/20_hidden.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.indices/20_hidden.yml index 619d2d5374b99..8de0cda460767 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.indices/20_hidden.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.indices/20_hidden.yml @@ -44,7 +44,9 @@ - requires: cluster_features: ["gte_v8.11.0"] reason: "dataset size was added in 8.11.0" + test_runner_features: ["headers"] - do: + headers: { X-elastic-product-origin: elastic } indices.create: index: .index1 body: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/10_basic.yml index 06139542c5e55..c50d7e6da2e3c 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/10_basic.yml @@ -30,6 +30,15 @@ - is_true: nodes.plugins - is_true: nodes.network_types +--- +"cluster stats accepts timeout param": + - do: + cluster.stats: + timeout: 24h + + - is_true: timestamp + - is_true: cluster_name + --- "cluster stats with human flag returns docs as human readable size": - requires: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/30_ccs_stats.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/30_ccs_stats.yml new file mode 100644 index 0000000000000..5f18bd496c6c8 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/30_ccs_stats.yml @@ -0,0 +1,156 @@ +--- +"cross-cluster search stats basic": + - requires: + test_runner_features: [ capabilities ] + capabilities: + - method: GET + path: /_cluster/stats + capabilities: + - "ccs-stats" + reason: "Capability required to run test" + + - do: + cluster.stats: { } + + - is_true: ccs + - is_true: ccs._search + - is_false: ccs.clusters # no ccs clusters configured + - exists: ccs._search.total + - exists: ccs._search.success + - exists: ccs._search.skipped + - is_true: ccs._search.took + - is_true: ccs._search.took_mrt_true + - is_true: ccs._search.took_mrt_false + - exists: ccs._search.remotes_per_search_max + - exists: ccs._search.remotes_per_search_avg + - exists: ccs._search.failure_reasons + - exists: ccs._search.features + - exists: ccs._search.clients + - exists: ccs._search.clusters + +--- +"cross-cluster search stats search": + - requires: + test_runner_features: [ capabilities ] + capabilities: + - method: GET + path: /_cluster/stats + capabilities: + - "ccs-stats" + reason: "Capability required to run test" + + - do: + cluster.state: {} + - set: { master_node: master } + - do: + nodes.info: + metric: [ http, transport ] + - set: {nodes.$master.http.publish_address: host} + - set: {nodes.$master.transport.publish_address: transport_host} + + - do: + cluster.put_settings: + body: + persistent: + cluster: + remote: + cluster_one: + seeds: + - "${transport_host}" + skip_unavailable: true + cluster_two: + seeds: + - "${transport_host}" + skip_unavailable: false + - is_true: persistent.cluster.remote.cluster_one + + - do: + indices.create: + index: test + body: + settings: + number_of_replicas: 0 + store.stats_refresh_interval: 0ms + + - do: + index: + index: test + id: "1" + refresh: true + body: + foo: bar + + - do: + indices.flush: + index: test + + - do: + cluster.health: + wait_for_status: green + + - do: + search: + index: "*,*:*" + body: + query: + match: + foo: bar + + - do: + cluster.stats: {} + - is_true: ccs + - is_true: ccs._search + - is_false: ccs.clusters # Still no remotes since include_remotes is not set + + - do: + cluster.stats: + include_remotes: true + - is_true: ccs + - is_true: ccs._search + - is_true: ccs.clusters # Now we have remotes + - is_true: ccs.clusters.cluster_one + - is_true: ccs.clusters.cluster_two + - is_true: ccs.clusters.cluster_one.cluster_uuid + - match: { ccs.clusters.cluster_one.mode: sniff } + - match: { ccs.clusters.cluster_one.skip_unavailable: true } + - match: { ccs.clusters.cluster_two.skip_unavailable: false } + - is_true: ccs.clusters.cluster_one.version + - match: { ccs.clusters.cluster_one.status: green } + - match: { ccs.clusters.cluster_two.status: green } + - is_true: ccs.clusters.cluster_one.nodes_count + - is_true: ccs.clusters.cluster_one.shards_count + - is_true: ccs.clusters.cluster_one.indices_count + - is_true: ccs.clusters.cluster_one.indices_total_size_in_bytes + - is_true: ccs.clusters.cluster_one.max_heap_in_bytes + - is_true: ccs.clusters.cluster_one.mem_total_in_bytes + - is_true: ccs._search.total + - is_true: ccs._search.success + - exists: ccs._search.skipped + - exists: ccs._search.took + - exists: ccs._search.took.max + - exists: ccs._search.took.avg + - exists: ccs._search.took.p90 + - is_true: ccs._search.took_mrt_true + - exists: ccs._search.took_mrt_true.max + - exists: ccs._search.took_mrt_true.avg + - exists: ccs._search.took_mrt_true.p90 + - is_true: ccs._search.took_mrt_false + - exists: ccs._search.took_mrt_false.max + - exists: ccs._search.took_mrt_false.avg + - exists: ccs._search.took_mrt_false.p90 + - match: { ccs._search.remotes_per_search_max: 2 } + - match: { ccs._search.remotes_per_search_avg: 2.0 } + - exists: ccs._search.failure_reasons + - exists: ccs._search.features + - exists: ccs._search.clients + - is_true: ccs._search.clusters + - is_true: ccs._search.clusters.cluster_one + - is_true: ccs._search.clusters.cluster_two + - gte: {ccs._search.clusters.cluster_one.total: 1} + - gte: {ccs._search.clusters.cluster_two.total: 1} + - exists: ccs._search.clusters.cluster_one.skipped + - exists: ccs._search.clusters.cluster_two.skipped + - exists: ccs._search.clusters.cluster_one.took + - exists: ccs._search.clusters.cluster_one.took.max + - exists: ccs._search.clusters.cluster_one.took.avg + - exists: ccs._search.clusters.cluster_one.took.p90 diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/40_source_modes.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/40_source_modes.yml new file mode 100644 index 0000000000000..64bbad7fb1c6d --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/40_source_modes.yml @@ -0,0 +1,50 @@ +--- +test source modes: + - requires: + cluster_features: ["cluster.stats.source_modes"] + reason: requires source modes features + + - do: + indices.create: + index: test-synthetic + body: + settings: + index: + mapping: + source.mode: synthetic + + - do: + indices.create: + index: test-stored + + - do: + indices.create: + index: test-disabled + body: + settings: + index: + mapping: + source.mode: disabled + + - do: + bulk: + refresh: true + body: + - '{ "create": { "_index": "test-synthetic" } }' + - '{ "name": "aaaa", "some_string": "AaAa", "some_int": 1000, "some_double": 123.456789, "some_bool": true }' + - '{ "create": { "_index": "test-stored" } }' + - '{ "name": "bbbb", "some_string": "BbBb", "some_int": 2000, "some_double": 321.987654, "some_bool": false }' + - '{ "create": { "_index": "test-disabled" } }' + - '{ "name": "cccc", "some_string": "CcCc", "some_int": 3000, "some_double": 421.484654, "some_bool": false }' + + - do: + search: + index: test-* + - match: { hits.total.value: 3 } + + - do: + cluster.stats: { } + + - match: { indices.mappings.source_modes.disabled: 1 } + - match: { indices.mappings.source_modes.stored: 1 } + - match: { indices.mappings.source_modes.synthetic: 1 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml index f1e296ed8e304..13f6ca58ea295 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml @@ -1,3 +1,9 @@ +setup: + - requires: + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" + +--- keyword: - requires: cluster_features: ["gte_v8.4.0"] @@ -7,9 +13,10 @@ keyword: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: kwd: type: keyword @@ -48,9 +55,8 @@ fetch without refresh also produces synthetic source: settings: index: refresh_interval: -1 + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: obj: properties: @@ -90,9 +96,10 @@ force_synthetic_source_ok: indices.create: index: test body: + settings: + index: + mapping.source.mode: stored mappings: - _source: - mode: stored properties: obj: properties: @@ -139,9 +146,10 @@ stored text: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: text: type: text @@ -212,9 +220,10 @@ stored keyword: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: kwd: type: keyword @@ -253,9 +262,10 @@ doc values keyword with ignore_above: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: kwd: type: keyword @@ -336,9 +346,10 @@ stored keyword with ignore_above: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: kwd: type: keyword @@ -421,9 +432,10 @@ indexed dense vectors: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -465,9 +477,10 @@ non-indexed dense vectors: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -508,9 +521,10 @@ _source filtering: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: kwd: type: keyword @@ -550,9 +564,9 @@ _doc_count: indices.create: index: test body: - mappings: - _source: - mode: synthetic + settings: + index: + mapping.source.mode: synthetic # with _doc_count - do: @@ -679,9 +693,10 @@ fields with ignore_malformed: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: ip: type: ip @@ -914,9 +929,10 @@ flattened field: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: flattened: type: flattened @@ -1006,9 +1022,10 @@ flattened field with ignore_above: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: field: type: flattened @@ -1050,6 +1067,64 @@ flattened field with ignore_above: - is_false: fields + +--- +flattened field with ignore_above and arrays: + - requires: + cluster_features: ["mapper.flattened.ignore_above_with_arrays_support"] + reason: requires support of ignore_above synthetic source with arrays + + - do: + indices.create: + index: test + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + field: + type: flattened + ignore_above: 10 + + - do: + index: + index: test + id: 1 + body: | + { + "field": [ + { "key1": { "key2": "key2", "key3": "key3_ignored" }, "key4": "key4_ignored", "key5": { "key6": "key6_ignored" }, "key7": "key7" }, + { "key1": { "key2": "key12", "key13": "key13_ignored" }, "key4": "key14_ignored", "key15": { "key16": "key16_ignored" }, "key17": [ "key17", "key18" ] } + ] + } + + - do: + get: + index: test + id: 1 + + - match: { _index: "test" } + - match: { _id: "1" } + - match: { _version: 1 } + - match: { found: true } + - match: + _source: + field: + key1: + key2: [ "key12", "key2" ] + key3: "key3_ignored" + key13: "key13_ignored" + key4: [ "key14_ignored", "key4_ignored" ] + key5: + key6: "key6_ignored" + key7: "key7" + key15: + key16: "key16_ignored" + key17: [ "key17", "key18" ] + + - is_false: fields + --- completion: - requires: @@ -1060,9 +1135,10 @@ completion: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: completion: type: completion diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/10_basic.yml index 8242b7cdd29e7..d0e1759073e1b 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/10_basic.yml @@ -149,3 +149,70 @@ indices.exists_alias: name: logs_2022-12-31 - is_true: '' + +--- +"Create lookup index": + - requires: + test_runner_features: [ capabilities, default_shards ] + capabilities: + - method: PUT + path: /{index} + capabilities: [ lookup_index_mode ] + reason: "Support for 'lookup' index mode capability required" + - do: + indices.create: + index: "test_lookup" + body: + settings: + index.mode: lookup + + - do: + indices.get_settings: + index: test_lookup + + - match: { test_lookup.settings.index.number_of_shards: "1"} + - match: { test_lookup.settings.index.auto_expand_replicas: "0-all"} + +--- +"Create lookup index with one shard": + - requires: + test_runner_features: [ capabilities, default_shards ] + capabilities: + - method: PUT + path: /{index} + capabilities: [ lookup_index_mode ] + reason: "Support for 'lookup' index mode capability required" + - do: + indices.create: + index: "test_lookup" + body: + settings: + index: + mode: lookup + number_of_shards: 1 + + - do: + indices.get_settings: + index: test_lookup + + - match: { test_lookup.settings.index.number_of_shards: "1"} + - match: { test_lookup.settings.index.auto_expand_replicas: "0-all"} + +--- +"Create lookup index with two shards": + - requires: + test_runner_features: [ capabilities ] + capabilities: + - method: PUT + path: /{index} + capabilities: [ lookup_index_mode ] + reason: "Support for 'lookup' index mode capability required" + - do: + catch: /illegal_argument_exception/ + indices.create: + index: test_lookup + body: + settings: + index.mode: lookup + index.number_of_shards: 2 + diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml index 9dd6cec6e657c..37f4fcc957f72 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml @@ -1,6 +1,12 @@ +setup: + - requires: + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" + +--- object with unmapped fields: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.ignored_source.dont_expand_dots"] reason: requires tracking ignored source - do: @@ -10,13 +16,11 @@ object with unmapped fields: settings: index: mapping: + source.mode: synthetic total_fields: ignore_dynamic_beyond_limit: true limit: 1 - mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -41,13 +45,13 @@ object with unmapped fields: - match: { hits.hits.0._source.some_string: AaAa } - match: { hits.hits.0._source.some_int: 1000 } - match: { hits.hits.0._source.some_double: 123.456789 } - - match: { hits.hits.0._source.a.very.deeply.nested.field: AAAA } + - match: { hits.hits.0._source.a: { very.deeply.nested.field: AAAA } } - match: { hits.hits.0._source.some_bool: true } - match: { hits.hits.1._source.name: bbbb } - match: { hits.hits.1._source.some_string: BbBb } - match: { hits.hits.1._source.some_int: 2000 } - match: { hits.hits.1._source.some_double: 321.987654 } - - match: { hits.hits.1._source.a.very.deeply.nested.field: BBBB } + - match: { hits.hits.1._source.a: { very.deeply.nested.field: BBBB } } --- @@ -63,13 +67,12 @@ unmapped arrays: settings: index: mapping: + source.mode: synthetic total_fields: ignore_dynamic_beyond_limit: true limit: 1 mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -100,7 +103,7 @@ unmapped arrays: --- nested object with unmapped fields: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.ignored_source.dont_expand_dots"] reason: requires tracking ignored source - do: @@ -110,13 +113,12 @@ nested object with unmapped fields: settings: index: mapping: + source.mode: synthetic total_fields: ignore_dynamic_beyond_limit: true limit: 3 mappings: - _source: - mode: synthetic properties: path: properties: @@ -143,16 +145,16 @@ nested object with unmapped fields: - match: { hits.total.value: 2 } - match: { hits.hits.0._source.path.to.name: aaaa } - match: { hits.hits.0._source.path.to.surname: AaAa } - - match: { hits.hits.0._source.path.some.other.name: AaAaAa } + - match: { hits.hits.0._source.path.some.other\.name: AaAaAa } - match: { hits.hits.1._source.path.to.name: bbbb } - match: { hits.hits.1._source.path.to.surname: BbBb } - - match: { hits.hits.1._source.path.some.other.name: BbBbBb } + - match: { hits.hits.1._source.path.some.other\.name: BbBbBb } --- empty object with unmapped fields: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.ignored_source.dont_expand_dots"] reason: requires tracking ignored source - do: @@ -162,13 +164,12 @@ empty object with unmapped fields: settings: index: mapping: + source.mode: synthetic total_fields: ignore_dynamic_beyond_limit: true limit: 3 mappings: - _source: - mode: synthetic properties: path: properties: @@ -191,22 +192,23 @@ empty object with unmapped fields: - match: { hits.total.value: 1 } - match: { hits.hits.0._source.path.to.surname: AaAa } - - match: { hits.hits.0._source.path.some.other.name: AaAaAa } + - match: { hits.hits.0._source.path.some.other\.name: AaAaAa } --- disabled root object: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.ignored_source.dont_expand_dots"] reason: requires tracking ignored source - do: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic enabled: false - do: @@ -222,26 +224,29 @@ disabled root object: index: test - match: { hits.total.value: 1 } - - match: { hits.hits.0._source.name: aaaa } - - match: { hits.hits.0._source.some_string: AaAa } - - match: { hits.hits.0._source.some_int: 1000 } - - match: { hits.hits.0._source.some_double: 123.456789 } - - match: { hits.hits.0._source.a.very.deeply.nested.field: AAAA } - + - match: + hits.hits.0._source: + name: aaaa + some_string: AaAa + some_int: 1000 + some_double: 123.456789 + some_bool: true + a.very.deeply.nested.field: AAAA --- disabled object: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.ignored_source.dont_expand_dots"] reason: requires tracking ignored source - do: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: path: enabled: false @@ -261,23 +266,25 @@ disabled object: - match: { hits.total.value: 1 } - match: { hits.hits.0._source.name: aaaa } - - match: { hits.hits.0._source.path.some_int: 1000 } - - match: { hits.hits.0._source.path.to.a.very.deeply.nested.field: AAAA } - + - match: + hits.hits.0._source.path: + some_int: 1000 + to.a.very.deeply.nested.field: AAAA --- disabled object contains array: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.ignored_source.dont_expand_dots"] reason: requires tracking ignored source - do: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: path: enabled: false @@ -297,10 +304,12 @@ disabled object contains array: - match: { hits.total.value: 1 } - match: { hits.hits.0._source.name: aaaa } - - match: { hits.hits.0._source.path.0.some_int: 1000 } - - match: { hits.hits.0._source.path.0.to.a.very.deeply.nested.field: AAAA } - - match: { hits.hits.0._source.path.1.some_double: 10.0 } - - match: { hits.hits.0._source.path.1.some_bool: true } + - match: + hits.hits.0._source.path: + - some_int: 1000 + to.a.very.deeply.nested.field: AAAA + - some_double: 10.0 + some_bool: true --- @@ -313,9 +322,10 @@ disabled subobject: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: path: properties: @@ -351,9 +361,10 @@ disabled subobject with array: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: path: properties: @@ -390,9 +401,10 @@ mixed disabled and enabled objects: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: path: properties: @@ -429,16 +441,17 @@ mixed disabled and enabled objects: --- object with dynamic override: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.ignored_source.dont_expand_dots"] reason: requires tracking ignored source - do: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: path_no: dynamic: false @@ -467,10 +480,10 @@ object with dynamic override: - match: { hits.hits.0._source.name: a } - match: { hits.hits.0._source.path_no.name: foo } - match: { hits.hits.0._source.path_no.some_int: 10 } - - match: { hits.hits.0._source.path_no.to.a.very.deeply.nested.field: A } + - match: { hits.hits.0._source.path_no.to: { a.very.deeply.nested.field: A } } - match: { hits.hits.0._source.path_runtime.name: bar } - match: { hits.hits.0._source.path_runtime.some_int: 20 } - - match: { hits.hits.0._source.path_runtime.to.a.very.deeply.nested.field: B } + - match: { hits.hits.0._source.path_runtime.to: { a.very.deeply.nested.field: B } } --- @@ -483,9 +496,10 @@ subobject with dynamic override: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: path: properties: @@ -524,17 +538,20 @@ subobject with dynamic override: --- object array in object with dynamic override: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.synthetic_source_keep"] reason: requires tracking ignored source - do: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: + id: + type: integer path_no: dynamic: false properties: @@ -552,18 +569,25 @@ object array in object with dynamic override: refresh: true body: - '{ "create": { } }' - - '{ "path_no": [ { "some_int": 10 }, {"name": "foo"} ], "path_runtime": [ { "some_int": 20 }, {"name": "bar"} ], "name": "baz" }' + - '{ "id": 1, "path_no": [ { "some_int": 30 }, {"name": "baz"}, { "some_int": 20 }, {"name": "bar"} ], "name": "A" }' + - '{ "create": { } }' + - '{ "id": 2, "path_runtime": [ { "some_int": 30 }, {"name": "baz"}, { "some_int": 20 }, {"name": "bar"} ], "name": "B" }' + - match: { errors: false } - do: search: index: test + sort: id - - match: { hits.total.value: 1 } - - match: { hits.hits.0._source.name: baz } - - match: { hits.hits.0._source.path_no.0.some_int: 10 } - - match: { hits.hits.0._source.path_no.1.name: foo } - - match: { hits.hits.0._source.path_runtime.0.some_int: 20 } - - match: { hits.hits.0._source.path_runtime.1.name: bar } + - match: { hits.hits.0._source.id: 1 } + - match: { hits.hits.0._source.name: A } + - match: { hits.hits.0._source.path_no.some_int: [ 30, 20 ] } + - match: { hits.hits.0._source.path_no.name: [ bar, baz ] } + + - match: { hits.hits.1._source.id: 2 } + - match: { hits.hits.1._source.name: B } + - match: { hits.hits.1._source.path_runtime.some_int: [ 30, 20 ] } + - match: { hits.hits.1._source.path_runtime.name: [ bar, baz ] } --- @@ -576,9 +600,10 @@ value array in object with dynamic override: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: path_no: dynamic: false @@ -619,9 +644,10 @@ nested object: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: nested_field: type: nested @@ -664,9 +690,10 @@ nested object next to regular: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: path: properties: @@ -710,9 +737,10 @@ nested object with disabled: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: obj_field: properties: @@ -798,9 +826,10 @@ doubly nested object: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: obj_field: properties: @@ -893,9 +922,10 @@ subobjects auto: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic subobjects: auto properties: id: @@ -911,7 +941,7 @@ subobjects auto: id: type: keyword stored: - store_array_source: true + synthetic_source_keep: arrays properties: span: properties: @@ -974,16 +1004,17 @@ subobjects auto: --- synthetic_source with copy_to: - requires: - cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"] + cluster_features: ["mapper.source.synthetic_source_copy_to_fix"] reason: requires copy_to support in synthetic source - do: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: number: type: integer @@ -1068,33 +1099,43 @@ synthetic_source with copy_to: fields: ["number_copy", "boolean_copy", "keyword_copy", "date_copy", "text_copy", "ip_copy", "ip_range_copy", "geo_point_copy", "binary_copy", "scaled_float_copy"] - match: { hits.hits.0._source.number: 100 } + - match: { hits.hits.0._source.number_copy: null } - match: { hits.hits.0.fields.number_copy.0: "100" } - match: { hits.hits.0._source.boolean: false } + - match: { hits.hits.0._source.boolean_copy: null } - match: { hits.hits.0.fields.boolean_copy.0: "false" } - match: { hits.hits.0._source.keyword: "hello_keyword" } + - match: { hits.hits.0._source.keyword_copy: null } - match: { hits.hits.0.fields.keyword_copy.0: "hello_keyword" } - match: { hits.hits.0._source.date: "2015-01-01T12:10:30Z" } + - match: { hits.hits.0._source.date_copy: null } - match: { hits.hits.0.fields.date_copy.0: "2015-01-01T12:10:30Z" } - match: { hits.hits.0._source.text: "hello_text" } + - match: { hits.hits.0._source.text_copy: null } - match: { hits.hits.0.fields.text_copy.0: "hello_text" } - match: { hits.hits.0._source.ip: "192.168.1.1" } + - match: { hits.hits.0._source.ip_copy: null } - match: { hits.hits.0.fields.ip_copy.0: "192.168.1.1" } - match: { hits.hits.0._source.ip_range: "10.0.0.0/24" } + - match: { hits.hits.0._source.ip_range_copy: null } - match: { hits.hits.0.fields.ip_range_copy.0: "10.0.0.0/24" } - match: { hits.hits.0._source.geo_point: "POINT (-71.34 41.12)" } + - match: { hits.hits.0._source.geo_point_copy: null } - match: { hits.hits.0.fields.geo_point_copy.0: "POINT (-71.34 41.12)" } - match: { hits.hits.0._source.binary: "aGVsbG8gY3VyaW91cyBwZXJzb24=" } + - match: { hits.hits.0._source.binary_copy: null } - match: { hits.hits.0.fields.binary_copy.0: "aGVsbG8gY3VyaW91cyBwZXJzb24=" } - match: { hits.hits.0._source.scaled_float: 1.5 } + - match: { hits.hits.0._source.scaled_float_copy: null } - match: { hits.hits.0.fields.scaled_float_copy.0: "1.5" } --- @@ -1107,9 +1148,10 @@ synthetic_source with disabled doc_values: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: number: type: integer @@ -1190,9 +1232,10 @@ fallback synthetic_source for text field: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: text: type: text @@ -1214,3 +1257,796 @@ fallback synthetic_source for text field: hits.hits.0._source: text: [ "world", "hello", "world" ] +--- +synthetic_source with copy_to and ignored values: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + name: + type: keyword + k: + type: keyword + ignore_above: 1 + copy_to: copy + long: + type: long + ignore_malformed: true + copy_to: copy + copy: + type: keyword + + - do: + index: + index: test + id: 1 + refresh: true + body: + name: "A" + k: "hello" + long: "world" + + - do: + index: + index: test + id: 2 + refresh: true + body: + name: "B" + k: ["55", "66"] + long: ["77", "88"] + + - do: + search: + index: test + sort: name + body: + docvalue_fields: [ "copy" ] + + - match: + hits.hits.0._source: + name: "A" + k: "hello" + long: "world" + - match: { hits.hits.0.fields.copy: ["hello", "world"] } + + - match: + hits.hits.1._source: + name: "B" + k: ["55", "66"] + long: ["77", "88"] + - match: { hits.hits.1.fields.copy: ["55", "66", "77", "88"] } + +--- +synthetic_source with copy_to field having values in source: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + name: + type: keyword + k: + type: keyword + copy_to: copy + copy: + type: keyword + + - do: + index: + index: test + id: 1 + refresh: true + body: + name: "A" + copy: "world" + k: "hello" + + - do: + index: + index: test + id: 2 + refresh: true + body: + name: "B" + k: ["5", "6"] + copy: ["7", "8"] + + - do: + search: + index: test + sort: name + body: + docvalue_fields: [ "copy" ] + + - match: + hits.hits.0._source: + name: "A" + k: "hello" + copy: "world" + - match: { hits.hits.0.fields.copy: ["hello", "world"] } + + - match: + hits.hits.1._source: + name: "B" + k: ["5", "6"] + copy: ["7", "8"] + - match: { hits.hits.1.fields.copy: ["5", "6", "7", "8"] } + +--- +synthetic_source with ignored source field using copy_to: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + name: + type: keyword + k: + type: keyword + doc_values: false + copy_to: copy + copy: + type: keyword + + - do: + index: + index: test + id: 1 + refresh: true + body: + name: "A" + copy: "world" + k: "hello" + + - do: + index: + index: test + id: 2 + refresh: true + body: + name: "B" + k: ["5", "6"] + copy: ["7", "8"] + + - do: + search: + index: test + sort: name + body: + docvalue_fields: [ "copy" ] + + - match: + hits.hits.0._source: + name: "A" + k: "hello" + copy: "world" + - match: { hits.hits.0.fields.copy: ["hello", "world"] } + + - match: + hits.hits.1._source: + name: "B" + k: ["5", "6"] + copy: ["7", "8"] + - match: { hits.hits.1.fields.copy: ["5", "6", "7", "8"] } + +--- +synthetic_source with copy_to field from dynamic template having values in source: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + dynamic_templates: + - copy_template: + match: "k" + mapping: + type: keyword + copy_to: copy + properties: + name: + type: keyword + copy: + type: keyword + + - do: + index: + index: test + id: 1 + refresh: true + body: + name: "A" + k: "hello" + + - do: + index: + index: test + id: 2 + refresh: true + body: + name: "B" + copy: "world" + k: "hello" + + - do: + index: + index: test + id: 3 + refresh: true + body: + name: "C" + k: ["5", "6"] + + - do: + index: + index: test + id: 4 + refresh: true + body: + name: "D" + k: ["5", "6"] + copy: ["7", "8"] + + - do: + search: + index: test + sort: name + body: + docvalue_fields: [ "copy" ] + + - match: + hits.hits.0._source: + name: "A" + k: "hello" + - match: { hits.hits.0.fields.copy: ["hello"] } + + - match: + hits.hits.1._source: + name: "B" + k: "hello" + copy: "world" + - match: { hits.hits.1.fields.copy: ["hello", "world"] } + + - match: + hits.hits.2._source: + name: "C" + k: ["5", "6"] + - match: { hits.hits.2.fields.copy: ["5", "6"] } + + - match: + hits.hits.3._source: + name: "D" + k: ["5", "6"] + copy: ["7", "8"] + - match: { hits.hits.3.fields.copy: ["5", "6", "7", "8"] } + +--- +synthetic_source with copy_to and invalid values for copy: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_fix"] + reason: requires copy_to support in synthetic source + test_runner_features: "contains" + + - do: + indices.create: + index: test + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + name: + type: keyword + p: + type: long_range + copy_to: copy + copy: + type: keyword + + - do: + catch: bad_request + index: + index: test + id: 1 + refresh: true + body: + name: "A" + p: + gte: 10 + + - match: { error.type: "document_parsing_exception" } + - contains: { error.reason: "Copy-to currently only works for value-type fields" } + +--- +synthetic_source with copy_to pointing inside object: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + name: + type: keyword + my_values: + properties: + k: + type: keyword + ignore_above: 1 + copy_to: c.copy + long: + type: long + copy_to: c.copy + c: + properties: + copy: + type: keyword + + - do: + index: + index: test + id: 1 + refresh: true + body: + name: "A" + my_values: + k: "hello" + long: 100 + + - do: + index: + index: test + id: 2 + refresh: true + body: + name: "B" + my_values: + k: ["55", "66"] + long: [77, 88] + + - do: + index: + index: test + id: 3 + refresh: true + body: + name: "C" + my_values: + k: "hello" + long: 100 + c: + copy: "zap" + + - do: + search: + index: test + sort: name + body: + docvalue_fields: [ "c.copy" ] + + - match: + hits.hits.0._source: + name: "A" + my_values: + k: "hello" + long: 100 + - match: + hits.hits.0.fields: + c.copy: [ "100", "hello" ] + + - match: + hits.hits.1._source: + name: "B" + my_values: + k: ["55", "66"] + long: [77, 88] + - match: + hits.hits.1.fields: + c.copy: ["55", "66", "77", "88"] + + - match: + hits.hits.2._source: + name: "C" + my_values: + k: "hello" + long: 100 + c: + copy: "zap" + - match: + hits.hits.2.fields: + c.copy: [ "100", "hello", "zap" ] + +--- +synthetic_source with copy_to pointing to ambiguous field: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + k: + type: keyword + copy_to: a.b.c + a: + properties: + b: + properties: + c: + type: keyword + b.c: + type: keyword + + - do: + index: + index: test + id: 1 + refresh: true + body: + k: "hey" + + - do: + search: + index: test + body: + docvalue_fields: [ "a.b.c" ] + + - match: + hits.hits.0._source: + k: "hey" + - match: + hits.hits.0.fields: + a.b.c: [ "hey" ] + +--- +synthetic_source with copy_to pointing to ambiguous field and subobjects false: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + subobjects: false + properties: + k: + type: keyword + copy_to: a.b.c + a: + properties: + b: + properties: + c: + type: keyword + b.c: + type: keyword + + - do: + index: + index: test + id: 1 + refresh: true + body: + k: "hey" + + - do: + search: + index: test + body: + docvalue_fields: [ "a.b.c" ] + + - match: + hits.hits.0._source: + k: "hey" + - match: + hits.hits.0.fields: + a.b.c: [ "hey" ] + +--- +synthetic_source with copy_to pointing to ambiguous field and subobjects auto: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + subobjects: auto + properties: + k: + type: keyword + copy_to: a.b.c + a: + properties: + b: + properties: + c: + type: keyword + b.c: + type: keyword + + - do: + index: + index: test + id: 1 + refresh: true + body: + k: "hey" + + - do: + search: + index: test + body: + docvalue_fields: [ "a.b.c" ] + + - match: + hits.hits.0._source: + k: "hey" + - match: + hits.hits.0.fields: + a.b.c: [ "hey" ] + +--- +synthetic_source with copy_to pointing at dynamic field: + - requires: + test_runner_features: contains + cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + name: + type: keyword + k: + type: keyword + copy_to: c.copy + c: + properties: + f: + type: float + + - do: + index: + index: test + id: 1 + refresh: true + body: + name: "A" + k: "hello" + + - do: + index: + index: test + id: 2 + refresh: true + body: + name: "B" + k: ["55", "66"] + + - do: + index: + index: test + id: 3 + refresh: true + body: + name: "C" + k: "hello" + c: + copy: "zap" + + - do: + search: + index: test + sort: name + body: + docvalue_fields: [ "c.copy.keyword" ] + + - match: + hits.hits.0._source: + name: "A" + k: "hello" + - match: + hits.hits.0.fields: + c.copy.keyword: [ "hello" ] + + - match: + hits.hits.1._source: + name: "B" + k: ["55", "66"] + - match: + hits.hits.1.fields: + c.copy.keyword: [ "55", "66" ] + + - match: + hits.hits.2._source: + name: "C" + k: "hello" + c: + copy: "zap" + - match: + hits.hits.2.fields: + c.copy.keyword: [ "hello", "zap" ] + +--- +synthetic_source with copy_to pointing inside dynamic object: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + name: + type: keyword + k: + type: keyword + copy_to: c.copy + + - do: + index: + index: test + id: 1 + refresh: true + body: + name: "A" + k: "hello" + + - do: + index: + index: test + id: 2 + refresh: true + body: + name: "B" + k: ["55", "66"] + + - do: + index: + index: test + id: 3 + refresh: true + body: + name: "C" + k: "hello" + c: + copy: "zap" + + - do: + search: + index: test + sort: name + body: + docvalue_fields: [ "c.copy.keyword" ] + + - match: + hits.hits.0._source: + name: "A" + k: "hello" + - match: + hits.hits.0.fields: + c.copy.keyword: [ "hello" ] + + - match: + hits.hits.1._source: + name: "B" + k: ["55", "66"] + - match: + hits.hits.1.fields: + c.copy.keyword: [ "55", "66" ] + + - match: + hits.hits.2._source: + name: "C" + k: "hello" + c: + copy: "zap" + - match: + hits.hits.2.fields: + c.copy.keyword: [ "hello", "zap" ] + +--- +create index with use_synthetic_source: + - requires: + cluster_features: ["mapper.synthetic_recovery_source"] + reason: requires synthetic recovery source + + - do: + indices.create: + index: test + body: + settings: + index: + recovery: + use_synthetic_source: true + mapping: + source: + mode: synthetic + + - do: + indices.get_settings: {} + - match: { test.settings.index.mapping.source.mode: synthetic} + - is_true: test.settings.index.recovery.use_synthetic_source + + - do: + index: + index: test + id: 1 + refresh: true + body: { foo: bar } + - match: { _version: 1 } + + - do: + indices.disk_usage: + index: test + run_expensive_tasks: true + flush: false + - gt: { test.store_size_in_bytes: 0 } + - is_false: test.fields._recovery_source diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml index 917f0540c4dd4..262e646cc882b 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml @@ -1,16 +1,87 @@ +setup: + - requires: + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" + +--- +object param - store complex object: + - requires: + cluster_features: ["mapper.synthetic_source_keep"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + id: + type: integer + stored: + synthetic_source_keep: all + properties: + object_array: + properties: + trace: + type: keyword + nested: + type: nested + kw: + type: keyword + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "id": 1, "stored": { "object_array": [ {"trace": "B"}, {"trace": "A"} ], "nested": [ {"foo": 20}, {"foo": 10} ], "kw": 100 } }' + - '{ "create": { } }' + - '{ "id": 2, "stored": { "object_array": { "trace": ["D", "C"] }, "nested": { "bar": [ 40, 30] }, "kw": 200, "baz": "2000" } }' + - '{ "create": { } }' + - '{ "id": 3, "stored": [ { "object_array": { "trace": "E" } }, { "nested": { "bar": [ 60, 50] } }, { "kw": 300 } ] }' + + - do: + search: + index: test + sort: id + + - match: { hits.hits.0._source.id: 1 } + - match: { hits.hits.0._source.stored.object_array.0.trace: B } + - match: { hits.hits.0._source.stored.object_array.1.trace: A } + - match: { hits.hits.0._source.stored.nested.0.foo: 20 } + - match: { hits.hits.0._source.stored.nested.1.foo: 10 } + - match: { hits.hits.0._source.stored.kw: 100 } + + - match: { hits.hits.1._source.id: 2 } + - match: { hits.hits.1._source.stored.object_array.trace: [D, C] } + - match: { hits.hits.1._source.stored.nested.bar: [40, 30] } + - match: { hits.hits.1._source.stored.kw: 200 } + - match: { hits.hits.1._source.stored.baz: "2000" } + + - match: { hits.hits.2._source.id: 3 } + - match: { hits.hits.2._source.stored.0.object_array.trace: E } + - match: { hits.hits.2._source.stored.1.nested.bar: [ 60, 50 ] } + - match: { hits.hits.2._source.stored.2.kw: 300 } + + --- object param - object array: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.synthetic_source_keep"] reason: requires tracking ignored source - do: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: id: type: integer @@ -25,7 +96,7 @@ object param - object array: id: type: keyword stored: - store_array_source: true + synthetic_source_keep: arrays properties: span: properties: @@ -65,22 +136,23 @@ object param - object array: --- object param - object array within array: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.synthetic_source_keep"] reason: requires tracking ignored source - do: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: stored: - store_array_source: true + synthetic_source_keep: arrays properties: path: - store_array_source: true + synthetic_source_keep: arrays properties: to: properties: @@ -108,19 +180,20 @@ object param - object array within array: --- object param - no object array: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.synthetic_source_keep"] reason: requires tracking ignored source - do: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: stored: - store_array_source: true + synthetic_source_keep: arrays properties: span: properties: @@ -150,21 +223,22 @@ object param - no object array: --- object param - field ordering in object array: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.synthetic_source_keep"] reason: requires tracking ignored source - do: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: a: type: keyword b: - store_array_source: true + synthetic_source_keep: arrays properties: aa: type: keyword @@ -173,7 +247,7 @@ object param - field ordering in object array: c: type: keyword d: - store_array_source: true + synthetic_source_keep: arrays properties: aa: type: keyword @@ -199,23 +273,24 @@ object param - field ordering in object array: --- object param - nested object array next to other fields: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.synthetic_source_keep"] reason: requires tracking ignored source - do: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: a: type: keyword b: properties: c: - store_array_source: true + synthetic_source_keep: arrays properties: aa: type: keyword @@ -255,16 +330,17 @@ object param - nested object array next to other fields: --- object param - nested object with stored array: - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source + cluster_features: ["mapper.ignored_source.always_store_object_arrays_in_nested"] + reason: requires fix to object array handling - do: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -272,7 +348,7 @@ object param - nested object with stored array: type: nested nested_array_stored: type: nested - store_array_source: true + synthetic_source_keep: all - do: bulk: @@ -292,8 +368,8 @@ object param - nested object with stored array: sort: name - match: { hits.total.value: 2 } - match: { hits.hits.0._source.name: A } - - match: { hits.hits.0._source.nested_array_regular.0.b.c: [ 10, 100] } - - match: { hits.hits.0._source.nested_array_regular.1.b.c: [ 20, 200] } + - match: { hits.hits.0._source.nested_array_regular.0.b.c: [ 10, 100 ] } + - match: { hits.hits.0._source.nested_array_regular.1.b.c: [ 20, 200 ] } - match: { hits.hits.1._source.name: B } - match: { hits.hits.1._source.nested_array_stored.0.b.0.c: 10 } - match: { hits.hits.1._source.nested_array_stored.0.b.1.c: 100 } @@ -302,10 +378,58 @@ object param - nested object with stored array: --- +index param - nested array within array: + - requires: + cluster_features: ["mapper.synthetic_source_keep"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + name: + type: keyword + path: + properties: + to: + properties: + some: + synthetic_source_keep: arrays + properties: + id: + type: integer + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "name": "A", "path": [ { "to": [ { "some" : [ { "id": 10 }, { "id": [1, 3, 2] } ] }, { "some": { "id": 100 } } ] }, { "to": { "some": { "id": [1000, 2000] } } } ] }' + - match: { errors: false } + + - do: + search: + index: test + sort: name + - match: { hits.hits.0._source.name: A } + - match: { hits.hits.0._source.path.to.some.0.id: 10 } + - match: { hits.hits.0._source.path.to.some.1.id: [ 1, 3, 2] } + - match: { hits.hits.0._source.path.to.some.2.id: 100 } + - match: { hits.hits.0._source.path.to.some.3.id: [ 1000, 2000 ] } + + +--- + # 112156 stored field under object with store_array_source: - requires: - cluster_features: ["mapper.source.synthetic_source_stored_fields_advance_fix"] + cluster_features: ["mapper.synthetic_source_keep"] reason: requires bug fix to be implemented - do: @@ -316,14 +440,13 @@ stored field under object with store_array_source: index: sort.field: "name" sort.order: "asc" + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: name: type: keyword obj: - store_array_source: true + synthetic_source_keep: arrays properties: foo: type: keyword @@ -354,11 +477,354 @@ stored field under object with store_array_source: --- -index param - root arrays: +field param - keep root array: + - requires: + cluster_features: ["mapper.synthetic_source_keep"] + reason: requires keeping array source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + id: + type: integer + kw_default: + type: keyword + kw_arrays: + type: keyword + synthetic_source_keep: arrays + kw_all: + type: keyword + synthetic_source_keep: all + int_default: + type: integer + int_arrays: + type: integer + synthetic_source_keep: arrays + int_all: + type: integer + synthetic_source_keep: all + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "id": 1, "kw_default": [ "B", "A" ], "kw_arrays": [ "B", "A" ], "kw_all": [ "B", "A" ] }' + - '{ "create": { } }' + - '{ "id": 2, "int_default": [ 2, 1 ], "int_arrays": [ 2, 1 ], "int_all": [ 2, 1 ] }' + + - do: + search: + index: test + sort: id + + - match: { hits.hits.0._source.kw_default: [ "A", "B" ] } + - match: { hits.hits.0._source.kw_arrays: [ "B", "A" ] } + - match: { hits.hits.0._source.kw_all: [ "B", "A" ] } + + - match: { hits.hits.1._source.int_default: [ 1, 2 ] } + - match: { hits.hits.1._source.int_arrays: [ 2, 1 ] } + - match: { hits.hits.1._source.int_all: [ 2, 1 ] } + + +--- +field param - keep nested array: + - requires: + cluster_features: ["mapper.synthetic_source_keep"] + reason: requires keeping array source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + id: + type: integer + path: + properties: + to: + properties: + ratio: + type: float + id: + type: integer + kw_default: + type: keyword + kw_arrays: + type: keyword + synthetic_source_keep: arrays + kw_all: + type: keyword + synthetic_source_keep: all + int_default: + type: integer + int_arrays: + type: integer + synthetic_source_keep: arrays + int_all: + type: integer + synthetic_source_keep: all + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "id": 1, "path": { "to": { "ratio": 10.0, "kw_default": [ "B", "A" ], "kw_arrays": [ "B", "A" ], "kw_all": [ "B", "A" ] } } }' + - '{ "create": { } }' + - '{ "id": 2, "path": { "to": { "ratio": 20.0, "int_default": [ 2, 1 ], "int_arrays": [ 2, 1 ], "int_all": [ 2, 1 ] } } }' + + - do: + search: + index: test + sort: id + + - match: { hits.hits.0._source.path.to.ratio: 10.0 } + - match: { hits.hits.0._source.path.to.kw_default: [ "A", "B" ] } + - match: { hits.hits.0._source.path.to.kw_arrays: [ "B", "A" ] } + - match: { hits.hits.0._source.path.to.kw_all: [ "B", "A" ] } + + - match: { hits.hits.1._source.path.to.ratio: 20.0 } + - match: { hits.hits.1._source.path.to.int_default: [ 1, 2 ] } + - match: { hits.hits.1._source.path.to.int_arrays: [ 2, 1 ] } + - match: { hits.hits.1._source.path.to.int_all: [ 2, 1 ] } + + +--- +field param - keep root singleton fields: + - requires: + test_runner_features: close_to + cluster_features: ["mapper.synthetic_source_keep"] + reason: requires keeping singleton source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + id: + type: integer + l: + type: long + synthetic_source_keep: all + l_def: + type: long + f: + type: float + synthetic_source_keep: all + f_def: + type: float + synthetic_source_keep: all + kw: + type: keyword + synthetic_source_keep: all + kw_def: + type: keyword + txt: + type: text + store: true + synthetic_source_keep: all + txt_def: + type: text + store: true + gp: + type: geo_point + synthetic_source_keep: all + gp_def: + type: geo_point + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "id": 1, "l": 10.0, "l_def": 10.0, "f": 10, "f_def": 10, "kw": 10, "kw_def": 10, "txt": 10, "txt_def": 10, "gp": "POINT(34.25 -21.76)", "gp_def": "POINT(34.25 -21.76)" }' + - '{ "create": { } }' + - '{ "id": 2, "l": 20.0, "l_def": 20.0, "f": 20, "f_def": 20, "kw": 20, "kw_def": 20, "txt": 20, "txt_def": 20, "gp": "POINT(44.25 -31.76)", "gp_def": "POINT(44.25 -31.76)" }' + - match: { errors: false } + + - do: + search: + index: test + sort: id + + - match: { hits.hits.0._source.id: 1 } + - match: { hits.hits.0._source.l: 10.0 } + - match: { hits.hits.0._source.l_def: 10 } + - match: { hits.hits.0._source.f: 10 } + - match: { hits.hits.0._source.f_def: 10.0 } + - match: { hits.hits.0._source.kw: 10 } + - match: { hits.hits.0._source.kw_def: "10" } + - match: { hits.hits.0._source.txt: 10 } + - match: { hits.hits.0._source.txt_def: "10" } + - match: { hits.hits.0._source.gp: "POINT(34.25 -21.76)" } + - close_to: { hits.hits.0._source.gp_def.lon: { value: 34.25, error: 0.01 } } + - close_to: { hits.hits.0._source.gp_def.lat: { value: -21.76, error: 0.01 } } + + - match: { hits.hits.1._source.id: 2 } + - match: { hits.hits.1._source.l: 20.0 } + - match: { hits.hits.1._source.l_def: 20 } + - match: { hits.hits.1._source.f: 20 } + - match: { hits.hits.1._source.f_def: 20.0 } + - match: { hits.hits.1._source.kw: 20 } + - match: { hits.hits.1._source.kw_def: "20" } + - match: { hits.hits.1._source.txt: 20 } + - match: { hits.hits.1._source.txt_def: "20" } + - match: { hits.hits.1._source.gp: "POINT(44.25 -31.76)" } + - close_to: { hits.hits.1._source.gp_def.lon: { value: 44.25, error: 0.01 } } + - close_to: { hits.hits.1._source.gp_def.lat: { value: -31.76, error: 0.01 } } + + +--- +field param - keep nested singleton fields: + - requires: + test_runner_features: close_to + cluster_features: ["mapper.synthetic_source_keep"] + reason: requires keeping singleton source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + id: + type: integer + path: + properties: + to: + properties: + l: + type: long + synthetic_source_keep: all + l_def: + type: long + f: + type: float + synthetic_source_keep: all + f_def: + type: float + synthetic_source_keep: all + kw: + type: keyword + synthetic_source_keep: all + kw_def: + type: keyword + gp: + type: geo_point + synthetic_source_keep: all + gp_def: + type: geo_point + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "id": 1, "path": { "to": { "l": 10.0, "l_def": 10.0, "f": 10, "f_def": 10, "kw": 10, "kw_def": 10, "gp": "POINT(34.25 -21.76)", "gp_def": "POINT(34.25 -21.76)" } } }' + - '{ "create": { } }' + - '{ "id": 2, "path": { "to": { "l": 20.0, "l_def": 20.0, "f": 20, "f_def": 20, "kw": 20, "kw_def": 20, "gp": "POINT(44.25 -31.76)", "gp_def": "POINT(44.25 -31.76)" } } }' + - match: { errors: false } + + - do: + search: + index: test + sort: id + + - match: { hits.hits.0._source.id: 1 } + - match: { hits.hits.0._source.path.to.l: 10.0 } + - match: { hits.hits.0._source.path.to.l_def: 10 } + - match: { hits.hits.0._source.path.to.f: 10 } + - match: { hits.hits.0._source.path.to.f_def: 10.0 } + - match: { hits.hits.0._source.path.to.kw: 10 } + - match: { hits.hits.0._source.path.to.kw_def: "10" } + - match: { hits.hits.0._source.path.to.gp: "POINT(34.25 -21.76)" } + - close_to: { hits.hits.0._source.path.to.gp_def.lon: { value: 34.25, error: 0.01 } } + - close_to: { hits.hits.0._source.path.to.gp_def.lat: { value: -21.76, error: 0.01 } } + + - match: { hits.hits.1._source.id: 2 } + - match: { hits.hits.1._source.path.to.l: 20.0 } + - match: { hits.hits.1._source.path.to.l_def: 20 } + - match: { hits.hits.1._source.path.to.f: 20 } + - match: { hits.hits.1._source.path.to.f_def: 20.0 } + - match: { hits.hits.1._source.path.to.kw: 20 } + - match: { hits.hits.1._source.path.to.kw_def: "20" } + - match: { hits.hits.1._source.path.to.gp: "POINT(44.25 -31.76)" } + - close_to: { hits.hits.1._source.path.to.gp_def.lon: { value: 44.25, error: 0.01 } } + - close_to: { hits.hits.1._source.path.to.gp_def.lat: { value: -31.76, error: 0.01 } } + + +--- +field param - nested array within array: - requires: cluster_features: ["mapper.synthetic_source_keep"] reason: requires tracking ignored source + - do: + indices.create: + index: test + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + name: + type: keyword + path: + properties: + to: + properties: + some: + properties: + id: + type: integer + synthetic_source_keep: arrays + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "name": "A", "path": [ { "to": [ { "some" : [ { "id": 10 }, { "id": [1, 3, 2] } ] }, { "some": { "id": 100 } } ] }, { "to": { "some": { "id": [1000, 2000] } } } ] }' + - match: { errors: false } + + - do: + search: + index: test + sort: name + - match: { hits.hits.0._source.name: A } + - match: { hits.hits.0._source.path.to.some.id: [ 10, 1, 3, 2, 100, 1000, 2000 ] } + + +--- +index param - root arrays: + - requires: + cluster_features: ["mapper.synthetic_source_keep"] + reason: requires keeping array source + - do: indices.create: index: test @@ -366,15 +832,17 @@ index param - root arrays: settings: index: mapping: + source.mode: synthetic synthetic_source_keep: arrays mappings: - _source: - mode: synthetic properties: id: type: integer leaf: type: integer + leaf_default: + type: integer + synthetic_source_keep: none obj: properties: span: @@ -385,6 +853,9 @@ index param - root arrays: properties: id: type: keyword + obj_default: + type: object + synthetic_source_keep: none - do: bulk: @@ -392,9 +863,11 @@ index param - root arrays: refresh: true body: - '{ "create": { } }' - - '{ "id": 1, "leaf": [30, 20, 10], "obj": [ { "trace": { "id": "a" }, "span": { "id": "1" } }, { "trace": { "id": "b" }, "span": { "id": "1" } } ] }' + - '{ "id": 1, "leaf": [30, 20, 10], "leaf_default": [30, 20, 10], "obj": [ { "trace": { "id": "a" }, "span": { "id": "1" } }, { "trace": { "id": "b" }, "span": { "id": "1" } } ] }' - '{ "create": { } }' - - '{ "id": 2, "leaf": [130, 120, 110], "obj": [ { "trace": { "id": "aa" }, "span": { "id": "2" } }, { "trace": { "id": "bb" }, "span": { "id": "2" } } ] }' + - '{ "id": 2, "leaf": [130, 120, 110], "leaf_default": [130, 120, 110], "obj": [ { "trace": { "id": "aa" }, "span": { "id": "2" } }, { "trace": { "id": "bb" }, "span": { "id": "2" } } ] }' + - '{ "create": { } }' + - '{ "id": 3, "obj_default": [ { "trace": { "id": "bb" }, "span": { "id": "2" } }, { "trace": { "id": "aa" }, "span": { "id": "2" } } ] }' - do: search: @@ -403,6 +876,7 @@ index param - root arrays: - match: { hits.hits.0._source.id: 1 } - match: { hits.hits.0._source.leaf: [30, 20, 10] } + - match: { hits.hits.0._source.leaf_default: [10, 20, 30] } - length: { hits.hits.0._source.obj: 2 } - match: { hits.hits.0._source.obj.0.trace.id: a } - match: { hits.hits.0._source.obj.0.span.id: "1" } @@ -411,18 +885,25 @@ index param - root arrays: - match: { hits.hits.1._source.id: 2 } - match: { hits.hits.1._source.leaf: [ 130, 120, 110 ] } + - match: { hits.hits.1._source.leaf_default: [110, 120, 130] } - length: { hits.hits.1._source.obj: 2 } - match: { hits.hits.1._source.obj.0.trace.id: aa } - match: { hits.hits.1._source.obj.0.span.id: "2" } - match: { hits.hits.1._source.obj.1.trace.id: bb } - match: { hits.hits.1._source.obj.1.span.id: "2" } + - match: { hits.hits.2._source.id: 3 } + - match: { hits.hits.2._source.obj_default.trace.0.id: bb } + - match: { hits.hits.2._source.obj_default.trace.1.id: aa } + - match: { hits.hits.2._source.obj_default.span.0.id: "2" } + - match: { hits.hits.2._source.obj_default.span.1.id: "2" } + --- index param - dynamic root arrays: - requires: cluster_features: ["mapper.synthetic_source_keep"] - reason: requires tracking ignored source + reason: requires keeping array source - do: indices.create: @@ -431,10 +912,9 @@ index param - dynamic root arrays: settings: index: mapping: + source.mode: synthetic synthetic_source_keep: arrays mappings: - _source: - mode: synthetic properties: id: type: integer @@ -475,7 +955,7 @@ index param - dynamic root arrays: index param - object array within array: - requires: cluster_features: ["mapper.synthetic_source_keep"] - reason: requires tracking ignored source + reason: requires keeping array source - do: indices.create: @@ -484,10 +964,9 @@ index param - object array within array: settings: index: mapping: + source.mode: synthetic synthetic_source_keep: arrays mappings: - _source: - mode: synthetic properties: stored: properties: @@ -525,7 +1004,7 @@ index param - object array within array: index param - no object array: - requires: cluster_features: ["mapper.synthetic_source_keep"] - reason: requires tracking ignored source + reason: requires keeping array source - do: indices.create: @@ -534,10 +1013,9 @@ index param - no object array: settings: index: mapping: + source.mode: synthetic synthetic_source_keep: arrays mappings: - _source: - mode: synthetic properties: stored: properties: @@ -570,7 +1048,7 @@ index param - no object array: index param - field ordering: - requires: cluster_features: ["mapper.synthetic_source_keep"] - reason: requires tracking ignored source + reason: requires keeping array source - do: indices.create: @@ -579,10 +1057,9 @@ index param - field ordering: settings: index: mapping: + source.mode: synthetic synthetic_source_keep: arrays mappings: - _source: - mode: synthetic properties: a: type: keyword @@ -621,7 +1098,7 @@ index param - field ordering: index param - nested arrays: - requires: cluster_features: ["mapper.synthetic_source_keep"] - reason: requires tracking ignored source + reason: requires keeping array source - do: indices.create: @@ -630,10 +1107,9 @@ index param - nested arrays: settings: index: mapping: + source.mode: synthetic synthetic_source_keep: arrays mappings: - _source: - mode: synthetic properties: a: type: keyword @@ -649,6 +1125,9 @@ index param - nested arrays: type: integer e: type: keyword + g: + type: integer + synthetic_source_keep: none f: type: keyword @@ -658,9 +1137,9 @@ index param - nested arrays: refresh: true body: - '{ "create": { } }' - - '{ "a": 1, "b": { "c": [ { "bb": 10, "aa": 20 }, { "aa": 30, "bb": 40 } ], "d": [ 300, 200, 100 ], "e": 1000 }, "f": 2000 }' + - '{ "a": 1, "b": { "c": [ { "bb": 10, "aa": 20 }, { "aa": 30, "bb": 40 } ], "d": [ 300, 200, 100 ], "e": 1000, "g": [ 300, 200, 100 ] }, "f": 2000 }' - '{ "create": { } }' - - '{ "a": 11, "b": { "c": [ { "bb": 110, "aa": 120 }, { "aa": 130, "bb": 140 } ], "d": [ 1300, 1200, 1100 ], "e": 11000 }, "f": 12000 }' + - '{ "a": 11, "b": { "c": [ { "bb": 110, "aa": 120 }, { "aa": 130, "bb": 140 } ], "d": [ 1300, 1200, 1100 ], "e": 11000, "g": [ 1300, 1200, 1100 ] }, "f": 12000 }' - do: @@ -671,12 +1150,14 @@ index param - nested arrays: - match: { hits.hits.0._source.a: "1" } - match: { hits.hits.0._source.b.c: [{ "bb": 10, "aa": 20 }, { "aa": 30, "bb": 40 }] } - match: { hits.hits.0._source.b.d: [ 300, 200, 100 ] } + - match: { hits.hits.0._source.b.g: [ 100, 200, 300 ] } - match: { hits.hits.0._source.b.e: "1000" } - match: { hits.hits.0._source.f: "2000" } - match: { hits.hits.1._source.a: "11" } - match: { hits.hits.1._source.b.c: [ { "bb": 110, "aa": 120 }, { "aa": 130, "bb": 140 } ] } - match: { hits.hits.1._source.b.d: [ 1300, 1200, 1100 ] } + - match: { hits.hits.1._source.b.g: [ 1100, 1200, 1300 ] } - match: { hits.hits.1._source.b.e: "11000" } - match: { hits.hits.1._source.f: "12000" } @@ -684,7 +1165,7 @@ index param - nested arrays: index param - nested object with stored array: - requires: cluster_features: ["mapper.synthetic_source_keep"] - reason: requires tracking ignored source + reason: requires keeping array source - do: indices.create: @@ -693,10 +1174,9 @@ index param - nested object with stored array: settings: index: mapping: + source.mode: synthetic synthetic_source_keep: arrays mappings: - _source: - mode: synthetic properties: name: type: keyword @@ -730,3 +1210,51 @@ index param - nested object with stored array: - match: { hits.hits.1._source.nested.0.b.1.c: 300 } - match: { hits.hits.1._source.nested.1.b.0.c: 40 } - match: { hits.hits.1._source.nested.1.b.1.c: 400 } + + +--- +index param - flattened fields: + - requires: + cluster_features: ["mapper.synthetic_source_keep"] + reason: requires keeping array source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping: + source.mode: synthetic + synthetic_source_keep: arrays + mappings: + properties: + name: + type: keyword + outer: + properties: + inner: + type: object + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "name": "A", "outer": { "inner": [ { "a.b": "AA", "a.c": "AAA" } ] } }' + - '{ "create": { } }' + - '{ "name": "B", "outer": { "inner": [ { "a.x.y.z": "BB", "a.z.y.x": "BBB" } ] } }' + + + - match: { errors: false } + + - do: + search: + index: test + sort: name + - match: { hits.total.value: 2 } + - match: { hits.hits.0._source.name: A } + - match: { hits.hits.0._source.outer.inner: [{ a.b: AA, a.c: AAA }] } + - match: { hits.hits.1._source.name: B } + - match: { hits.hits.1._source.outer.inner: [{ a.x.y.z: BB, a.z.y.x: BBB }] } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_mapping/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_mapping/10_basic.yml index e46f67326a8d2..371a9961122a7 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_mapping/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_mapping/10_basic.yml @@ -1,85 +1,85 @@ --- setup: - do: - indices.create: - index: test_1 - body: - mappings: {} + indices.create: + index: test_1 + body: + mappings: {} - do: - indices.create: - index: test_2 - body: - mappings: {} + indices.create: + index: test_2 + body: + mappings: {} --- "Get /{index}/_mapping with empty mappings": - - do: - indices.create: - index: t + - do: + indices.create: + index: t - - do: - indices.get_mapping: - index: t + - do: + indices.get_mapping: + index: t - - match: { t.mappings: {}} + - match: { t.mappings: {}} --- "Get /_mapping": - - do: - indices.get_mapping: {} + - do: + indices.get_mapping: {} - - is_true: test_1.mappings - - is_true: test_2.mappings + - is_true: test_1.mappings + - is_true: test_2.mappings --- "Get /{index}/_mapping": - - do: - indices.get_mapping: - index: test_1 + - do: + indices.get_mapping: + index: test_1 - - is_true: test_1.mappings - - is_false: test_2 + - is_true: test_1.mappings + - is_false: test_2 --- "Get /_all/_mapping": - - do: - indices.get_mapping: - index: _all + - do: + indices.get_mapping: + index: _all - - is_true: test_1.mappings - - is_true: test_2.mappings + - is_true: test_1.mappings + - is_true: test_2.mappings --- "Get /*/_mapping": - - do: - indices.get_mapping: - index: '*' + - do: + indices.get_mapping: + index: '*' - - is_true: test_1.mappings - - is_true: test_2.mappings + - is_true: test_1.mappings + - is_true: test_2.mappings --- "Get /index,index/_mapping": - - do: - indices.get_mapping: - index: test_1,test_2 + - do: + indices.get_mapping: + index: test_1,test_2 - - is_true: test_1.mappings - - is_true: test_2.mappings + - is_true: test_1.mappings + - is_true: test_2.mappings --- "Get /index*/_mapping/": - - do: - indices.get_mapping: - index: '*2' + - do: + indices.get_mapping: + index: '*2' - - is_true: test_2.mappings - - is_false: test_1 + - is_true: test_2.mappings + - is_false: test_1 diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_mapping/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_mapping/10_basic.yml index 75d282d524607..335e0b4783bf0 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_mapping/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_mapping/10_basic.yml @@ -145,41 +145,19 @@ - is_false: test_index.mappings.properties.foo.meta.bar - match: { test_index.mappings.properties.foo.meta.baz: "quux" } ---- -"disabling synthetic source fails": - - requires: - cluster_features: ["gte_v8.4.0"] - reason: "Added in 8.4.0" - - - do: - indices.create: - index: test_index - body: - mappings: - _source: - mode: synthetic - - - do: - catch: /Cannot update parameter \[mode\] from \[synthetic\] to \[stored\]/ - indices.put_mapping: - index: test_index - body: - _source: - mode: stored - --- "enabling synthetic source from explicit succeeds": - requires: - cluster_features: [ "gte_v8.4.0" ] - reason: "Added in 8.4.0" + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" - do: indices.create: index: test_index body: - mappings: - _source: - mode: stored + settings: + index: + mapping.source.mode: stored - do: indices.put_mapping: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.recovery/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.recovery/20_synthetic_source.yml new file mode 100644 index 0000000000000..493b834fc5a90 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.recovery/20_synthetic_source.yml @@ -0,0 +1,33 @@ +--- +test recovery empty index with use_synthetic_source: + - requires: + cluster_features: ["mapper.synthetic_recovery_source"] + reason: requires synthetic recovery source + + - do: + indices.create: + index: test + body: + settings: + index: + number_of_replicas: 0 + recovery: + use_synthetic_source: true + mapping: + source: + mode: synthetic + + - do: + indices.get_settings: {} + - match: { test.settings.index.mapping.source.mode: synthetic} + - is_true: test.settings.index.recovery.use_synthetic_source + + - do: + indices.put_settings: + index: test + body: + index.number_of_replicas: 1 + + - do: + cluster.health: + wait_for_events: languid diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.resolve_cluster/20_resolve_system_index.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.resolve_cluster/20_resolve_system_index.yml index 0773d58758486..29e16dc084fa7 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.resolve_cluster/20_resolve_system_index.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.resolve_cluster/20_resolve_system_index.yml @@ -3,9 +3,10 @@ setup: - requires: cluster_features: ["gte_v8.13.0"] reason: "resolve cluster introduced in 8.13" - test_runner_features: "warnings" + test_runner_features: ["warnings", "headers"] - do: + headers: { X-elastic-product-origin: elastic } indices.create: index: ".tasks" --- diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.sort/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.sort/10_basic.yml index 68508ec9ad45b..625ccc84e4261 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.sort/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.sort/10_basic.yml @@ -101,7 +101,7 @@ index: test body: sort: ["rank"] - query: {"range": { "rank": { "from": 0 } } } + query: {"range": { "rank": { "gte": 0 } } } track_total_hits: false size: 1 @@ -142,7 +142,7 @@ index: test body: sort: ["rank"] - query: {"range": { "rank": { "from": 0 } } } + query: {"range": { "rank": { "gte": 0 } } } track_total_hits: false size: 3 @@ -160,6 +160,6 @@ scroll: 1m body: sort: ["rank"] - query: {"range": { "rank": { "from": 0 } } } + query: {"range": { "rank": { "gte": 0 } } } track_total_hits: false size: 3 diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml index 07cb154449a70..1d0c42434c058 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml @@ -77,11 +77,6 @@ create logs index: - is_true: test - match: { test.settings.index.mode: "logsdb" } - - do: - indices.get_mapping: - index: test - - match: { test.mappings._source.mode: synthetic } - --- using default timestamp field mapping: - requires: @@ -519,6 +514,48 @@ routing path not allowed in logs mode: - match: { error.type: "illegal_argument_exception" } - match: { error.reason: "[index.routing_path] requires [index.mode=time_series]" } +--- +routing path allowed in logs mode with routing on sort fields: + - requires: + cluster_features: [ "routing.logsb_route_on_sort_fields" ] + reason: introduction of route on index sorting fields + + - do: + indices.create: + index: test + body: + settings: + index: + mode: logsdb + number_of_replicas: 0 + number_of_shards: 2 + routing_path: [ host.name, agent_id ] + logsdb: + route_on_sort_fields: true + mappings: + properties: + "@timestamp": + type: date + host.name: + type: keyword + agent_id: + type: keyword + process_id: + type: integer + http_method: + type: keyword + message: + type: text + + - do: + indices.get_settings: + index: test + + - is_true: test + - match: { test.settings.index.mode: logsdb } + - match: { test.settings.index.logsdb.route_on_sort_fields: "true" } + - match: { test.settings.index.routing_path: [ host.name, agent_id ] } + --- start time not allowed in logs mode: - requires: @@ -600,3 +637,287 @@ end time not allowed in logs mode: - match: { error.root_cause.0.type: "illegal_argument_exception" } - match: { error.type: "illegal_argument_exception" } - match: { error.reason: "[index.time_series.end_time] requires [index.mode=time_series]" } + +--- +ignore dynamic beyond limit logsdb default value: + - requires: + cluster_features: [ "mapper.logsdb_default_ignore_dynamic_beyond_limit" ] + reason: requires logsdb default value for `index.mapping.total_fields.ignore_dynamic_beyond_limit` + + - do: + indices.create: + index: test-ignore-dynamic-default + body: + settings: + index: + mode: logsdb + + - do: + indices.get_settings: + index: test-ignore-dynamic-default + include_defaults: true + + - match: { test-ignore-dynamic-default.settings.index.mode: "logsdb" } + - match: { test-ignore-dynamic-default.defaults.index.mapping.total_fields.limit: "1000" } + - match: { test-ignore-dynamic-default.defaults.index.mapping.total_fields.ignore_dynamic_beyond_limit: "true" } + +--- +ignore dynamic beyond limit logsdb override value: + - requires: + cluster_features: [ "mapper.logsdb_default_ignore_dynamic_beyond_limit" ] + reason: requires logsdb default value for `index.mapping.total_fields.ignore_dynamic_beyond_limit` + + - do: + indices.create: + index: test-ignore-dynamic-override + body: + settings: + index: + mode: logsdb + mapping: + total_fields: + ignore_dynamic_beyond_limit: false + + - do: + indices.get_settings: + index: test-ignore-dynamic-override + + - match: { test-ignore-dynamic-override.settings.index.mode: "logsdb" } + - match: { test-ignore-dynamic-override.settings.index.mapping.total_fields.ignore_dynamic_beyond_limit: "false" } + +--- +logsdb with default ignore dynamic beyond limit and default sorting: + - requires: + cluster_features: ["mapper.logsdb_default_ignore_dynamic_beyond_limit"] + reason: requires default value for ignore_dynamic_beyond_limit + + - do: + indices.create: + index: test-logsdb-default-sort + body: + settings: + index: + mode: logsdb + mapping: + # NOTE: When the index mode is set to `logsdb`, the `host.name` field is automatically injected if + # sort settings are not overridden. + # With `subobjects` set to `true` (default), this creates a `host` object field and a nested `name` + # keyword field (`host.name`). + # + # As a result, there are always at least 4 statically mapped fields (`@timestamp`, `host`, `host.name` + # and `name`). We cannot use a field limit lower than 4 because these fields are always present. + # + # Indeed, if `index.mapping.total_fields.ignore_dynamic_beyond_limit` is `true`, any dynamically + # mapped fields beyond the limit `index.mapping.total_fields.limit` are ignored, but the statically + # mapped fields are always counted. + total_fields: + limit: 4 + mappings: + properties: + "@timestamp": + type: date + name: + type: keyword + + - do: + indices.get_settings: + index: test-logsdb-default-sort + + - match: { test-logsdb-default-sort.settings.index.mode: "logsdb" } + + - do: + bulk: + index: test-logsdb-default-sort + refresh: true + body: + - '{ "index": { } }' + - '{ "@timestamp": "2024-08-13T12:30:00Z", "name": "foo", "host.name": "92f4a67c", "value": 10, "message": "the quick brown fox", "region": "us-west", "pid": 153462 }' + - '{ "index": { } }' + - '{ "@timestamp": "2024-08-13T12:01:00Z", "name": "bar", "host.name": "24eea278", "value": 20, "message": "jumps over the lazy dog", "region": "us-central", "pid": 674972 }' + - match: { errors: false } + + - do: + search: + index: test-logsdb-default-sort + body: + query: + match_all: {} + sort: "@timestamp" + + - match: { hits.total.value: 2 } + - match: { hits.hits.0._source.name: "bar" } + - match: { hits.hits.0._source.value: 20 } + - match: { hits.hits.0._source.message: "jumps over the lazy dog" } + - match: { hits.hits.0._ignored: [ "message", "pid", "region", "value" ] } + - match: { hits.hits.1._source.name: "foo" } + - match: { hits.hits.1._source.value: 10 } + - match: { hits.hits.1._source.message: "the quick brown fox" } + - match: { hits.hits.1._ignored: [ "message", "pid", "region", "value" ] } + +--- +logsdb with default ignore dynamic beyond limit and non-default sorting: + - requires: + cluster_features: ["mapper.logsdb_default_ignore_dynamic_beyond_limit"] + reason: requires default value for ignore_dynamic_beyond_limit + + - do: + indices.create: + index: test-logsdb-non-default-sort + body: + settings: + index: + sort.field: [ "name" ] + sort.order: [ "desc" ] + mode: logsdb + mapping: + # NOTE: Here sort settings are overridden and we do not have any additional statically mapped field other + # than `name` and `timestamp`. As a result, there are only 2 statically mapped fields. + total_fields: + limit: 2 + mappings: + properties: + "@timestamp": + type: date + name: + type: keyword + + - do: + indices.get_settings: + index: test-logsdb-non-default-sort + + - match: { test-logsdb-non-default-sort.settings.index.mode: "logsdb" } + + - do: + bulk: + index: test-logsdb-non-default-sort + refresh: true + body: + - '{ "index": { } }' + - '{ "@timestamp": "2024-08-13T12:30:00Z", "name": "foo", "host.name": "92f4a67c", "value": 10, "message": "the quick brown fox", "region": "us-west", "pid": 153462 }' + - '{ "index": { } }' + - '{ "@timestamp": "2024-08-13T12:01:00Z", "name": "bar", "host.name": "24eea278", "value": 20, "message": "jumps over the lazy dog", "region": "us-central", "pid": 674972 }' + - match: { errors: false } + + - do: + search: + index: test-logsdb-non-default-sort + body: + query: + match_all: {} + sort: "@timestamp" + + - match: { hits.total.value: 2 } + - match: { hits.hits.0._source.name: "bar" } + - match: { hits.hits.0._source.value: 20 } + - match: { hits.hits.0._source.message: "jumps over the lazy dog" } + - match: { hits.hits.0._ignored: [ "host", "message", "pid", "region", "value" ] } + - match: { hits.hits.1._source.name: "foo" } + - match: { hits.hits.1._source.value: 10 } + - match: { hits.hits.1._source.message: "the quick brown fox" } + - match: { hits.hits.1._ignored: [ "host", "message", "pid", "region", "value" ] } + +--- +logsdb with default ignore dynamic beyond limit and too low limit: + - requires: + cluster_features: ["mapper.logsdb_default_ignore_dynamic_beyond_limit"] + reason: requires default value for ignore_dynamic_beyond_limit + + - do: + catch: bad_request + indices.create: + index: test-logsdb-low-limit + body: + settings: + index: + mode: logsdb + mapping: + # NOTE: When the index mode is set to `logsdb`, the `host.name` field is automatically injected if + # sort settings are not overridden. + # With `subobjects` set to `true` (default), this creates a `host` object field and a nested `name` + # keyword field (`host.name`). + # + # As a result, there are always at least 4 statically mapped fields (`@timestamp`, `host`, `host.name` + # and `name`). We cannot use a field limit lower than 4 because these fields are always present. + # + # Indeed, if `index.mapping.total_fields.ignore_dynamic_beyond_limit` is `true`, any dynamically + # mapped fields beyond the limit `index.mapping.total_fields.limit` are ignored, but the statically + # mapped fields are always counted. + total_fields: + limit: 3 + mappings: + properties: + "@timestamp": + type: date + name: + type: keyword + - match: { error.type: "illegal_argument_exception" } + - match: { error.reason: "Limit of total fields [3] has been exceeded" } + +--- +logsdb with default ignore dynamic beyond limit and subobjects false: + - requires: + cluster_features: ["mapper.logsdb_default_ignore_dynamic_beyond_limit"] + reason: requires default value for ignore_dynamic_beyond_limit + + - do: + indices.create: + index: test-logsdb-subobjects-false + body: + settings: + index: + mode: logsdb + mapping: + # NOTE: When the index mode is set to `logsdb`, the `host.name` field is automatically injected if + # sort settings are not overridden. + # With `subobjects` set to `false` anyway, a single `host.name` keyword field is automatically mapped. + # + # As a result, there are just 3 statically mapped fields (`@timestamp`, `host.name` and `name`). + # We cannot use a field limit lower than 3 because these fields are always present. + # + # Indeed, if `index.mapping.total_fields.ignore_dynamic_beyond_limit` is `true`, any dynamically + # mapped fields beyond the limit `index.mapping.total_fields.limit` are ignored, but the statically + # mapped fields are always counted. + total_fields: + limit: 3 + mappings: + subobjects: false + properties: + "@timestamp": + type: date + name: + type: keyword + + - do: + indices.get_settings: + index: test-logsdb-subobjects-false + + - match: { test-logsdb-subobjects-false.settings.index.mode: "logsdb" } + + - do: + bulk: + index: test-logsdb-subobjects-false + refresh: true + body: + - '{ "index": { } }' + - '{ "@timestamp": "2024-08-13T12:30:00Z", "name": "foo", "host.name": "92f4a67c", "value": 10, "message": "the quick brown fox", "region": "us-west", "pid": 153462 }' + - '{ "index": { } }' + - '{ "@timestamp": "2024-08-13T12:01:00Z", "name": "bar", "host.name": "24eea278", "value": 20, "message": "jumps over the lazy dog", "region": "us-central", "pid": 674972 }' + - match: { errors: false } + + - do: + search: + index: test-logsdb-subobjects-false + body: + query: + match_all: {} + sort: "@timestamp" + + - match: { hits.total.value: 2 } + - match: { hits.hits.0._source.name: "bar" } + - match: { hits.hits.0._source.value: 20 } + - match: { hits.hits.0._source.message: "jumps over the lazy dog" } + - match: { hits.hits.0._ignored: [ "message", "pid", "region", "value" ] } + - match: { hits.hits.1._source.name: "foo" } + - match: { hits.hits.1._source.value: 10 } + - match: { hits.hits.1._source.message: "the quick brown fox" } + - match: { hits.hits.1._ignored: [ "message", "pid", "region", "value" ] } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml index d209c839d904b..06a007b8aaca5 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml @@ -1,56 +1,42 @@ ---- -stored _source mode is not supported: +setup: - requires: - test_runner_features: [capabilities] - capabilities: - - method: PUT - path: /{index} - capabilities: [logsdb_index_mode] - reason: "Support for 'logsdb' index mode capability required" + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" - - skip: - known_issues: - - cluster_feature: "gte_v8.15.0" - fixed_by: "gte_v8.16.0" - reason: "Development of logs index mode spans 8.15 and 8.16" +--- +synthetic _source is default: + - do: + indices.create: + index: test-default-source + body: + settings: + index: + mode: logsdb + - do: + indices.get_settings: + index: test-default-source + - match: { test-default-source.settings.index.mode: logsdb } + - match: { test-default-source.settings.index.mapping.source.mode: null } +--- +stored _source mode is supported: - do: - catch: bad_request indices.create: index: test-stored-source body: settings: index: mode: logsdb - mappings: - _source: - mode: stored - properties: - "@timestamp": - type: date - host.name: - type: keyword + mapping.source.mode: stored - - match: { error.type: "mapper_parsing_exception" } - - match: { error.root_cause.0.type: "mapper_parsing_exception" } - - match: { error.reason: "Failed to parse mapping: Indices with with index mode [logsdb] only support synthetic source" } + - do: + indices.get_settings: + index: test-stored-source + - match: { test-stored-source.settings.index.mode: logsdb } + - match: { test-stored-source.settings.index.mapping.source.mode: stored } --- disabled _source is not supported: - - requires: - test_runner_features: [capabilities] - capabilities: - - method: PUT - path: /{index} - capabilities: [logsdb_index_mode] - reason: "Support for 'logsdb' index mode capability required" - - - skip: - known_issues: - - cluster_feature: "gte_v8.15.0" - fixed_by: "gte_v8.16.0" - reason: "Development of logs index mode spans 8.15 and 8.16" - - do: catch: bad_request indices.create: @@ -62,33 +48,85 @@ disabled _source is not supported: mappings: _source: enabled: false - properties: - "@timestamp": - type: date - host.name: - type: keyword - match: { error.type: "mapper_parsing_exception" } - match: { error.root_cause.0.type: "mapper_parsing_exception" } - - match: { error.reason: "Failed to parse mapping: Indices with with index mode [logsdb] only support synthetic source" } + - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode" } - do: catch: bad_request indices.create: - index: test-disabled-source + index: test-disabled-mode-source body: settings: index: mode: logsdb - mappings: - _source: - mode: disabled - properties: - "@timestamp": - type: date - host.name: - type: keyword + mapping.source.mode: disabled - match: { error.type: "mapper_parsing_exception" } - match: { error.root_cause.0.type: "mapper_parsing_exception" } - - match: { error.reason: "Failed to parse mapping: Indices with with index mode [logsdb] only support synthetic source" } + - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode" } + +--- +include/exclude is not supported with synthetic _source: + - do: + catch: '/filtering the stored _source is incompatible with synthetic source/' + indices.create: + index: test-includes + body: + settings: + index: + mode: logsdb + mappings: + _source: + includes: [a] + + - do: + catch: '/filtering the stored _source is incompatible with synthetic source/' + indices.create: + index: test-excludes + body: + settings: + index: + mode: logsdb + mappings: + _source: + excludes: [b] + +--- +include/exclude is supported with stored _source: + - do: + indices.create: + index: test-includes + body: + settings: + index: + mode: logsdb + mapping.source.mode: stored + mappings: + _source: + includes: [a] + + - do: + indices.get: + index: test-includes + + - match: { test-includes.mappings._source.includes: ["a"] } + + - do: + indices.create: + index: test-excludes + body: + settings: + index: + mode: logsdb + mapping.source.mode: stored + mappings: + _source: + excludes: [b] + + - do: + indices.get: + index: test-excludes + + - match: { test-excludes.mappings._source.excludes: ["b"] } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml index 2f3d2fa2f974d..084f104932d99 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml @@ -1,3 +1,9 @@ +setup: + - requires: + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" + +--- keyword: - requires: cluster_features: ["gte_v8.4.0"] @@ -7,9 +13,10 @@ keyword: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: kwd: type: keyword @@ -62,9 +69,9 @@ keyword with normalizer: type: custom filter: - lowercase + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: keyword: type: keyword @@ -144,9 +151,10 @@ stored text: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: text: type: text @@ -193,9 +201,10 @@ force_synthetic_source_ok: indices.create: index: test body: + settings: + index: + mapping.source.mode: stored mappings: - _source: - mode: stored properties: obj: properties: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/10_basic.yml index c92d75cf2f5cb..3432a1e34c018 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/10_basic.yml @@ -446,6 +446,77 @@ setup: - match: { hits.total: 2 } +--- +"Date range edge cases": + - requires: + cluster_features: ["mapper.range.date_range_indexing_fix"] + reason: "tests rounding fixes in 8.16.0 that previously caused non-intuitive indexing and query because ranges were assumed to always index with 0's as the default such as when time is missing 00:00:00.000 time was assumed but for lte indexing and query missing time should be 23:59:59.999 as per docs here: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-range-query.html" + + - do: + indices.create: + index: test_date_range_edge_cases + body: + mappings: + "properties": + "date_range": + "type": "date_range" + "format": "yyyy-MM-dd" + + - do: + index: + index: test_date_range_edge_cases + id: "1" + body: { "date_range": { "gte": "1980-12-14", "lte": "1980-12-17" } } + + - do: + index: + index: test_date_range_edge_cases + id: "2" + body: { "date_range": { "gt": "1990-12-15", "lt": "1990-12-18" } } + + - do: + index: + index: test_date_range_edge_cases + id: "3" + body: { "date_range": { "gte": "1985-12-16||/M", "lte": "1986-02-10||/M" } } + + - do: + index: + index: test_date_range_edge_cases + id: "4" + body: { "date_range": { "gt": "1995-12-16||/M", "lt": "1996-02-10||/M" } } + + - do: + indices.refresh: { } + + - do: + search: + rest_total_hits_as_int: true + body: { "size": 0, "query": { "range": { "date_range": { "gte": "1980-12-14", "lte": "1980-12-17", "relation": "contains" } } } } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + body: { "size": 0, "query": { "range": { "date_range": { "gt": "1990-12-15", "lt": "1990-12-18", "relation": "contains" } } } } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + body: { "size": 0, "query": { "range": { "date_range": { "gte": "1985-12-16||/M", "lte": "1986-02-10||/M", "relation": "contains" } } } } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + body: { "size": 0, "query": { "range": { "date_range": { "gt": "1995-12-16||/M", "lt": "1996-02-10||/M", "relation": "contains" } } } } + + - match: { hits.total: 1 } + --- "Null bounds": diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/20_synthetic_source.yml index 07bd372b60058..de20f82f8ba2f 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/20_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/20_synthetic_source.yml @@ -526,49 +526,52 @@ setup: ip_range: { "gte": "2001:db8::", "lte": null } --- -"Date range": +"Date range Rounding Fixes": + - requires: + cluster_features: ["mapper.range.date_range_indexing_fix"] + reason: "tests rounding fixes in 8.16.0 that previously caused non-intuitive indexing and query because ranges were assumed to always index with 0's as the default such as when time is missing 00:00:00.000 time was assumed but for lte indexing and query missing time should be 23:59:59.999 as per docs here: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-range-query.html" - do: index: index: synthetic_source_test id: "1" - body: { "date_range" : { "gte": "2017-09-01", "lte": "2017-09-05" } } + body: { "date_range": { "gte": "2017-09-01", "lte": "2017-09-05" } } - do: index: index: synthetic_source_test id: "2" - body: { "date_range" : { "gt": "2017-09-01", "lte": "2017-09-03" } } + body: { "date_range": { "gt": "2017-09-01", "lte": "2017-09-03" } } - do: index: index: synthetic_source_test id: "3" - body: { "date_range" : [ { "gte": "2017-09-04", "lt": "2017-09-05" } ] } + body: { "date_range": [ { "gte": "2017-09-04", "lt": "2017-09-05" } ] } - do: index: index: synthetic_source_test id: "4" - body: { "date_range" : [ { "gt": "2017-09-04", "lt": "2017-09-08" }, { "gt": "2017-09-04", "lt": "2017-09-07" } ] } + body: { "date_range": [ { "gt": "2017-09-04", "lt": "2017-09-08" }, { "gt": "2017-09-04", "lt": "2017-09-07" } ] } - do: index: index: synthetic_source_test id: "5" - body: { "date_range" : { "gte": 1504224000000, "lte": 1504569600000 } } + body: { "date_range": { "gte": 1504224000000, "lte": 1504569600000 } } - do: index: index: synthetic_source_test id: "6" - body: { "date_range" : { "gte": "2017-09-01T10:20:30.123Z", "lte": "2017-09-05T03:04:05.789Z" } } + body: { "date_range": { "gte": "2017-09-01T10:20:30.123Z", "lte": "2017-09-05T03:04:05.789Z" } } - do: index: index: synthetic_source_test id: "7" - body: { "date_range" : null } + body: { "date_range": null } - do: index: @@ -583,7 +586,7 @@ setup: body: { "date_range": { "gte": "2017-09-05" } } - do: - indices.refresh: {} + indices.refresh: { } - do: get: @@ -591,7 +594,7 @@ setup: id: "1" - match: _source: - date_range: { "gte": "2017-09-01T00:00:00.000Z", "lte": "2017-09-05T00:00:00.000Z" } + date_range: { "gte": "2017-09-01T00:00:00.000Z", "lte": "2017-09-05T23:59:59.999Z" } - do: get: @@ -599,7 +602,7 @@ setup: id: "2" - match: _source: - date_range: { "gte": "2017-09-01T00:00:00.001Z", "lte": "2017-09-03T00:00:00.000Z" } + date_range: { "gte": "2017-09-02T00:00:00.000Z", "lte": "2017-09-03T23:59:59.999Z" } - do: get: @@ -615,7 +618,7 @@ setup: id: "4" - match: _source: - date_range: [ { "gte": "2017-09-04T00:00:00.001Z", "lte": "2017-09-06T23:59:59.999Z" }, { "gte": "2017-09-04T00:00:00.001Z", "lte": "2017-09-07T23:59:59.999Z" } ] + date_range: [ { "gte": "2017-09-05T00:00:00.000Z", "lte": "2017-09-06T23:59:59.999Z" }, { "gte": "2017-09-05T00:00:00.000Z", "lte": "2017-09-07T23:59:59.999Z" } ] - do: get: @@ -638,7 +641,7 @@ setup: index: synthetic_source_test id: "7" - match: - _source: {} + _source: { } - do: get: @@ -646,7 +649,7 @@ setup: id: "8" - match: _source: - date_range: { "gte": null, "lte": "2017-09-05T00:00:00.000Z" } + date_range: { "gte": null, "lte": "2017-09-05T23:59:59.999Z" } - do: get: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/50_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/50_synthetic_source.yml index a2fd448f5044d..657eeb759c6c6 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/50_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/50_synthetic_source.yml @@ -1,7 +1,7 @@ setup: - requires: - cluster_features: ["gte_v8.4.0"] - reason: introduced in 8.4.0 + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" - do: indices.create: @@ -9,9 +9,9 @@ setup: body: settings: number_of_shards: 1 + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: foo: type: keyword @@ -21,6 +21,7 @@ setup: index_options: positions vectors: type: text + store: false term_vector: with_positions_offsets positions: type: text diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.retrievers/20_knn_retriever.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.retrievers/20_knn_retriever.yml index d08a8e2a6d39c..e49f0634a4887 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.retrievers/20_knn_retriever.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.retrievers/20_knn_retriever.yml @@ -18,7 +18,7 @@ setup: dims: 5 index: true index_options: - type: hnsw + type: int8_hnsw similarity: l2_norm - do: @@ -73,3 +73,59 @@ setup: - match: {hits.total.value: 1} - match: {hits.hits.0._id: "3"} - match: {hits.hits.0.fields.name.0: "rabbit.jpg"} + +--- +"Vector rescoring has no effect for non-quantized vectors and provides same results as non-rescored knn": + - requires: + reason: 'Quantized vector rescoring is required' + test_runner_features: [capabilities] + capabilities: + - method: GET + path: /_search + capabilities: [knn_quantized_vector_rescore] + - skip: + features: "headers" + + # Rescore + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: index1 + body: + knn: + field: vector + query_vector: [2, 2, 2, 2, 3] + k: 3 + num_candidates: 3 + rescore_vector: + num_candidates_factor: 1.5 + + # Get rescoring scores - hit ordering may change depending on how things are distributed + - match: { hits.total: 3 } + - set: { hits.hits.0._score: rescore_score0 } + - set: { hits.hits.1._score: rescore_score1 } + - set: { hits.hits.2._score: rescore_score2 } + + # Exact knn via script score + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: index1 + body: + query: + script_score: + query: {match_all: {} } + script: + source: "1.0 / (1.0 + Math.pow(l2norm(params.query_vector, 'vector'), 2.0))" + params: + query_vector: [2, 2, 2, 2, 3] + + # Compare scores as hit IDs may change depending on how things are distributed + - match: { hits.total: 3 } + - match: { hits.hits.0._score: $rescore_score0 } + - match: { hits.hits.1._score: $rescore_score1 } + - match: { hits.hits.2._score: $rescore_score2 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.retrievers/30_rescorer_retriever.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.retrievers/30_rescorer_retriever.yml new file mode 100644 index 0000000000000..2c16de61c6b15 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.retrievers/30_rescorer_retriever.yml @@ -0,0 +1,225 @@ +setup: + - requires: + cluster_features: [ "search.retriever.rescorer.enabled" ] + reason: "Support for rescorer retriever" + + - do: + indices.create: + index: test + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + available: + type: boolean + features: + type: rank_features + + - do: + bulk: + refresh: true + index: test + body: + - '{"index": {"_id": 1 }}' + - '{"features": { "first_stage": 1, "second_stage": 10}, "available": true, "group": 1}' + - '{"index": {"_id": 2 }}' + - '{"features": { "first_stage": 2, "second_stage": 9}, "available": false, "group": 1}' + - '{"index": {"_id": 3 }}' + - '{"features": { "first_stage": 3, "second_stage": 8}, "available": false, "group": 3}' + - '{"index": {"_id": 4 }}' + - '{"features": { "first_stage": 4, "second_stage": 7}, "available": true, "group": 1}' + - '{"index": {"_id": 5 }}' + - '{"features": { "first_stage": 5, "second_stage": 6}, "available": true, "group": 3}' + - '{"index": {"_id": 6 }}' + - '{"features": { "first_stage": 6, "second_stage": 5}, "available": false, "group": 2}' + - '{"index": {"_id": 7 }}' + - '{"features": { "first_stage": 7, "second_stage": 4}, "available": true, "group": 3}' + - '{"index": {"_id": 8 }}' + - '{"features": { "first_stage": 8, "second_stage": 3}, "available": true, "group": 1}' + - '{"index": {"_id": 9 }}' + - '{"features": { "first_stage": 9, "second_stage": 2}, "available": true, "group": 2}' + - '{"index": {"_id": 10 }}' + - '{"features": { "first_stage": 10, "second_stage": 1}, "available": false, "group": 1}' + +--- +"Rescorer retriever basic": + - do: + search: + index: test + body: + retriever: + rescorer: + rescore: + window_size: 10 + query: + rescore_query: + rank_feature: + field: "features.second_stage" + linear: { } + query_weight: 0 + retriever: + standard: + query: + rank_feature: + field: "features.first_stage" + linear: { } + size: 2 + + - match: { hits.total.value: 10 } + - match: { hits.hits.0._id: "1" } + - match: { hits.hits.0._score: 10.0 } + - match: { hits.hits.1._id: "2" } + - match: { hits.hits.1._score: 9.0 } + + - do: + search: + index: test + body: + retriever: + rescorer: + rescore: + window_size: 3 + query: + rescore_query: + rank_feature: + field: "features.second_stage" + linear: {} + query_weight: 0 + retriever: + standard: + query: + rank_feature: + field: "features.first_stage" + linear: {} + size: 2 + + - match: {hits.total.value: 10} + - match: {hits.hits.0._id: "8"} + - match: { hits.hits.0._score: 3.0 } + - match: {hits.hits.1._id: "9"} + - match: { hits.hits.1._score: 2.0 } + +--- +"Rescorer retriever with pre-filters": + - do: + search: + index: test + body: + retriever: + rescorer: + filter: + match: + available: true + rescore: + window_size: 10 + query: + rescore_query: + rank_feature: + field: "features.second_stage" + linear: { } + query_weight: 0 + retriever: + standard: + query: + rank_feature: + field: "features.first_stage" + linear: { } + size: 2 + + - match: { hits.total.value: 6 } + - match: { hits.hits.0._id: "1" } + - match: { hits.hits.0._score: 10.0 } + - match: { hits.hits.1._id: "4" } + - match: { hits.hits.1._score: 7.0 } + + - do: + search: + index: test + body: + retriever: + rescorer: + rescore: + window_size: 4 + query: + rescore_query: + rank_feature: + field: "features.second_stage" + linear: { } + query_weight: 0 + retriever: + standard: + filter: + match: + available: true + query: + rank_feature: + field: "features.first_stage" + linear: { } + size: 2 + + - match: { hits.total.value: 6 } + - match: { hits.hits.0._id: "5" } + - match: { hits.hits.0._score: 6.0 } + - match: { hits.hits.1._id: "7" } + - match: { hits.hits.1._score: 4.0 } + +--- +"Rescorer retriever and collapsing": + - do: + search: + index: test + body: + retriever: + rescorer: + rescore: + window_size: 10 + query: + rescore_query: + rank_feature: + field: "features.second_stage" + linear: { } + query_weight: 0 + retriever: + standard: + query: + rank_feature: + field: "features.first_stage" + linear: { } + collapse: + field: group + size: 3 + + - match: { hits.total.value: 10 } + - match: { hits.hits.0._id: "1" } + - match: { hits.hits.0._score: 10.0 } + - match: { hits.hits.1._id: "3" } + - match: { hits.hits.1._score: 8.0 } + - match: { hits.hits.2._id: "6" } + - match: { hits.hits.2._score: 5.0 } + +--- +"Rescorer retriever and invalid window size": + - do: + catch: "/\\[rescorer\\] requires \\[window_size: 5\\] be greater than or equal to \\[size: 10\\]/" + search: + index: test + body: + retriever: + rescorer: + rescore: + window_size: 5 + query: + rescore_query: + rank_feature: + field: "features.second_stage" + linear: { } + query_weight: 0 + retriever: + standard: + query: + rank_feature: + field: "features.first_stage" + linear: { } + size: 10 diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/110_knn_query_with_filter.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/110_knn_query_with_filter.yml index 618951711cffd..3d4841a16d82d 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/110_knn_query_with_filter.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/110_knn_query_with_filter.yml @@ -59,7 +59,9 @@ setup: --- "Simple knn query": - + - requires: + cluster_features: "search.vectors.k_param_supported" + reason: 'k param for knn as query is required' - do: search: index: my_index @@ -71,8 +73,9 @@ setup: field: my_vector query_vector: [1, 1, 1, 1] num_candidates: 5 + k: 5 - - match: { hits.total.value: 5 } # collector sees num_candidates docs + - match: { hits.total.value: 5 } - length: {hits.hits: 3} - match: { hits.hits.0._id: "1" } - match: { hits.hits.0.fields.my_name.0: v1 } @@ -93,8 +96,9 @@ setup: field: my_vector query_vector: [1, 1, 1, 1] num_candidates: 5 + k: 5 - - match: { hits.total.value: 5 } # collector sees num_candidates docs + - match: { hits.total.value: 5 } - length: {hits.hits: 3} - match: { hits.hits.0._id: "2" } - match: { hits.hits.0.fields.my_name.0: v2 } @@ -140,6 +144,7 @@ setup: field: my_vector query_vector: [1, 1, 1, 1] num_candidates: 5 + k: 5 - match: { hits.total.value: 5 } - length: { hits.hits: 3 } @@ -184,6 +189,7 @@ setup: field: my_vector query_vector: [1, 1, 1, 1] num_candidates: 100 + k: 100 - match: { hits.total.value: 10 } # 5 docs from each alias - length: {hits.hits: 6} @@ -213,6 +219,7 @@ setup: field: my_vector query_vector: [1, 1, 1, 1] num_candidates: 5 + k: 5 filter: term: my_name: v2 @@ -243,9 +250,10 @@ setup: field: my_vector query_vector: [1, 1, 1, 1] num_candidates: 5 + k: 5 - match: { hits.total.value: 2 } - - length: {hits.hits: 2} # knn query returns top 5 docs, but they are post-filtered to 2 docs + - length: {hits.hits: 2} # knn query returns top 3 docs, but they are post-filtered to 2 docs - match: { hits.hits.0._id: "2" } - match: { hits.hits.0.fields.my_name.0: v2 } - match: { hits.hits.1._id: "4" } @@ -271,4 +279,4 @@ setup: my_name: v1 - match: { hits.total.value: 0} - - length: { hits.hits: 0 } # knn query returns top 5 docs, but they are post-filtered to 0 docs + - length: { hits.hits: 0 } # knn query returns top 3 docs, but they are post-filtered to 0 docs diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/120_knn_query_multiple_shards.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/120_knn_query_multiple_shards.yml index c6f3e187f7953..c68565e6629f5 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/120_knn_query_multiple_shards.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/120_knn_query_multiple_shards.yml @@ -166,55 +166,3 @@ setup: - close_to: { hits.hits.2._score: { value: 120, error: 0.00001 } } - close_to: { hits.hits.2.matched_queries.bm25_query: { value: 100.0, error: 0.00001 } } - close_to: { hits.hits.2.matched_queries.knn_query: { value: 20.0, error: 0.00001 } } - ---- -"Aggregations with collected number of docs depends on num_candidates": - - do: - search: - index: my_index - body: - size: 2 - query: - knn: - field: my_vector - query_vector: [1, 1, 1, 1] - num_candidates: 100 # collect up to 100 candidates from each shard - aggs: - my_agg: - terms: - field: my_name - order: - _key: asc - - - length: {hits.hits: 2} - - match: {hits.total.value: 12} - - match: {aggregations.my_agg.buckets.0.key: 'v1'} - - match: {aggregations.my_agg.buckets.1.key: 'v2'} - - match: {aggregations.my_agg.buckets.0.doc_count: 6} - - match: {aggregations.my_agg.buckets.1.doc_count: 6} - - - do: - search: - index: my_index - body: - size: 2 - query: - knn: - field: my_vector - query_vector: [ 1, 1, 1, 1 ] - num_candidates: 3 # collect 3 candidates from each shard - aggs: - my_agg2: - terms: - field: my_name - order: - _key: asc - my_sum_buckets: - sum_bucket: - buckets_path: "my_agg2>_count" - - - length: { hits.hits: 2 } - - match: { hits.total.value: 6 } - - match: { aggregations.my_agg2.buckets.0.key: 'v1' } - - match: { aggregations.my_agg2.buckets.1.key: 'v2' } - - match: { aggregations.my_sum_buckets.value: 6.0 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/130_knn_query_nested_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/130_knn_query_nested_search.yml index 79ff3f61742f8..bf07144975650 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/130_knn_query_nested_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/130_knn_query_nested_search.yml @@ -273,6 +273,7 @@ setup: knn: field: nested.vector query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] + k: 5 num_candidates: 5 inner_hits: { size: 2, "fields": [ "nested.paragraph_id" ], _source: false } @@ -295,6 +296,7 @@ setup: knn: field: nested.vector query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] + k: 5 num_candidates: 5 inner_hits: { size: 2, "fields": [ "nested.paragraph_id" ], _source: false } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/140_knn_query_with_other_queries.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/140_knn_query_with_other_queries.yml index d52a5daf22344..1e54e497f286f 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/140_knn_query_with_other_queries.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/140_knn_query_with_other_queries.yml @@ -69,6 +69,7 @@ setup: field: my_vector query_vector: [ 1, 1, 1, 1 ] num_candidates: 5 + k: 5 functions: - filter: { match: { my_name: v1 } } weight: 10 diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/160_knn_query_missing_params.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/160_knn_query_missing_params.yml index 02962e049e267..26c52060dfb22 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/160_knn_query_missing_params.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/160_knn_query_missing_params.yml @@ -100,8 +100,9 @@ setup: knn: field: vector query_vector: [1, 1, 1] + k: 2 size: 1 - - match: { hits.total: 2 } # due to num_candidates defined as round(1.5 * size), so we only see 2 results + - match: { hits.total: 2 } # k defaults to size - length: { hits.hits: 1 } # one result is only returned though --- @@ -117,6 +118,7 @@ setup: field: vector query_vector: [-1, -1, -1] num_candidates: 1 + k: 1 size: 10 - match: { hits.total: 1 } @@ -137,9 +139,10 @@ setup: - knn: field: vector query_vector: [ 1, 1, 0] + k: 1 size: 1 - - match: { hits.total: 2 } # due to num_candidates defined as round(1.5 * size), so we only see 2 results from cat:A + - match: { hits.total: 1 } - length: { hits.hits: 1 } --- @@ -154,6 +157,7 @@ setup: - knn: field: vector query_vector: [1, 1, 0] + k: 2 - match: category: B tie_breaker: 0.8 @@ -175,6 +179,7 @@ setup: knn: field: nested.vector query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] + k: 2 inner_hits: { size: 1, "fields": [ "nested.paragraph_id" ], _source: false } size: 1 diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/180_update_dense_vector_type.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/180_update_dense_vector_type.yml index 855daeaa7f163..99943ef2671bb 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/180_update_dense_vector_type.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/180_update_dense_vector_type.yml @@ -109,6 +109,7 @@ setup: field: embedding query_vector: [1, 1, 1, 1] num_candidates: 10 + k: 10 - match: { hits.total.value: 10 } - length: {hits.hits: 3} @@ -215,6 +216,7 @@ setup: field: embedding query_vector: [ 1, 1, 1, 1 ] num_candidates: 20 + k: 20 - match: { hits.total.value: 20 } - length: { hits.hits: 3 } @@ -322,6 +324,7 @@ setup: field: embedding query_vector: [ 1, 1, 1, 1 ] num_candidates: 30 + k: 30 - match: { hits.total.value: 30 } - length: { hits.hits: 4 } @@ -430,6 +433,7 @@ setup: field: embedding query_vector: [ 1, 1, 1, 1 ] num_candidates: 40 + k: 40 - match: { hits.total.value: 40 } - length: { hits.hits: 5 } @@ -499,6 +503,7 @@ setup: field: embedding query_vector: [1, 1, 1, 1] num_candidates: 10 + k: 10 - match: { hits.total.value: 10 } - length: {hits.hits: 3} @@ -559,6 +564,7 @@ setup: field: embedding query_vector: [ 1, 1, 1, 1 ] num_candidates: 20 + k: 20 - match: { hits.total.value: 20 } - length: { hits.hits: 3 } @@ -620,6 +626,7 @@ setup: field: embedding query_vector: [ 1, 1, 1, 1 ] num_candidates: 30 + k: 30 - match: { hits.total.value: 30 } - length: { hits.hits: 4 } @@ -682,6 +689,7 @@ setup: field: embedding query_vector: [ 1, 1, 1, 1 ] num_candidates: 40 + k: 40 - match: { hits.total.value: 40 } - length: { hits.hits: 5 } @@ -751,6 +759,7 @@ setup: field: embedding query_vector: [ 1, 1, 1, 1 ] num_candidates: 10 + k: 10 - match: { hits.total.value: 10 } - length: { hits.hits: 3 } @@ -791,6 +800,7 @@ setup: field: embedding query_vector: [ 1, 1, 1, 1 ] num_candidates: 10 + k: 10 - match: { hits.total.value: 10 } - length: { hits.hits: 3 } @@ -833,6 +843,7 @@ setup: field: embedding query_vector: [ 1, 1, 1, 1 ] num_candidates: 20 + k: 20 - match: { hits.total.value: 20 } - length: { hits.hits: 3 } @@ -869,6 +880,7 @@ setup: field: embedding query_vector: [ 1, 1, 1, 1 ] num_candidates: 20 + k: 20 - match: { hits.total.value: 20 } - length: { hits.hits: 3 } @@ -911,6 +923,7 @@ setup: field: embedding query_vector: [ 1, 1, 1, 1 ] num_candidates: 30 + k: 30 - match: { hits.total.value: 30 } - length: { hits.hits: 4 } @@ -933,6 +946,7 @@ setup: knn: field: embedding query_vector: [ 1, 1, 1, 1 ] + k: 30 num_candidates: 30 - match: { hits.total.value: 30 } @@ -1769,6 +1783,7 @@ setup: field: embedding query_vector: [1, 1, 1, 1] num_candidates: 10 + k: 10 - match: { hits.total.value: 10 } - length: {hits.hits: 3} @@ -1875,6 +1890,7 @@ setup: field: embedding query_vector: [ 1, 1, 1, 1 ] num_candidates: 20 + k: 20 - match: { hits.total.value: 20 } - length: { hits.hits: 3 } @@ -1982,6 +1998,7 @@ setup: field: embedding query_vector: [ 1, 1, 1, 1 ] num_candidates: 30 + k: 30 - match: { hits.total.value: 30 } - length: { hits.hits: 4 } @@ -2090,6 +2107,7 @@ setup: field: embedding query_vector: [ 1, 1, 1, 1 ] num_candidates: 40 + k: 40 - match: { hits.total.value: 40 } - length: { hits.hits: 5 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/200_dense_vector_docvalue_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/200_dense_vector_docvalue_fields.yml new file mode 100644 index 0000000000000..161fc23a84651 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/200_dense_vector_docvalue_fields.yml @@ -0,0 +1,163 @@ +setup: + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ dense_vector_docvalue_fields ] + test_runner_features: [ capabilities, close_to ] + reason: Capability required to run test + - do: + indices.create: + index: test + body: + mappings: + properties: + name: + type: keyword + vector1: + type: dense_vector + element_type: float + dims: 5 + index: true + vector2: + type: dense_vector + element_type: float + dims: 5 + index: false + vector3: + type: dense_vector + element_type: byte + dims: 5 + index: true + vector4: + type: dense_vector + element_type: byte + dims: 5 + index: false + vector5: + type: dense_vector + element_type: bit + dims: 40 + index: true + vector6: + type: dense_vector + element_type: bit + dims: 40 + index: false + - do: + index: + index: test + id: "1" + body: + name: cow.jpg + vector1: [230.0, 300.33, -34.8988, 15.555, -200.0] + vector2: [130.0, 115.0, -1.02, 15.555, -100.0] + vector3: [-1, 100, -13, 15, -128] + vector4: [-1, 50, -1, 1, 120] + vector5: [1, 111, -13, 15, -128] + vector6: [-1, 11, 0, 12, 111] + - do: + index: + index: test + id: "2" + body: + name: moose.jpg + vector1: [-0.5, 100.0, -13, 14.8, -156.0] + vector4: [-1, 50, -1, 1, 120] + vector5: [1, 111, -13, 15, -128] + vector6: null + - do: + index: + index: test + id: "3" + body: + name: rabbit.jpg + vector2: [130.0, 115.0, -1.02, 15.555, -100.0] + vector3: [-1, 100, -13, 15, -128] + + - do: + indices.refresh: {} + +--- +"Enable docvalue_fields parameter for dense_vector fields": + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ dense_vector_docvalue_fields ] + test_runner_features: capabilities + reason: "Support for dense vector doc value fields capability required" + - do: + search: + _source: false + index: test + body: + docvalue_fields: [name, vector1, vector2, vector3, vector4, vector5, vector6] + sort: name + + + - match: {hits.hits.0._id: "1"} + - match: {hits.hits.0.fields.name.0: "cow.jpg"} + + - length: {hits.hits.0.fields.vector1.0: 5} + - length: {hits.hits.0.fields.vector2.0: 5} + - length: {hits.hits.0.fields.vector3.0: 5} + - length: {hits.hits.0.fields.vector4.0: 5} + - length: {hits.hits.0.fields.vector5.0: 5} + - length: {hits.hits.0.fields.vector6.0: 5} + + - close_to: { hits.hits.0.fields.vector1.0.0: { value: 230.0, error: 0.001 } } + - close_to: { hits.hits.0.fields.vector1.0.1: { value: 300.33, error: 0.001 } } + - close_to: { hits.hits.0.fields.vector1.0.2: { value: -34.8988, error: 0.001 } } + - close_to: { hits.hits.0.fields.vector1.0.3: { value: 15.555, error: 0.001 } } + - close_to: { hits.hits.0.fields.vector1.0.4: { value: -200.0, error: 0.001 } } + + - close_to: { hits.hits.0.fields.vector2.0.0: { value: 130.0, error: 0.001 } } + - close_to: { hits.hits.0.fields.vector2.0.1: { value: 115.0, error: 0.001 } } + - close_to: { hits.hits.0.fields.vector2.0.2: { value: -1.02, error: 0.001 } } + - close_to: { hits.hits.0.fields.vector2.0.3: { value: 15.555, error: 0.001 } } + - close_to: { hits.hits.0.fields.vector2.0.4: { value: -100.0, error: 0.001 } } + + - match: {hits.hits.0.fields.vector3.0: [-1, 100, -13, 15, -128]} + - match: {hits.hits.0.fields.vector4.0: [-1, 50, -1, 1, 120]} + - match: {hits.hits.0.fields.vector5.0: [1, 111, -13, 15, -128]} + - match: {hits.hits.0.fields.vector6.0: [-1, 11, 0, 12, 111]} + + + - match: {hits.hits.1._id: "2"} + - match: {hits.hits.1.fields.name.0: "moose.jpg"} + + - length: {hits.hits.1.fields.vector1.0: 5} + - length: {hits.hits.1.fields.vector4.0: 5} + - length: {hits.hits.1.fields.vector5.0: 5} + - match: {hits.hits.1.fields.vector2: null} + - match: {hits.hits.1.fields.vector3: null} + - match: {hits.hits.1.fields.vector6: null} + + - close_to: { hits.hits.1.fields.vector1.0.0: { value: -0.5, error: 0.001 } } + - close_to: { hits.hits.1.fields.vector1.0.1: { value: 100.0, error: 0.001 } } + - close_to: { hits.hits.1.fields.vector1.0.2: { value: -13, error: 0.001 } } + - close_to: { hits.hits.1.fields.vector1.0.3: { value: 14.8, error: 0.001 } } + - close_to: { hits.hits.1.fields.vector1.0.4: { value: -156.0, error: 0.001 } } + + - match: {hits.hits.1.fields.vector4.0: [-1, 50, -1, 1, 120]} + - match: {hits.hits.1.fields.vector5.0: [1, 111, -13, 15, -128]} + + + - match: {hits.hits.2._id: "3"} + - match: {hits.hits.2.fields.name.0: "rabbit.jpg"} + + - length: {hits.hits.2.fields.vector2.0: 5} + - length: {hits.hits.2.fields.vector3.0: 5} + - match: {hits.hits.2.fields.vector1: null} + - match: {hits.hits.2.fields.vector4: null} + - match: {hits.hits.2.fields.vector5: null} + - match: {hits.hits.2.fields.vector6: null} + + - close_to: { hits.hits.2.fields.vector2.0.0: { value: 130.0, error: 0.001 } } + - close_to: { hits.hits.2.fields.vector2.0.1: { value: 115.0, error: 0.001 } } + - close_to: { hits.hits.2.fields.vector2.0.2: { value: -1.02, error: 0.001 } } + - close_to: { hits.hits.2.fields.vector2.0.3: { value: 15.555, error: 0.001 } } + - close_to: { hits.hits.2.fields.vector2.0.4: { value: -100.0, error: 0.001 } } + + - match: {hits.hits.2.fields.vector3.0: [-1, 100, -13, 15, -128]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/210_knn_search_profile.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/210_knn_search_profile.yml new file mode 100644 index 0000000000000..d4bf5e7e9807f --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/210_knn_search_profile.yml @@ -0,0 +1,137 @@ +setup: + - requires: + reason: 'Quantized vector rescoring is required' + test_runner_features: [ capabilities ] + capabilities: + - method: GET + path: /_search + capabilities: [ knn_quantized_vector_rescore ] + - skip: + features: "headers" + + - do: + indices.create: + index: bbq_hnsw + body: + settings: + index: + number_of_shards: 1 + mappings: + properties: + vector: + type: dense_vector + dims: 64 + index: true + similarity: max_inner_product + index_options: + type: bbq_hnsw + + - do: + index: + index: bbq_hnsw + id: "1" + body: + vector: [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, + 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, + 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, + -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, + -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, + -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, + -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, + -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] + # Flush in order to provoke a merge later + - do: + indices.flush: + index: bbq_hnsw + + - do: + index: + index: bbq_hnsw + id: "2" + body: + vector: [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, + -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, + 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, + -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, + -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, + -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, + 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, + -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] + # Flush in order to provoke a merge later + - do: + indices.flush: + index: bbq_hnsw + + - do: + index: + index: bbq_hnsw + id: "3" + body: + name: rabbit.jpg + vector: [0.139, 0.178, -0.117, 0.399, 0.014, -0.139, 0.347, -0.33 , + 0.139, 0.34 , -0.052, -0.052, -0.249, 0.327, -0.288, 0.049, + 0.464, 0.338, 0.516, 0.247, -0.104, 0.259, -0.209, -0.246, + -0.11 , 0.323, 0.091, 0.442, -0.254, 0.195, -0.109, -0.058, + -0.279, 0.402, -0.107, 0.308, -0.273, 0.019, 0.082, 0.399, + -0.658, -0.03 , 0.276, 0.041, 0.187, -0.331, 0.165, 0.017, + 0.171, -0.203, -0.198, 0.115, -0.007, 0.337, -0.444, 0.615, + -0.657, 1.285, 0.2 , -0.062, 0.038, 0.089, -0.068, -0.058] + # Flush in order to provoke a merge later + - do: + indices.flush: + index: bbq_hnsw + + - do: + indices.forcemerge: + index: bbq_hnsw + max_num_segments: 1 +--- +"Profile rescored knn search": + + - do: + search: + index: bbq_hnsw + body: + profile: true + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + k: 3 + num_candidates: 3 + "rescore_vector": + "num_candidates_factor": 2.0 + + # We expect the knn search ops + rescoring num_cnaidates (for rescoring) per shard + - match: { profile.shards.0.dfs.knn.0.vector_operations_count: 6 } + + # Search with similarity to check number of operations are propagated correctly + - do: + search: + index: bbq_hnsw + body: + profile: true + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + k: 3 + num_candidates: 3 + similarity: 100000 + "rescore_vector": + "num_candidates_factor": 2.0 + + # We expect the knn search ops + rescoring num_cnaidates (for rescoring) per shard + - match: { profile.shards.0.dfs.knn.0.vector_operations_count: 6 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml index b3d86a066550e..534db18b5eb9c 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml @@ -541,3 +541,58 @@ setup: num_candidates: 3 - match: { hits.total.value: 0 } +--- +"Vector rescoring has no effect for non-quantized vectors and provides same results as non-rescored knn": + - requires: + reason: 'Quantized vector rescoring is required' + test_runner_features: [capabilities] + capabilities: + - method: GET + path: /_search + capabilities: [knn_quantized_vector_rescore] + - skip: + features: "headers" + + # Non-rescored knn + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: test + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + k: 3 + num_candidates: 3 + + # Get scores - hit ordering may change depending on how things are distributed + - match: { hits.total: 3 } + - set: { hits.hits.0._score: knn_score0 } + - set: { hits.hits.1._score: knn_score1 } + - set: { hits.hits.2._score: knn_score2 } + + # Rescored knn + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: test + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + k: 3 + num_candidates: 3 + rescore_vector: + num_candidates_factor: 1.5 + + # Compare scores as hit IDs may change depending on how things are distributed + - match: { hits.total: 3 } + - match: { hits.hits.0._score: $knn_score0 } + - match: { hits.hits.1._score: $knn_score1 } + - match: { hits.hits.2._score: $knn_score2 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml new file mode 100644 index 0000000000000..2567a4ac597d9 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml @@ -0,0 +1,249 @@ +setup: + - requires: + cluster_features: "mapper.vectors.bbq" + reason: 'kNN float to better-binary quantization is required' + - do: + indices.create: + index: bbq_hnsw + body: + settings: + index: + number_of_shards: 1 + mappings: + properties: + vector: + type: dense_vector + dims: 64 + index: true + similarity: max_inner_product + index_options: + type: bbq_hnsw + + - do: + index: + index: bbq_hnsw + id: "1" + body: + vector: [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, + 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, + 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, + -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, + -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, + -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, + -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, + -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] + # Flush in order to provoke a merge later + - do: + indices.flush: + index: bbq_hnsw + + - do: + index: + index: bbq_hnsw + id: "2" + body: + vector: [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, + -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, + 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, + -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, + -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, + -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, + 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, + -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] + # Flush in order to provoke a merge later + - do: + indices.flush: + index: bbq_hnsw + + - do: + index: + index: bbq_hnsw + id: "3" + body: + name: rabbit.jpg + vector: [0.139, 0.178, -0.117, 0.399, 0.014, -0.139, 0.347, -0.33 , + 0.139, 0.34 , -0.052, -0.052, -0.249, 0.327, -0.288, 0.049, + 0.464, 0.338, 0.516, 0.247, -0.104, 0.259, -0.209, -0.246, + -0.11 , 0.323, 0.091, 0.442, -0.254, 0.195, -0.109, -0.058, + -0.279, 0.402, -0.107, 0.308, -0.273, 0.019, 0.082, 0.399, + -0.658, -0.03 , 0.276, 0.041, 0.187, -0.331, 0.165, 0.017, + 0.171, -0.203, -0.198, 0.115, -0.007, 0.337, -0.444, 0.615, + -0.657, 1.285, 0.2 , -0.062, 0.038, 0.089, -0.068, -0.058] + # Flush in order to provoke a merge later + - do: + indices.flush: + index: bbq_hnsw + + - do: + indices.forcemerge: + index: bbq_hnsw + max_num_segments: 1 +--- +"Test knn search": + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ optimized_scalar_quantization_bbq ] + test_runner_features: capabilities + reason: "BBQ scoring improved and changed with optimized_scalar_quantization_bbq" + - do: + search: + index: bbq_hnsw + body: + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + k: 3 + num_candidates: 3 + + - match: { hits.hits.0._id: "1" } + - match: { hits.hits.1._id: "3" } + - match: { hits.hits.2._id: "2" } +--- +"Vector rescoring has same scoring as exact search for kNN section": + - requires: + reason: 'Quantized vector rescoring is required' + test_runner_features: [capabilities] + capabilities: + - method: GET + path: /_search + capabilities: [knn_quantized_vector_rescore] + - skip: + features: "headers" + + # Rescore + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: bbq_hnsw + body: + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + k: 3 + num_candidates: 3 + rescore_vector: + num_candidates_factor: 1.5 + + # Get rescoring scores - hit ordering may change depending on how things are distributed + - match: { hits.total: 3 } + - set: { hits.hits.0._score: rescore_score0 } + - set: { hits.hits.1._score: rescore_score1 } + - set: { hits.hits.2._score: rescore_score2 } + + # Exact knn via script score + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + body: + query: + script_score: + query: {match_all: {} } + script: + source: "double similarity = dotProduct(params.query_vector, 'vector'); return similarity < 0 ? 1 / (1 + -1 * similarity) : similarity + 1" + params: + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + + # Compare scores as hit IDs may change depending on how things are distributed + - match: { hits.total: 3 } + - match: { hits.hits.0._score: $rescore_score0 } + - match: { hits.hits.1._score: $rescore_score1 } + - match: { hits.hits.2._score: $rescore_score2 } + +--- +"Test bad quantization parameters": + - do: + catch: bad_request + indices.create: + index: bad_bbq_hnsw + body: + mappings: + properties: + vector: + type: dense_vector + dims: 64 + element_type: byte + index: true + index_options: + type: bbq_hnsw + + - do: + catch: bad_request + indices.create: + index: bad_bbq_hnsw + body: + mappings: + properties: + vector: + type: dense_vector + dims: 64 + index: false + index_options: + type: bbq_hnsw +--- +"Test few dimensions fail indexing": + - do: + catch: bad_request + indices.create: + index: bad_bbq_hnsw + body: + mappings: + properties: + vector: + type: dense_vector + dims: 42 + index: true + index_options: + type: bbq_hnsw + + - do: + indices.create: + index: dynamic_dim_bbq_hnsw + body: + mappings: + properties: + vector: + type: dense_vector + index: true + similarity: l2_norm + index_options: + type: bbq_hnsw + + - do: + catch: bad_request + index: + index: dynamic_dim_bbq_hnsw + body: + vector: [1.0, 2.0, 3.0, 4.0, 5.0] + + - do: + index: + index: dynamic_dim_bbq_hnsw + body: + vector: [1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0] diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_byte_quantized.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_byte_quantized.yml index b7a5517309949..b1e35789e8737 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_byte_quantized.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_byte_quantized.yml @@ -368,6 +368,65 @@ setup: - match: {hits.hits.2._id: "1"} - gte: {hits.hits.2._score: 0.78} - lte: {hits.hits.2._score: 0.791} + +--- +# Won't be true for larger datasets, but this helps checking kNN vs rescoring vs exact search +"Vector rescoring has the same scoring as exact search for kNN section": + - requires: + reason: 'Quantized vector rescoring is required' + test_runner_features: [capabilities] + capabilities: + - method: GET + path: /_search + capabilities: [knn_quantized_vector_rescore] + - skip: + features: "headers" + + # Rescore + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: hnsw_byte_quantized + body: + size: 3 + query: + knn: + k: 3 + num_candidates: 3 + field: vector + query_vector: [0.5, 111.3, -13.0, 14.8, -156.0] + rescore_vector: + num_candidates_factor: 1.5 + + # Get rescoring scores - hit ordering may change depending on how things are distributed + - match: { hits.total: 3 } + - set: { hits.hits.0._score: rescore_score0 } + - set: { hits.hits.1._score: rescore_score1 } + - set: { hits.hits.2._score: rescore_score2 } + + # Exact knn via script score + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + body: + query: + script_score: + query: {match_all: {} } + script: + source: "1.0 / (1.0 + Math.pow(l2norm(params.query_vector, 'vector'), 2.0))" + params: + query_vector: [0.5, 111.3, -13.0, 14.8, -156.0] + + # Compare scores as hit IDs may change depending on how things are distributed + - match: { hits.total: 3 } + - match: { hits.hits.0._score: $rescore_score0 } + - match: { hits.hits.1._score: $rescore_score1 } + - match: { hits.hits.2._score: $rescore_score2 } + --- "Test bad quantization parameters": - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_half_byte_quantized.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_half_byte_quantized.yml index 5f1af2ca5c52f..54e9eadf42e0b 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_half_byte_quantized.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_half_byte_quantized.yml @@ -549,6 +549,62 @@ setup: - match: { hits.hits.1._id: "2"} - match: { hits.hits.2._id: "3"} --- +"Vector rescoring has the same scoring as exact search for kNN section": + - requires: + reason: 'Quantized vector rescoring is required' + test_runner_features: [capabilities] + capabilities: + - method: GET + path: /_search + capabilities: [knn_quantized_vector_rescore] + - skip: + features: "headers" + + # Rescore + - do: + headers: + Content-Type: application/json + search: + index: hnsw_byte_quantized + rest_total_hits_as_int: true + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8] + k: 3 + num_candidates: 3 + rescore_vector: + num_candidates_factor: 1.5 + + # Get rescoring scores - hit ordering may change depending on how things are distributed + - match: { hits.total: 3 } + - set: { hits.hits.0._score: rescore_score0 } + - set: { hits.hits.1._score: rescore_score1 } + - set: { hits.hits.2._score: rescore_score2 } + + # Exact knn via script score + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + body: + query: + script_score: + query: {match_all: {} } + script: + source: "1.0 / (1.0 + Math.pow(l2norm(params.query_vector, 'vector'), 2.0))" + params: + query_vector: [-0.5, 90.0, -10, 14.8] + + # Compare scores as hit IDs may change depending on how things are distributed + - match: { hits.total: 3 } + - match: { hits.hits.0._score: $rescore_score0 } + - match: { hits.hits.1._score: $rescore_score1 } + - match: { hits.hits.2._score: $rescore_score2 } + +--- "Test odd dimensions fail indexing": - do: catch: bad_request diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_bbq_flat.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_bbq_flat.yml new file mode 100644 index 0000000000000..a3cd624ef0ab8 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_bbq_flat.yml @@ -0,0 +1,254 @@ +setup: + - requires: + cluster_features: "mapper.vectors.bbq" + reason: 'kNN float to better-binary quantization is required' + - do: + indices.create: + index: bbq_flat + body: + settings: + index: + number_of_shards: 1 + mappings: + properties: + vector: + type: dense_vector + dims: 64 + index: true + similarity: max_inner_product + index_options: + type: bbq_flat + + - do: + index: + index: bbq_flat + id: "1" + body: + vector: [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, + 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, + 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, + -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, + -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, + -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, + -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, + -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] + # Flush in order to provoke a merge later + - do: + indices.flush: + index: bbq_flat + + - do: + index: + index: bbq_flat + id: "2" + body: + vector: [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, + -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, + 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, + -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, + -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, + -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, + 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, + -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] + # Flush in order to provoke a merge later + - do: + indices.flush: + index: bbq_flat + + - do: + index: + index: bbq_flat + id: "3" + body: + vector: [0.139, 0.178, -0.117, 0.399, 0.014, -0.139, 0.347, -0.33 , + 0.139, 0.34 , -0.052, -0.052, -0.249, 0.327, -0.288, 0.049, + 0.464, 0.338, 0.516, 0.247, -0.104, 0.259, -0.209, -0.246, + -0.11 , 0.323, 0.091, 0.442, -0.254, 0.195, -0.109, -0.058, + -0.279, 0.402, -0.107, 0.308, -0.273, 0.019, 0.082, 0.399, + -0.658, -0.03 , 0.276, 0.041, 0.187, -0.331, 0.165, 0.017, + 0.171, -0.203, -0.198, 0.115, -0.007, 0.337, -0.444, 0.615, + -0.657, 1.285, 0.2 , -0.062, 0.038, 0.089, -0.068, -0.058] + # Flush in order to provoke a merge later + - do: + indices.flush: + index: bbq_flat + + - do: + indices.forcemerge: + index: bbq_flat + max_num_segments: 1 +--- +"Test knn search": + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ optimized_scalar_quantization_bbq ] + test_runner_features: capabilities + reason: "BBQ scoring improved and changed with optimized_scalar_quantization_bbq" + - do: + search: + index: bbq_flat + body: + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + k: 3 + num_candidates: 3 + + - match: { hits.hits.0._id: "1" } + - match: { hits.hits.1._id: "3" } + - match: { hits.hits.2._id: "2" } +--- +"Vector rescoring has same scoring as exact search for kNN section": + - requires: + reason: 'Quantized vector rescoring is required' + test_runner_features: [capabilities] + capabilities: + - method: GET + path: /_search + capabilities: [knn_quantized_vector_rescore] + - skip: + features: "headers" + + # Rescore + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: bbq_flat + body: + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17, + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + k: 3 + num_candidates: 3 + rescore_vector: + num_candidates_factor: 1.5 + + # Get rescoring scores - hit ordering may change depending on how things are distributed + - match: { hits.total: 3 } + - set: { hits.hits.0._score: rescore_score0 } + - set: { hits.hits.1._score: rescore_score1 } + - set: { hits.hits.2._score: rescore_score2 } + + # Exact knn via script score + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: bbq_flat + body: + query: + script_score: + query: { match_all: {} } + script: + source: "double similarity = dotProduct(params.query_vector, 'vector'); return similarity < 0 ? 1 / (1 + -1 * similarity) : similarity + 1" + params: + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17, + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + # Compare scores as hit IDs may change depending on how things are distributed + - match: { hits.total: 3 } + - match: { hits.hits.0._score: $rescore_score0 } + - match: { hits.hits.1._score: $rescore_score1 } + - match: { hits.hits.2._score: $rescore_score2 } + +--- +"Test bad parameters": + - do: + catch: bad_request + indices.create: + index: bad_bbq_flat + body: + mappings: + properties: + vector: + type: dense_vector + dims: 64 + index: true + index_options: + type: bbq_flat + m: 42 + + - do: + catch: bad_request + indices.create: + index: bad_bbq_flat + body: + mappings: + properties: + vector: + type: dense_vector + dims: 64 + element_type: byte + index: true + index_options: + type: bbq_flat +--- +"Test few dimensions fail indexing": + # verify index creation fails + - do: + catch: bad_request + indices.create: + index: bad_bbq_flat + body: + mappings: + properties: + vector: + type: dense_vector + dims: 42 + index: true + similarity: l2_norm + index_options: + type: bbq_flat + + # verify dynamic dimension fails + - do: + indices.create: + index: dynamic_dim_bbq_flat + body: + mappings: + properties: + vector: + type: dense_vector + index: true + similarity: l2_norm + index_options: + type: bbq_flat + + # verify index fails for odd dim vector + - do: + catch: bad_request + index: + index: dynamic_dim_bbq_flat + body: + vector: [1.0, 2.0, 3.0, 4.0, 5.0] + + # verify that we can index an even dim vector after the odd dim vector failure + - do: + index: + index: dynamic_dim_bbq_flat + body: + vector: [1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0] diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_flat.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_flat.yml index 1b439967ba163..a59aedceff3d3 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_flat.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_flat.yml @@ -257,6 +257,61 @@ setup: - gte: {hits.hits.2._score: 0.78} - lte: {hits.hits.2._score: 0.791} --- +"Vector rescoring has no effect for non-quantized vectors and provides same results as non-rescored knn": + - requires: + reason: 'Quantized vector rescoring is required' + test_runner_features: [capabilities] + capabilities: + - method: GET + path: /_search + capabilities: [knn_quantized_vector_rescore] + - skip: + features: "headers" + + # Non-rescored knn + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: flat + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + k: 3 + num_candidates: 3 + + # Get scores - hit ordering may change depending on how things are distributed + - match: { hits.total: 3 } + - set: { hits.hits.0._score: knn_score0 } + - set: { hits.hits.1._score: knn_score1 } + - set: { hits.hits.2._score: knn_score2 } + + # Rescored knn + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: flat + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + k: 3 + num_candidates: 3 + rescore_vector: + num_candidates_factor: 1.5 + + # Compare scores as hit IDs may change depending on how things are distributed + - match: { hits.total: 3 } + - match: { hits.hits.0._score: $knn_score0 } + - match: { hits.hits.1._score: $knn_score1 } + - match: { hits.hits.2._score: $knn_score2 } +--- "Test bad parameters": - do: catch: bad_request diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int4_flat.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int4_flat.yml index b9a0b16f2bd7a..6796a92122f9a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int4_flat.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int4_flat.yml @@ -344,3 +344,58 @@ setup: index: dynamic_dim_hnsw_quantized body: vector: [1.0, 2.0, 3.0, 4.0, 5.0, 6.0] +--- +"Vector rescoring has the same scoring as exact search for kNN section": + - requires: + reason: 'Quantized vector rescoring is required' + test_runner_features: [capabilities] + capabilities: + - method: GET + path: /_search + capabilities: [knn_quantized_vector_rescore] + - skip: + features: "headers" + + # Rescore + - do: + headers: + Content-Type: application/json + search: + index: int4_flat + rest_total_hits_as_int: true + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8] + k: 3 + num_candidates: 3 + rescore_vector: + num_candidates_factor: 1.5 + + # Get rescoring scores - hit ordering may change depending on how things are distributed + - match: { hits.total: 3 } + - set: { hits.hits.0._score: rescore_score0 } + - set: { hits.hits.1._score: rescore_score1 } + - set: { hits.hits.2._score: rescore_score2 } + + # Exact knn via script score + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + body: + query: + script_score: + query: {match_all: {} } + script: + source: "1.0 / (1.0 + Math.pow(l2norm(params.query_vector, 'vector'), 2.0))" + params: + query_vector: [-0.5, 90.0, -10, 14.8] + + # Get rescoring scores - hit ordering may change depending on how things are distributed + - match: { hits.total: 3 } + - match: { hits.hits.0._score: $rescore_score0 } + - match: { hits.hits.1._score: $rescore_score1 } + - match: { hits.hits.2._score: $rescore_score2 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int8_flat.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int8_flat.yml index 139747c5e7ee5..d1d312449cb70 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int8_flat.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int8_flat.yml @@ -262,6 +262,60 @@ setup: - gte: {hits.hits.2._score: 0.78} - lte: {hits.hits.2._score: 0.791} --- +"Vector rescoring has the same scoring as exact search for kNN section": + - requires: + reason: 'Quantized vector rescoring is required' + test_runner_features: [capabilities] + capabilities: + - method: GET + path: /_search + capabilities: [knn_quantized_vector_rescore] + - skip: + features: "headers" + + # Rescore + - do: + headers: + Content-Type: application/json + search: + index: int8_flat + rest_total_hits_as_int: true + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + k: 3 + num_candidates: 3 + rescore_vector: + num_candidates_factor: 1.5 + + # Get rescoring scores - hit ordering may change depending on how things are distributed + - match: { hits.total: 3 } + - set: { hits.hits.0._score: rescore_score0 } + - set: { hits.hits.1._score: rescore_score1 } + - set: { hits.hits.2._score: rescore_score2 } + + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + body: + query: + script_score: + query: {match_all: {} } + script: + source: "1.0 / (1.0 + Math.pow(l2norm(params.query_vector, 'vector'), 2.0))" + params: + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + + # Get rescoring scores - hit ordering may change depending on how things are distributed + - match: { hits.total: 3 } + - match: { hits.hits.0._score: $rescore_score0 } + - match: { hits.hits.1._score: $rescore_score1 } + - match: { hits.hits.2._score: $rescore_score2 } +--- "Test bad parameters": - do: catch: bad_request diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit.yml index ed469ffd7ff16..effa3fff61525 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit.yml @@ -354,3 +354,110 @@ setup: dims: 40 index: true similarity: max_inner_product + + +--- +"Search with synthetic source": + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ bit_dense_vector_synthetic_source ] + test_runner_features: capabilities + reason: "Support for bit dense vector synthetic source capability required" + - do: + indices.create: + index: test_synthetic_source + body: + mappings: + properties: + name: + type: keyword + vector1: + type: dense_vector + element_type: bit + dims: 40 + index: false + vector2: + type: dense_vector + element_type: bit + dims: 40 + index: true + similarity: l2_norm + + - do: + index: + index: test_synthetic_source + id: "1" + body: + name: cow.jpg + vector1: [2, -1, 1, 4, -3] + vector2: [2, -1, 1, 4, -3] + + - do: + indices.refresh: {} + + - do: + search: + force_synthetic_source: true + index: test_synthetic_source + + - match: {hits.hits.0._id: "1"} + - match: {hits.hits.0._source.vector1: [2, -1, 1, 4, -3]} + - match: {hits.hits.0._source.vector2: [2, -1, 1, 4, -3]} + +--- +"Vector rescoring has no effect for non-quantized vectors and provides same results as non-rescored knn": + - requires: + reason: 'Quantized vector rescoring is required' + test_runner_features: [capabilities] + capabilities: + - method: GET + path: /_search + capabilities: [knn_quantized_vector_rescore] + - skip: + features: "headers" + + # Non-rescored knn + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: test + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [127.0, -128.0, 0.0, 1.0, -1.0] + k: 3 + num_candidates: 3 + + # Get scores - hit ordering may change depending on how things are distributed + - match: { hits.total: 3 } + - set: { hits.hits.0._score: knn_score0 } + - set: { hits.hits.1._score: knn_score1 } + - set: { hits.hits.2._score: knn_score2 } + + # Rescored knn + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: test + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [127.0, -128.0, 0.0, 1.0, -1.0] + k: 3 + num_candidates: 3 + rescore_vector: + num_candidates_factor: 1.5 + + # Compare scores as hit IDs may change depending on how things are distributed + - match: { hits.total: 3 } + - match: { hits.hits.0._score: $knn_score0 } + - match: { hits.hits.1._score: $knn_score1 } + - match: { hits.hits.2._score: $knn_score2 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit_flat.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit_flat.yml index ec7bde4de8435..cdc1d9c64763e 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit_flat.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit_flat.yml @@ -221,3 +221,59 @@ setup: similarity: l2_norm index_options: type: int8_hnsw + +--- +"Vector rescoring has no effect for non-quantized vectors and provides same results as non-rescored knn": + - requires: + reason: 'Quantized vector rescoring is required' + test_runner_features: [capabilities] + capabilities: + - method: GET + path: /_search + capabilities: [knn_quantized_vector_rescore] + - skip: + features: "headers" + + # Non-rescored knn + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: test + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [127, 127, -128, -128, 127] + k: 3 + num_candidates: 3 + + # Get scores - hit ordering may change depending on how things are distributed + - match: { hits.total: 3 } + - set: { hits.hits.0._score: knn_score0 } + - set: { hits.hits.1._score: knn_score1 } + - set: { hits.hits.2._score: knn_score2 } + + # Rescored knn + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: test + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [127, 127, -128, -128, 127] + k: 3 + num_candidates: 3 + rescore_vector: + num_candidates_factor: 1.5 + + # Compare scores as hit IDs may change depending on how things are distributed + - match: { hits.total: 3 } + - match: { hits.hits.0._score: $knn_score0 } + - match: { hits.hits.1._score: $knn_score1 } + - match: { hits.hits.2._score: $knn_score2 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_byte.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_byte.yml index 0cedfaa873095..213b571a0b4be 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_byte.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_byte.yml @@ -254,3 +254,60 @@ setup: filter: {"term": {"name": "cow.jpg"}} - length: {hits.hits: 0} + +--- +"Vector rescoring has no effect for non-quantized vectors and provides same results as non-rescored knn": + - requires: + reason: 'Quantized vector rescoring is required' + test_runner_features: [capabilities] + capabilities: + - method: GET + path: /_search + capabilities: [knn_quantized_vector_rescore] + - skip: + features: "headers" + + # Non-rescored knn + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: test + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [127, 127, -128, -128, 127] + k: 3 + num_candidates: 3 + + # Get scores - hit ordering may change depending on how things are distributed + - match: { hits.total: 3 } + - set: { hits.hits.0._score: knn_score0 } + - set: { hits.hits.1._score: knn_score1 } + - set: { hits.hits.2._score: knn_score2 } + + # Rescored knn + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: test + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [127, 127, -128, -128, 127] + k: 3 + num_candidates: 3 + rescore_vector: + num_candidates_factor: 1.5 + + # Compare scores as hit IDs may change depending on how things are distributed + - match: { hits.total: 3 } + - match: { hits.hits.0._score: $knn_score0 } + - match: { hits.hits.1._score: $knn_score1 } + - match: { hits.hits.2._score: $knn_score2 } + diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/70_dense_vector_telemetry.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/70_dense_vector_telemetry.yml index 43d820e03481f..8f1b246334f19 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/70_dense_vector_telemetry.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/70_dense_vector_telemetry.yml @@ -1,7 +1,26 @@ setup: + - requires: + cluster_features: [ "gte_v8.4.0" ] + reason: "Cluster mappings stats for indexed dense vector was added in 8.4" - skip: features: headers + - do: + indices.create: + index: index2 + body: + settings: + index.number_of_shards: 2 + mappings: + properties: + vector1: + type: dense_vector + dims: 768 + index: true + similarity: l2_norm + +--- +"Field mapping stats": - do: indices.create: index: index1 @@ -15,6 +34,10 @@ setup: dims: 768 index: true similarity: l2_norm + index_options: + type: hnsw + m: 16 + ef_construction: 100 vector2: type: dense_vector dims: 1024 @@ -24,10 +47,28 @@ setup: type: dense_vector dims: 100 index: false + - do: { cluster.stats: { } } + - length: { indices.mappings.field_types: 1 } + - match: { indices.mappings.field_types.0.name: dense_vector } + - match: { indices.mappings.field_types.0.count: 4 } + - match: { indices.mappings.field_types.0.index_count: 2 } + - match: { indices.mappings.field_types.0.indexed_vector_count: 3 } + - match: { indices.mappings.field_types.0.indexed_vector_dim_min: 768 } + - match: { indices.mappings.field_types.0.indexed_vector_dim_max: 1024 } +--- +"Field mapping stats with field details": + - requires: + test_runner_features: [ capabilities ] + capabilities: + - method: GET + path: /_cluster/stats + capabilities: + - "verbose-dense-vector-mapping-stats" + reason: "Capability required to run test" - do: indices.create: - index: index2 + index: index1 body: settings: index.number_of_shards: 2 @@ -38,12 +79,21 @@ setup: dims: 768 index: true similarity: l2_norm + element_type: byte + index_options: + type: hnsw + m: 16 + ef_construction: 100 + vector2: + type: dense_vector + dims: 1024 + index: true + similarity: dot_product + vector3: + type: dense_vector + dims: 100 + index: false ---- -"Field mapping stats": - - requires: - cluster_features: ["gte_v8.4.0"] - reason: "Cluster mappings stats for indexed dense vector was added in 8.4" - do: { cluster.stats: { } } - length: { indices.mappings.field_types: 1 } - match: { indices.mappings.field_types.0.name: dense_vector } @@ -52,3 +102,10 @@ setup: - match: { indices.mappings.field_types.0.indexed_vector_count: 3 } - match: { indices.mappings.field_types.0.indexed_vector_dim_min: 768 } - match: { indices.mappings.field_types.0.indexed_vector_dim_max: 1024 } + - match: { indices.mappings.field_types.0.vector_index_type_count.hnsw: 1 } + - match: { indices.mappings.field_types.0.vector_index_type_count.int8_hnsw: 2 } + - match: { indices.mappings.field_types.0.vector_index_type_count.not_indexed: 1 } + - match: { indices.mappings.field_types.0.vector_similarity_type_count.l2_norm: 2 } + - match: { indices.mappings.field_types.0.vector_similarity_type_count.dot_product: 1 } + - match: { indices.mappings.field_types.0.vector_element_type_count.float: 3 } + - match: { indices.mappings.field_types.0.vector_element_type_count.byte: 1 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml index 27f12f394c6a4..0b65a69bf500e 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml @@ -387,16 +387,17 @@ "sparse_vector synthetic source": - requires: - cluster_features: ["mapper.source.synthetic_source_fallback"] - reason: introduced in 8.15.0 + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" - do: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: ml.tokens: type: sparse_vector @@ -471,3 +472,120 @@ - match: _source.ml.tokens: {} + +--- +"stored sparse_vector": + + - requires: + cluster_features: [ "mapper.sparse_vector.store_support" ] + reason: "sparse_vector supports store parameter" + + - do: + indices.create: + index: test + body: + mappings: + properties: + ml.tokens: + type: sparse_vector + store: true + + - match: { acknowledged: true } + - do: + index: + index: test + id: "1" + body: + ml: + tokens: + running: 2 + good: 3 + run: 5 + race: 7 + for: 9 + + - match: { result: "created" } + + - do: + indices.refresh: { } + + - do: + search: + index: test + body: + fields: [ "ml.tokens" ] + + - length: { hits.hits.0.fields.ml\\.tokens: 1 } + - length: { hits.hits.0.fields.ml\\.tokens.0: 5 } + - match: { hits.hits.0.fields.ml\\.tokens.0.running: 2.0 } + - match: { hits.hits.0.fields.ml\\.tokens.0.good: 3.0 } + - match: { hits.hits.0.fields.ml\\.tokens.0.run: 5.0 } + - match: { hits.hits.0.fields.ml\\.tokens.0.race: 7.0 } + - match: { hits.hits.0.fields.ml\\.tokens.0.for: 9.0 } + +--- +"stored sparse_vector synthetic source": + + - requires: + cluster_features: [ "mapper.source.mode_from_index_setting", "mapper.sparse_vector.store_support" ] + reason: "sparse_vector supports store parameter" + + - do: + indices.create: + index: test + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + ml.tokens: + type: sparse_vector + store: true + + - match: { acknowledged: true } + + - do: + index: + index: test + id: "1" + body: + ml: + tokens: + running: 2 + good: 3 + run: 5 + race: 7 + for: 9 + + - match: { result: "created" } + + - do: + indices.refresh: { } + + - do: + search: + index: test + body: + fields: [ "ml.tokens" ] + + - match: + hits.hits.0._source: { + ml: { + tokens: { + running: 2.0, + good: 3.0, + run: 5.0, + race: 7.0, + for: 9.0 + } + } + } + + - length: { hits.hits.0.fields.ml\\.tokens: 1 } + - length: { hits.hits.0.fields.ml\\.tokens.0: 5 } + - match: { hits.hits.0.fields.ml\\.tokens.0.running: 2.0 } + - match: { hits.hits.0.fields.ml\\.tokens.0.good: 3.0 } + - match: { hits.hits.0.fields.ml\\.tokens.0.run: 5.0 } + - match: { hits.hits.0.fields.ml\\.tokens.0.race: 7.0 } + - match: { hits.hits.0.fields.ml\\.tokens.0.for: 9.0 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml index 7c345b7d4d3ac..47876d2820aac 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml @@ -1,6 +1,11 @@ +setup: + - requires: + test_runner_features: allowed_warnings_regex --- "Test Index and Search locale dependent mappings / dates": - do: + allowed_warnings_regex: + - 'Date format \[E, d MMM yyyy HH:mm:ss Z] contains textual field specifiers that could change in JDK 23.*' indices.create: index: test_index body: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/230_interval_query.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/230_interval_query.yml index 99bd001bd95e2..6a5f34b5207ce 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/230_interval_query.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/230_interval_query.yml @@ -476,3 +476,53 @@ setup: - match: { hits.hits.0._id: "6" } - match: { hits.hits.1._id: "5" } +--- +"Test regexp": + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ range_regexp_interval_queries ] + test_runner_features: capabilities + reason: "Support for range and regexp interval queries capability required" + - do: + search: + index: test + body: + query: + intervals: + text: + all_of: + intervals: + - match: + query: cold + - regexp: + pattern: ou.*ide + - match: { hits.total.value: 3 } + + +--- +"Test range": + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ range_regexp_interval_queries ] + test_runner_features: capabilities + reason: "Support for range and regexp interval queries capability required" + - do: + search: + index: test + body: + query: + intervals: + text: + all_of: + intervals: + - match: + query: cold + - range: + gte: out + lte: ouu + - match: { hits.total.value: 3 } + diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml index c120bed2d369d..44d966b76f34e 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml @@ -252,7 +252,7 @@ query: range: date: - from: "1990-12-29T22:30:00.000Z" + gte: "1990-12-29T22:30:00.000Z" fields: - field: date format: "yyyy/MM/dd" @@ -1177,3 +1177,121 @@ fetch geo_point: - is_false: hits.hits.0.fields.message - match: { hits.hits.0._source.message.foo: 10 } - match: { hits.hits.0._source.message.foo\.bar: 20 } + +--- +root with subobjects false and dynamic false: + - requires: + cluster_features: mapper.fix_parsing_subobjects_false_dynamic_false + reason: bug fix + + - do: + indices.create: + index: test + body: + mappings: + subobjects: false + dynamic: false + properties: + id: + type: integer + my.keyword.field: + type: keyword + + - do: + bulk: + index: test + refresh: true + body: + - '{ "index": { } }' + - '{ "id": 1, "my": { "keyword.field": "abc" } }' + - match: { errors: false } + + # indexing a dynamically-mapped field still fails (silently) + - do: + bulk: + index: test + refresh: true + body: + - '{ "index": { } }' + - '{ "id": 2, "my": { "random.field": "abc" } }' + - match: { errors: false } + + - do: + search: + index: test + body: + sort: id + fields: [ "*" ] + + - match: { hits.hits.0.fields: { my.keyword.field: [ abc ], id: [ 1 ] } } + - match: { hits.hits.1.fields: { id: [ 2 ] } } + + - do: + search: + index: test + body: + query: + match: + my.keyword.field: abc + + - match: { hits.total.value: 1 } + +--- +object with subobjects false and dynamic false: + - requires: + cluster_features: mapper.fix_parsing_subobjects_false_dynamic_false + reason: bug fix + + - do: + indices.create: + index: test + body: + mappings: + properties: + my: + subobjects: false + dynamic: false + properties: + id: + type: integer + nested.keyword.field: + type: keyword + + - do: + bulk: + index: test + refresh: true + body: + - '{ "index": { } }' + - '{ "id": 1, "my": { "nested": { "keyword.field": "abc" } } }' + - match: { errors: false } + + # indexing a dynamically-mapped field still fails (silently) + - do: + bulk: + index: test + refresh: true + body: + - '{ "index": { } }' + - '{ "id": 2, "my": { "nested": { "random.field": "abc" } } }' + - match: { errors: false } + + - do: + search: + index: test + body: + sort: id + fields: [ "*" ] + + - match: { hits.hits.0.fields: { my.nested.keyword.field: [ abc ], id: [ 1 ] } } + - match: { hits.hits.1.fields: { id: [ 2 ] } } + + - do: + search: + index: test + body: + query: + match: + my.nested.keyword.field: abc + + - match: { hits.total.value: 1 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/350_binary_field.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/350_binary_field.yml index 455d06ba2a984..82b85363b8ad6 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/350_binary_field.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/350_binary_field.yml @@ -49,15 +49,17 @@ --- "binary synthetic source": - requires: - cluster_features: "gte_v8.15.0" - reason: synthetic source support introduced in 8.15 + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" + - do: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: binary: type: binary diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/400_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/400_synthetic_source.yml index 0cc1796bb47de..0d3cbfb696d13 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/400_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/400_synthetic_source.yml @@ -1,3 +1,9 @@ +setup: + - requires: + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" + +--- keyword: - requires: cluster_features: ["gte_v8.4.0"] @@ -7,9 +13,10 @@ keyword: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: kwd: type: keyword @@ -44,9 +51,10 @@ stored text: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: text: type: text @@ -83,8 +91,6 @@ stored keyword: index: test body: mappings: - _source: - mode: synthetic properties: kwd: type: keyword @@ -120,9 +126,10 @@ stored keyword without sibling fields: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: kwd: type: keyword @@ -165,9 +172,10 @@ force_synthetic_source_ok: indices.create: index: test body: + settings: + index: + mapping.source.mode: stored mappings: - _source: - mode: stored properties: obj: properties: @@ -218,9 +226,10 @@ doc values keyword with ignore_above: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: kwd: type: keyword @@ -286,9 +295,10 @@ stored keyword with ignore_above: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: kwd: type: keyword @@ -356,9 +366,10 @@ _source filtering: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: kwd: type: keyword @@ -397,9 +408,9 @@ _doc_count: indices.create: index: test body: - mappings: - _source: - mode: synthetic + settings: + index: + mapping.source.mode: synthetic - do: index: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/500_date_range.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/500_date_range.yml index 76057b5a364fb..e9bfffb8da604 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/500_date_range.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/500_date_range.yml @@ -123,3 +123,29 @@ setup: - match: { hits.total: 1 } - length: { hits.hits: 1 } - match: { hits.hits.0._id: "4" } + + +--- +"from, to, include_lower, include_upper deprecated": + - requires: + cluster_features: "gte_v8.16.0" + reason: 'from, to, include_lower, include_upper parameters are deprecated since 8.16.0' + test_runner_features: warnings + + - do: + warnings: + - "Deprecated field [from] used, this field is unused and will be removed entirely" + - "Deprecated field [to] used, this field is unused and will be removed entirely" + - "Deprecated field [include_lower] used, this field is unused and will be removed entirely" + - "Deprecated field [include_upper] used, this field is unused and will be removed entirely" + search: + index: dates + body: + sort: field + query: + range: + date: + from: 1000 + to: 2023 + include_lower: false + include_upper: false diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/520_fetch_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/520_fetch_fields.yml index 2b309f502f0c2..9a43199755d75 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/520_fetch_fields.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/520_fetch_fields.yml @@ -128,18 +128,88 @@ fetch _seq_no via stored_fields: --- fetch _seq_no via fields: + - requires: + cluster_features: ["meta_fetch_fields_error_code_changed"] + reason: The fields_api returns a 400 instead a 5xx when _seq_no is requested via fields - do: - catch: "request" + catch: bad_request search: index: test body: fields: [ _seq_no ] - # This should be `unauthorized` (401) or `forbidden` (403) or at least `bad request` (400) - # while instead it is mapped to an `internal_server_error (500)` - - match: { status: 500 } - - match: { error.root_cause.0.type: unsupported_operation_exception } + - match: { status: 400 } + - match: { error.root_cause.0.type: illegal_argument_exception } + - match: { error.root_cause.0.reason: "error fetching [_seq_no]: Cannot fetch values for internal field [_seq_no]." } + +--- +fetch _source via fields: + - requires: + cluster_features: ["meta_fetch_fields_error_code_changed"] + reason: The fields_api returns a 400 instead a 5xx when _seq_no is requested via fields + + - do: + catch: bad_request + search: + index: test + body: + fields: [ _source ] + + - match: { status: 400 } + - match: { error.root_cause.0.type: illegal_argument_exception } + - match: { error.root_cause.0.reason: "error fetching [_source]: Cannot fetch values for internal field [_source]." } + +--- +fetch _feature via fields: + - requires: + cluster_features: ["meta_fetch_fields_error_code_changed"] + reason: The fields_api returns a 400 instead a 5xx when _seq_no is requested via fields + + - do: + catch: bad_request + search: + index: test + body: + fields: [ _feature ] + + - match: { status: 400 } + - match: { error.root_cause.0.type: illegal_argument_exception } + - match: { error.root_cause.0.reason: "error fetching [_feature]: Cannot fetch values for internal field [_feature]." } + +--- +fetch _nested_path via fields: + - requires: + cluster_features: ["meta_fetch_fields_error_code_changed"] + reason: The fields_api returns a 400 instead a 5xx when _seq_no is requested via fields + + - do: + catch: bad_request + search: + index: test + body: + fields: [ _nested_path ] + + - match: { status: 400 } + - match: { error.root_cause.0.type: illegal_argument_exception } + - match: { error.root_cause.0.reason: "error fetching [_nested_path]: Cannot fetch values for internal field [_nested_path]." } + +--- +fetch _field_names via fields: + - requires: + cluster_features: ["meta_fetch_fields_error_code_changed"] + reason: The fields_api returns a 400 instead a 5xx when _seq_no is requested via fields + + - do: + catch: bad_request + search: + index: test + body: + fields: [ _field_names ] + + - match: { status: 400 } + - match: { error.root_cause.0.type: illegal_argument_exception } + - match: { error.root_cause.0.reason: "error fetching [_field_names]: Cannot fetch values for internal field [_field_names]." } --- fetch fields with none stored_fields: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/530_ignore_above_stored_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/530_ignore_above_stored_source.yml new file mode 100644 index 0000000000000..1730a49f743d9 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/530_ignore_above_stored_source.yml @@ -0,0 +1,214 @@ +--- +ignore_above mapping level setting: + - requires: + cluster_features: [ "mapper.ignore_above_index_level_setting" ] + reason: introduce ignore_above index level setting + - do: + indices.create: + index: test + body: + settings: + index: + mapping: + ignore_above: 10 + mappings: + properties: + keyword: + type: keyword + flattened: + type: flattened + + - do: + index: + index: test + refresh: true + id: "1" + body: { "keyword": "foo bar", "flattened": { "value": "the quick brown fox" } } + + - do: + search: + body: + fields: + - keyword + - flattened + query: + match_all: {} + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.keyword: "foo bar" } + - match: { hits.hits.0._source.flattened.value: "the quick brown fox" } + - match: { hits.hits.0.fields.keyword.0: "foo bar" } + - match: { hits.hits.0.fields.flattened: null } + +--- +ignore_above mapping level setting on arrays: + - requires: + cluster_features: [ "mapper.ignore_above_index_level_setting" ] + reason: introduce ignore_above index level setting + - do: + indices.create: + index: test + body: + settings: + index: + mapping: + ignore_above: 10 + mappings: + properties: + keyword: + type: keyword + flattened: + type: flattened + + - do: + index: + index: test + refresh: true + id: "1" + body: { "keyword": ["foo bar", "the quick brown fox"], "flattened": { "value": ["the quick brown fox", "jumps over"] } } + + - do: + search: + body: + fields: + - keyword + - flattened + query: + match_all: {} + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.keyword: ["foo bar", "the quick brown fox"] } + - match: { hits.hits.0._source.flattened.value: ["the quick brown fox", "jumps over"] } + - match: { hits.hits.0.fields.keyword.0: "foo bar" } + - match: { hits.hits.0.fields.flattened.0.value: "jumps over" } + +--- +ignore_above mapping overrides setting: + - requires: + cluster_features: [ "mapper.ignore_above_index_level_setting" ] + reason: introduce ignore_above index level setting + - do: + indices.create: + index: test + body: + settings: + index: + mapping: + ignore_above: 10 + mappings: + properties: + keyword: + type: keyword + ignore_above: 100 + flattened: + type: flattened + ignore_above: 100 + + - do: + index: + index: test + refresh: true + id: "1" + body: { "keyword": "foo bar baz foo bar baz", "flattened": { "value": "the quick brown fox" } } + + - do: + search: + body: + fields: + - keyword + - flattened + query: + match_all: { } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.keyword: "foo bar baz foo bar baz" } + - match: { hits.hits.0._source.flattened.value: "the quick brown fox" } + - match: { hits.hits.0.fields.keyword.0: "foo bar baz foo bar baz" } + - match: { hits.hits.0.fields.flattened.0.value: "the quick brown fox" } + +--- +ignore_above mapping overrides setting on arrays: + - requires: + cluster_features: [ "mapper.ignore_above_index_level_setting" ] + reason: introduce ignore_above index level setting + - do: + indices.create: + index: test + body: + settings: + index: + mapping: + ignore_above: 10 + mappings: + properties: + keyword: + type: keyword + ignore_above: 100 + flattened: + type: flattened + ignore_above: 100 + + - do: + index: + index: test + refresh: true + id: "1" + body: { "keyword": ["foo bar baz foo bar baz", "the quick brown fox jumps over"], "flattened": { "value": ["the quick brown fox", "jumps over the lazy dog"] } } + + - do: + search: + body: + fields: + - keyword + - flattened + query: + match_all: { } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.keyword: ["foo bar baz foo bar baz", "the quick brown fox jumps over"] } + - match: { hits.hits.0._source.flattened.value: ["the quick brown fox", "jumps over the lazy dog"] } + - match: { hits.hits.0.fields.keyword: ["foo bar baz foo bar baz", "the quick brown fox jumps over"] } + - match: { hits.hits.0.fields.flattened.0.value: ["the quick brown fox", "jumps over the lazy dog"] } + +--- +date ignore_above index level setting: + - requires: + cluster_features: [ "mapper.ignore_above_index_level_setting" ] + reason: introduce ignore_above index level setting + - do: + indices.create: + index: test + body: + settings: + index: + mapping: + ignore_above: 10 + mappings: + properties: + keyword: + type: keyword + date: + type: date + format: "yyyy-MM-dd'T'HH:mm:ss" + + - do: + index: + index: test + refresh: true + id: "1" + body: { "keyword": ["2023-09-17T15:30:00", "2023-09-17T15:31:00"], "date": ["2023-09-17T15:30:00", "2023-09-17T15:31:00"] } + + - do: + search: + body: + fields: + - keyword + - date + query: + match_all: {} + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.keyword: ["2023-09-17T15:30:00", "2023-09-17T15:31:00"] } + - match: { hits.hits.0._source.date: ["2023-09-17T15:30:00", "2023-09-17T15:31:00"] } + - match: { hits.hits.0.fields.keyword: null } + - match: { hits.hits.0.fields.date: ["2023-09-17T15:30:00","2023-09-17T15:31:00"] } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/540_ignore_above_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/540_ignore_above_synthetic_source.yml new file mode 100644 index 0000000000000..772c3c24170cd --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/540_ignore_above_synthetic_source.yml @@ -0,0 +1,179 @@ +setup: + - requires: + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" + +--- +ignore_above mapping level setting: + - requires: + cluster_features: [ "mapper.ignore_above_index_level_setting" ] + reason: introduce ignore_above index level setting + - do: + indices.create: + index: test + body: + settings: + index: + mapping: + source.mode: synthetic + ignore_above: 10 + mappings: + properties: + keyword: + type: keyword + flattened: + type: flattened + + - do: + index: + index: test + refresh: true + id: "1" + body: { "keyword": "foo bar", "flattened": { "value": "the quick brown fox" } } + + - do: + search: + body: + fields: + - keyword + - flattened + query: + match_all: {} + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.keyword: "foo bar" } + - match: { hits.hits.0._source.flattened.value: "the quick brown fox" } + - match: { hits.hits.0.fields.keyword.0: "foo bar" } + +--- +ignore_above mapping level setting on arrays: + - requires: + cluster_features: [ "mapper.flattened.ignore_above_with_arrays_support" ] + reason: requires support of ignore_above with arrays for flattened fields + - do: + indices.create: + index: test + body: + settings: + index: + mapping: + source.mode: synthetic + ignore_above: 10 + mappings: + properties: + keyword: + type: keyword + flattened: + type: flattened + + - do: + index: + index: test + refresh: true + id: "1" + body: { "keyword": ["foo bar", "the quick brown fox"], "flattened": { "value": ["the quick brown fox", "jumps over"] } } + + - do: + search: + body: + fields: + - keyword + - flattened + query: + match_all: {} + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.keyword: ["foo bar", "the quick brown fox"] } + - match: { hits.hits.0._source.flattened.value: [ "jumps over", "the quick brown fox" ] } + - match: { hits.hits.0.fields.keyword.0: "foo bar" } + - match: { hits.hits.0.fields.flattened.0.value: "jumps over" } + +--- +ignore_above mapping overrides setting: + - requires: + cluster_features: [ "mapper.ignore_above_index_level_setting" ] + reason: introduce ignore_above index level setting + - do: + indices.create: + index: test + body: + settings: + index: + mapping: + source.mode: synthetic + ignore_above: 10 + mappings: + properties: + keyword: + type: keyword + ignore_above: 100 + flattened: + type: flattened + ignore_above: 100 + + - do: + index: + index: test + refresh: true + id: "1" + body: { "keyword": "foo bar baz foo bar baz", "flattened": { "value": "the quick brown fox" } } + + - do: + search: + body: + fields: + - keyword + - flattened + query: + match_all: { } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.keyword: "foo bar baz foo bar baz" } + - match: { hits.hits.0._source.flattened.value: "the quick brown fox" } + - match: { hits.hits.0.fields.keyword.0: "foo bar baz foo bar baz" } + - match: { hits.hits.0.fields.flattened.0.value: "the quick brown fox" } + +--- +ignore_above mapping overrides setting on arrays: + - requires: + cluster_features: [ "mapper.ignore_above_index_level_setting" ] + reason: introduce ignore_above index level setting + - do: + indices.create: + index: test + body: + settings: + index: + mapping: + source.mode: synthetic + ignore_above: 10 + mappings: + properties: + keyword: + type: keyword + ignore_above: 100 + flattened: + type: flattened + ignore_above: 100 + + - do: + index: + index: test + refresh: true + id: "1" + body: { "keyword": ["foo bar baz foo bar baz", "the quick brown fox jumps over"], "flattened": { "value": ["the quick brown fox", "jumps over the lazy dog"] } } + + - do: + search: + body: + fields: + - keyword + - flattened + query: + match_all: { } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.keyword: ["foo bar baz foo bar baz", "the quick brown fox jumps over"] } + - match: { hits.hits.0._source.flattened.value: ["jumps over the lazy dog", "the quick brown fox"] } + - match: { hits.hits.0.fields.keyword: ["foo bar baz foo bar baz", "the quick brown fox jumps over"] } + - match: { hits.hits.0.fields.flattened.0.value: ["jumps over the lazy dog", "the quick brown fox"] } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/550_ignore_above_invalid.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/550_ignore_above_invalid.yml new file mode 100644 index 0000000000000..3c29845871fe7 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/550_ignore_above_invalid.yml @@ -0,0 +1,63 @@ +--- +ignore_above index setting negative value: + - do: + catch: bad_request + indices.create: + index: test + body: + settings: + index: + mapping: + ignore_above: -1 + mappings: + properties: + keyword: + type: keyword + +--- +keyword ignore_above mapping setting negative value: + - requires: + cluster_features: [ "mapper.ignore_above_index_level_setting" ] + reason: introduce ignore_above index level setting + - do: + catch: bad_request + indices.create: + index: test + body: + mappings: + properties: + keyword: + ignore_above: -2 + type: keyword + +--- +flattened ignore_above mapping setting negative value: + - requires: + cluster_features: [ "mapper.ignore_above_index_level_setting" ] + reason: introduce ignore_above index level setting + - do: + catch: bad_request + indices.create: + index: test + body: + mappings: + properties: + flattened: + ignore_above: -2 + type: flattened + +--- +wildcard ignore_above mapping setting negative value: + - requires: + cluster_features: [ "mapper.ignore_above_index_level_setting" ] + reason: introduce ignore_above index level setting + - do: + catch: bad_request + indices.create: + index: test + body: + mappings: + properties: + wildcard: + ignore_above: -2 + type: wildcard diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/600_flattened_ignore_above.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/600_flattened_ignore_above.yml new file mode 100644 index 0000000000000..a4a9b1aaecb22 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/600_flattened_ignore_above.yml @@ -0,0 +1,131 @@ +--- +flattened ignore_above single-value field: + - requires: + cluster_features: [ "flattened.ignore_above_support" ] + reason: introduce ignore_above support in flattened fields + - do: + indices.create: + index: test + body: + mappings: + properties: + name: + type: keyword + keyword: + type: keyword + ignore_above: 5 + flat: + type: flattened + ignore_above: 5 + + - do: + index: + index: test + id: "1" + refresh: true + body: + name: "A" + keyword: "foo" + flat: { "value": "foo", "key": "foo key" } + + - do: + index: + index: test + id: "2" + refresh: true + body: + name: "B" + keyword: "foo bar" + flat: { "value": "foo bar", "key": "foo bar key"} + + - do: + search: + index: test + sort: name + body: + fields: + - keyword + - flat + query: + match_all: {} + + - match: { hits.total.value: 2 } + + - match: { hits.hits.0._source.keyword: "foo" } + - match: { hits.hits.0._source.flat.value: "foo" } + - match: { hits.hits.0._source.flat.key: "foo key" } + - match: { hits.hits.1._source.keyword: "foo bar" } + - match: { hits.hits.1._source.flat.value: "foo bar" } + - match: { hits.hits.1._source.flat.key: "foo bar key" } + + - match: { hits.hits.0.fields.keyword.0: "foo" } + - match: { hits.hits.0.fields.flat.0.value: "foo" } + - match: { hits.hits.1.fields.keyword.0: null } + - match: { hits.hits.1.fields.flat.0.value: null } + +--- +flattened ignore_above multi-value field: + - requires: + cluster_features: [ "flattened.ignore_above_support" ] + reason: introduce ignore_above support in flattened fields + - do: + indices.create: + index: test + body: + mappings: + properties: + name: + type: keyword + keyword: + type: keyword + ignore_above: 5 + flat: + type: flattened + ignore_above: 5 + + - do: + index: + index: test + id: "1" + refresh: true + body: + name: "A" + keyword: ["foo","bar"] + flat: { "value": ["foo", "bar"], "key": "foo bar array key" } + + - do: + index: + index: test + id: "2" + refresh: true + body: + name: "B" + keyword: ["foobar", "foo", "bar"] + flat: { "value": ["foobar", "foo"], "key": ["foo key", "bar key"]} + + - do: + search: + index: test + sort: name + body: + fields: + - keyword + - flat + query: + match_all: { } + + - match: { hits.total.value: 2 } + + - match: { hits.hits.0._source.keyword: ["foo", "bar"] } + - match: { hits.hits.0._source.flat.value: ["foo", "bar"] } + - match: { hits.hits.0._source.flat.key: "foo bar array key" } + - match: { hits.hits.1._source.keyword: ["foobar", "foo", "bar"] } + - match: { hits.hits.1._source.flat.value: ["foobar", "foo"] } + - match: { hits.hits.1._source.flat.key: ["foo key", "bar key"] } + + - match: { hits.hits.0.fields.keyword: [ "foo", "bar" ] } + - match: { hits.hits.0.fields.flat.0.value: ["foo", "bar"] } + - match: { hits.hits.0.fields.flat.0.key: null } + - match: { hits.hits.1.fields.keyword: [ "foo", "bar" ] } + - match: { hits.hits.1.fields.flat.0.value: "foo" } + - match: { hits.hits.1.fields.flat.0.key: null } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/90_search_after.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/90_search_after.yml index 1fefc8bffffa1..d3b2b5a412717 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/90_search_after.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/90_search_after.yml @@ -218,31 +218,6 @@ - match: {hits.hits.0._source.timestamp: "2019-10-21 00:30:04.828740" } - match: {hits.hits.0.sort: [1571617804828740000] } - ---- -"_shard_doc sort": - - requires: - cluster_features: ["gte_v7.12.0"] - reason: _shard_doc sort was added in 7.12 - - - do: - indices.create: - index: test - - do: - index: - index: test - id: "1" - body: { id: 1, foo: bar, age: 18 } - - - do: - catch: /\[_shard_doc\] sort field cannot be used without \[point in time\]/ - search: - index: test - body: - size: 1 - sort: ["_shard_doc"] - search_after: [ 0L ] - --- "Format sort values": - requires: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/10_synonyms_put.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/10_synonyms_put.yml index bcd58f3f7bd64..93f1fafa7ab85 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/10_synonyms_put.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/10_synonyms_put.yml @@ -15,6 +15,13 @@ setup: - match: { result: "created" } + - do: + cluster.health: + index: .synonyms-2 + timeout: 2s + wait_for_status: green + ignore: 408 + - do: synonyms.get_synonym: id: test-update-synonyms @@ -58,6 +65,13 @@ setup: - match: { result: "created" } + - do: + cluster.health: + index: .synonyms-2 + timeout: 2s + wait_for_status: green + ignore: 408 + - do: synonyms.get_synonym: id: test-empty-synonyms diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/110_synonyms_invalid.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/110_synonyms_invalid.yml index d3d0a3bb4df70..7f545b466e65f 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/110_synonyms_invalid.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/110_synonyms_invalid.yml @@ -11,6 +11,14 @@ setup: synonyms_set: synonyms: "foo => bar, baz" + # This is to ensure that all index shards (write and read) are available. In serverless this can take some time. + - do: + cluster.health: + index: .synonyms-2 + timeout: 2s + wait_for_status: green + ignore: 408 + - do: indices.create: index: test_index diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/20_synonyms_get.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/20_synonyms_get.yml index 3494f33466ce4..24ff80948716e 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/20_synonyms_get.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/20_synonyms_get.yml @@ -14,6 +14,13 @@ setup: - synonyms: "test => check" id: "test-id-3" + # This is to ensure that all index shards (write and read) are available. In serverless this can take some time. + - do: + cluster.health: + index: .synonyms-2 + timeout: 2s + wait_for_status: green + ignore: 408 --- "Get synonyms set": diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/30_synonyms_delete.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/30_synonyms_delete.yml index 351ff4e186d8a..62e8fe333ce99 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/30_synonyms_delete.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/30_synonyms_delete.yml @@ -12,6 +12,14 @@ setup: - synonyms: "bye => goodbye" id: "test-id-2" + # This is to ensure that all index shards (write and read) are available. In serverless this can take some time. + - do: + cluster.health: + index: .synonyms-2 + timeout: 2s + wait_for_status: green + ignore: 408 + --- "Delete synonyms set": - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/40_synonyms_sets_get.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/40_synonyms_sets_get.yml index 723c41e163eb8..24f776ba4b210 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/40_synonyms_sets_get.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/40_synonyms_sets_get.yml @@ -9,6 +9,15 @@ setup: synonyms_set: - synonyms: "hello, hi" - synonyms: "goodbye, bye" + + # This is to ensure that all index shards (write and read) are available. In serverless this can take some time. + - do: + cluster.health: + index: .synonyms-2 + timeout: 2s + wait_for_status: green + ignore: 408 + - do: synonyms.put_synonym: id: test-synonyms-1 diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/50_synonym_rule_put.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/50_synonym_rule_put.yml index f3711bb0774ca..02757f711f690 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/50_synonym_rule_put.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/50_synonym_rule_put.yml @@ -14,6 +14,13 @@ setup: - synonyms: "test => check" id: "test-id-3" + # This is to ensure that all index shards (write and read) are available. In serverless this can take some time. + - do: + cluster.health: + index: .synonyms-2 + timeout: 2s + wait_for_status: green + ignore: 408 --- "Update a synonyms rule": diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/60_synonym_rule_get.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/60_synonym_rule_get.yml index 79c666e44661a..9f1aa1d254169 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/60_synonym_rule_get.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/60_synonym_rule_get.yml @@ -14,6 +14,13 @@ setup: - synonyms: "test => check" id: "test-id-3" + # This is to ensure that all index shards (write and read) are available. In serverless this can take some time. + - do: + cluster.health: + index: .synonyms-2 + timeout: 2s + wait_for_status: green + ignore: 408 --- "Get a synonym rule": diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/70_synonym_rule_delete.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/70_synonym_rule_delete.yml index a4853b0b6d414..d2c706decf4fd 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/70_synonym_rule_delete.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/70_synonym_rule_delete.yml @@ -14,6 +14,14 @@ setup: - synonyms: "test => check" id: "test-id-3" + # This is to ensure that all index shards (write and read) are available. In serverless this can take some time. + - do: + cluster.health: + index: .synonyms-2 + timeout: 2s + wait_for_status: green + ignore: 408 + --- "Delete synonym rule": - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/80_synonyms_from_index.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/80_synonyms_from_index.yml index 89ad933370e1c..90c35cc8f4888 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/80_synonyms_from_index.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/80_synonyms_from_index.yml @@ -14,6 +14,14 @@ setup: - synonyms: "bye => goodbye" id: "synonym-rule-2" + # This is to ensure that all index shards (write and read) are available. In serverless this can take some time. + - do: + cluster.health: + index: .synonyms-2 + timeout: 2s + wait_for_status: green + ignore: 408 + # Create an index with synonym_filter that uses that synonyms set - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/90_synonyms_reloading_for_synset.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/90_synonyms_reloading_for_synset.yml index dc94b36222402..4e6bd83f07955 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/90_synonyms_reloading_for_synset.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/90_synonyms_reloading_for_synset.yml @@ -1,5 +1,4 @@ ---- -"Reload analyzers for specific synonym set": +setup: - requires: cluster_features: ["gte_v8.10.0"] reason: Reloading analyzers for specific synonym set is introduced in 8.10.0 @@ -26,6 +25,14 @@ - synonyms: "bye => goodbye" id: "synonym-rule-2" + # This is to ensure that all index shards (write and read) are available. In serverless this can take some time. + - do: + cluster.health: + index: .synonyms-2 + timeout: 2s + wait_for_status: green + ignore: 408 + # Create my_index1 with synonym_filter that uses synonyms_set1 - do: indices.create: @@ -34,7 +41,6 @@ settings: index: number_of_shards: 1 - number_of_replicas: 0 analysis: filter: my_synonym_filter: @@ -68,7 +74,6 @@ settings: index: number_of_shards: 1 - number_of_replicas: 0 analysis: filter: my_synonym_filter: @@ -95,8 +100,12 @@ - '{"index": {"_index": "my_index2", "_id": "2"}}' - '{"my_field": "goodbye"}' +--- +"Reload analyzers for specific synonym set": +# These specific tests can't succeed in BwC, as synonyms auto-expand replicas are 0-all. Replicas can't be associated to +# upgraded nodes, and thus we are not able to guarantee that the shards are not failed. +# This test is skipped for BwC until synonyms index has auto-exapnd replicas set to 0-1. - # An update of synonyms_set1 must trigger auto-reloading of analyzers only for synonyms_set1 - do: synonyms.put_synonym: id: synonyms_set1 @@ -104,12 +113,12 @@ synonyms_set: - synonyms: "hello, salute" - synonyms: "ciao => goodbye" + - match: { result: "updated" } - - match: { reload_analyzers_details._shards.total: 2 } # shard requests are still sent to 2 indices - - match: { reload_analyzers_details._shards.successful: 2 } - - length: { reload_analyzers_details.reload_details: 1 } # reload details contain only a single index - - match: { reload_analyzers_details.reload_details.0.index: "my_index1" } - - match: { reload_analyzers_details.reload_details.0.reloaded_analyzers.0: "my_analyzer1" } + - gt: { reload_analyzers_details._shards.total: 0 } + - gt: { reload_analyzers_details._shards.successful: 0 } + - match: { reload_analyzers_details._shards.failed: 0 } + # Confirm that the index analyzers are reloaded for my_index1 - do: @@ -122,6 +131,23 @@ query: salute - match: { hits.total.value: 1 } +--- +"Check analyzer reloaded and non failed shards for bwc tests": + + - do: + synonyms.put_synonym: + id: synonyms_set1 + body: + synonyms_set: + - synonyms: "hello, salute" + - synonyms: "ciao => goodbye" + - match: { result: "updated" } + - gt: { reload_analyzers_details._shards.total: 0 } + - gt: { reload_analyzers_details._shards.successful: 0 } + - length: { reload_analyzers_details.reload_details: 1 } # reload details contain only a single index + - match: { reload_analyzers_details.reload_details.0.index: "my_index1" } + - match: { reload_analyzers_details.reload_details.0.reloaded_analyzers.0: "my_analyzer1" } + # Confirm that the index analyzers are still the same for my_index2 - do: search: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml index 6eb7a8dcad7aa..616afd3cf67ad 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml @@ -119,11 +119,11 @@ missing dimension on routing path field: type: keyword --- -multi-value routing path field: +multi-value routing path field succeeds: - requires: test_runner_features: close_to - cluster_features: ["gte_v8.13.0"] - reason: _tsid hashing introduced in 8.13 + cluster_features: ["routing.multi_value_routing_path"] + reason: support for multi-value dimensions - do: indices.create: @@ -172,12 +172,7 @@ multi-value routing path field: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:35:54.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.8, "unmapped_field": 40, "tag": [ "one", "three" ] }' - - is_true: errors - - - match: {items.1.index.error.reason: "Error extracting routing: Routing values must be strings but found [START_ARRAY]" } - - match: {items.3.index.error.reason: "Error extracting routing: Routing values must be strings but found [START_ARRAY]" } - - match: {items.4.index.error.reason: "Error extracting routing: Routing values must be strings but found [START_ARRAY]" } - - match: {items.7.index.error.reason: "Error extracting routing: Routing values must be strings but found [START_ARRAY]" } + - is_false: errors - do: search: @@ -195,13 +190,21 @@ multi-value routing path field: avg: field: voltage - - match: {hits.total.value: 4} - - length: {aggregations.tsids.buckets: 2} + - match: {hits.total.value: 8} + - length: {aggregations.tsids.buckets: 4} - - match: {aggregations.tsids.buckets.0.key: "KDODRmbj7vu4rLWvjrJbpUuaET_vOYoRw6ImzKEcF4sEaGKnXSaKfM0" } + - match: {aggregations.tsids.buckets.0.key: "KDODRmbj7vu4rLWvjrJbpUtt0uPSOYoRw_LI4DD7DFEGEJ3NR3eQkMY" } - match: {aggregations.tsids.buckets.0.doc_count: 2 } - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 6.70, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key: "KDODRmbj7vu4rLWvjrJbpUvcUWJEddqA4Seo8jbBBBFxwC0lrefCb6A" } + - match: { aggregations.tsids.buckets.1.key: "KDODRmbj7vu4rLWvjrJbpUtt0uPSddqA4WYKglGPR_C0cJe8QGaiC2c" } - match: {aggregations.tsids.buckets.1.doc_count: 2 } - - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.30, error: 0.01 }} + - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.15, error: 0.01 }} + + - match: { aggregations.tsids.buckets.2.key: "KDODRmbj7vu4rLWvjrJbpUuaET_vOYoRw6ImzKEcF4sEaGKnXSaKfM0" } + - match: {aggregations.tsids.buckets.2.doc_count: 2 } + - close_to: {aggregations.tsids.buckets.2.voltage.value: { value: 6.70, error: 0.01 }} + + - match: { aggregations.tsids.buckets.3.key: "KDODRmbj7vu4rLWvjrJbpUvcUWJEddqA4Seo8jbBBBFxwC0lrefCb6A" } + - match: {aggregations.tsids.buckets.3.doc_count: 2 } + - close_to: {aggregations.tsids.buckets.3.voltage.value: { value: 7.30, error: 0.01 }} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml index ade153d284548..f25601fc2e228 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml @@ -127,7 +127,7 @@ exact match object type: reason: routing_path error message updated in 8.14.0 - do: - catch: '/All fields that match routing_path must be configured with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[dim\] was \[object\]./' + catch: '/All fields that match routing_path must be .*flattened fields.* \[dim\] was \[object\]./' indices.create: index: tsdb_index body: @@ -427,8 +427,8 @@ nested fields: --- "Synthetic source": - requires: - cluster_features: ["gte_v8.10.0"] - reason: Synthetic source shows up in the mapping in 8.10 + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" - do: indices.create: @@ -450,19 +450,13 @@ nested fields: type: long time_series_metric: gauge - - do: - indices.get_mapping: {} - - - match: {tsdb-synthetic.mappings._source.mode: synthetic} - --- -regular source: +stored source is supported: - requires: - cluster_features: ["gte_v8.7.0"] - reason: synthetic source + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" - do: - catch: '/time series indices only support synthetic source/' indices.create: index: tsdb_index body: @@ -473,9 +467,9 @@ regular source: time_series: start_time: 2021-04-28T00:00:00Z end_time: 2021-04-29T00:00:00Z + mapping: + source.mode: stored mappings: - _source: - mode: stored properties: "@timestamp": type: date @@ -486,14 +480,45 @@ regular source: uid: type: keyword time_series_dimension: true + --- -disabled source: +disabled source is not supported: - requires: - cluster_features: ["gte_v8.7.0"] - reason: synthetic source + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" + + - do: + catch: bad_request + indices.create: + index: tsdb_index + body: + settings: + index: + mode: time_series + routing_path: [k8s.pod.uid] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mapping: + source.mode: disabled + mappings: + properties: + "@timestamp": + type: date + k8s: + properties: + pod: + properties: + uid: + type: keyword + time_series_dimension: true + + - match: { error.type: "mapper_parsing_exception" } + - match: { error.root_cause.0.type: "mapper_parsing_exception" } + - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [time_series] index mode" } - do: - catch: '/time series indices only support synthetic source/' + catch: bad_request indices.create: index: tsdb_index body: @@ -506,7 +531,7 @@ disabled source: end_time: 2021-04-29T00:00:00Z mappings: _source: - mode: disabled + enabled: false properties: "@timestamp": type: date @@ -518,6 +543,10 @@ disabled source: type: keyword time_series_dimension: true + - match: { error.type: "mapper_parsing_exception" } + - match: { error.root_cause.0.type: "mapper_parsing_exception" } + - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [time_series] index mode" } + --- source include/exclude: - requires: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml index 973832cf3ca73..beba6f2752a11 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml @@ -65,6 +65,9 @@ setup: --- generates a consistent id: + - requires: + cluster_features: "tsdb.ts_routing_hash_doc_value_parse_byte_ref" + reason: _tsid routing hash doc value parsing has been fixed - do: bulk: refresh: true @@ -152,6 +155,50 @@ generates a consistent id: - match: { hits.hits.8._source.@timestamp: 2021-04-28T18:52:04.467Z } - match: { hits.hits.8._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + - do: + search: + index: id_generation_test + body: + query: + match_all: {} + sort: ["@timestamp", "_ts_routing_hash"] + _source: true + search_after: [ "2021-04-28T18:50:03.142Z", "cn4exQ" ] + docvalue_fields: [_ts_routing_hash] + + - match: {hits.total.value: 9} + + - match: { hits.hits.0._id: cZZNs7B9sSWsyrL5AAABeRnRGTM } + - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:50:04.467Z } + - match: { hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - match: { hits.hits.1._id: cn4excfoxSs_KdA5AAABeRnRYiY } + - match: { hits.hits.1._source.@timestamp: 2021-04-28T18:50:23.142Z } + - match: { hits.hits.1._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + + - match: { hits.hits.2._id: cZZNs7B9sSWsyrL5AAABeRnRZ1M } + - match: { hits.hits.2._source.@timestamp: 2021-04-28T18:50:24.467Z } + - match: { hits.hits.2._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - match: { hits.hits.3._id: cZZNs7B9sSWsyrL5AAABeRnRtXM } + - match: { hits.hits.3._source.@timestamp: 2021-04-28T18:50:44.467Z } + - match: { hits.hits.3._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - match: { hits.hits.4._id: cn4excfoxSs_KdA5AAABeRnR11Y } + - match: { hits.hits.4._source.@timestamp: 2021-04-28T18:50:53.142Z } + - match: { hits.hits.4._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + + - match: { hits.hits.5._id: cn4excfoxSs_KdA5AAABeRnR_mY } + - match: { hits.hits.5._source.@timestamp: 2021-04-28T18:51:03.142Z } + - match: { hits.hits.5._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + + - match: { hits.hits.6._id: cZZNs7B9sSWsyrL5AAABeRnSA5M } + - match: { hits.hits.6._source.@timestamp: 2021-04-28T18:51:04.467Z } + - match: { hits.hits.6._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - match: { hits.hits.7._id: cZZNs7B9sSWsyrL5AAABeRnS7fM } + - match: { hits.hits.7._source.@timestamp: 2021-04-28T18:52:04.467Z } + - match: { hits.hits.7._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } --- index a new document on top of an old one: - do: @@ -380,7 +427,7 @@ delete over _bulk: - match: {items.0.delete.result: deleted} - match: {items.1.delete.result: deleted} - match: {items.2.delete.status: 404} - - match: {items.2.delete.error.reason: "invalid id [not found ++ not found] for index [id_generation_test] in time series mode"} + - match: {items.2.delete.error.reason: '/invalid\ id\ \[not\ found\ \+\+\ not\ found\]\ for\ index\ \[id_generation_test\]\ in\ time.series\ mode/'} --- routing_path matches deep object: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml index c32d3c50b0784..c71555dd073d6 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml @@ -95,7 +95,7 @@ split: reason: tsdb indexing changed in 8.2.0 - do: - catch: /index-split is not supported because the destination index \[test\] is in time series mode/ + catch: /index-split is not supported because the destination index \[test\] is in time.series mode/ indices.split: index: test target: test_split diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml index db718959919da..142d1281ad12b 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml @@ -75,7 +75,7 @@ index with routing: reason: tsdb indexing changed in 8.2.0 - do: - catch: /specifying routing is not supported because the destination index \[test\] is in time series mode/ + catch: /specifying routing is not supported because the destination index \[test\] is in time.series mode/ index: index: test routing: foo @@ -104,7 +104,7 @@ index with routing over _bulk: body: - '{"index": {"routing": "foo"}}' - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' - - match: {items.0.index.error.reason: "specifying routing is not supported because the destination index [test] is in time series mode"} + - match: {items.0.index.error.reason: '/specifying\ routing\ is\ not\ supported\ because\ the\ destination\ index\ \[test\]\ is\ in\ time.series\ mode/'} --- noop update: @@ -120,7 +120,7 @@ noop update: - length: {hits.hits: 1} - do: - catch: /update is not supported because the destination index \[test\] is in time series mode/ + catch: /update is not supported because the destination index \[test\] is in time.series mode/ update: index: test id: "1" @@ -129,14 +129,14 @@ noop update: {} --- -update: +regular update: - requires: cluster_features: ["gte_v8.2.0"] reason: tsdb indexing changed in 8.2.0 # We fail even though the document isn't found. - do: - catch: /update is not supported because the destination index \[test\] is in time series mode/ + catch: /update is not supported because the destination index \[test\] is in time.series mode/ update: index: test id: "1" @@ -165,7 +165,7 @@ update over _bulk: body: - '{"update": {"_id": 1}}' - '{"doc":{"@timestamp": "2021-04-28T18:03:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}}}' - - match: {items.0.update.error.reason: "update is not supported because the destination index [test] is in time series mode"} + - match: {items.0.update.error.reason: '/update\ is\ not\ supported\ because\ the\ destination\ index\ \[test\]\ is\ in\ time.series\ mode/'} --- search with routing: @@ -175,7 +175,7 @@ search with routing: # We fail even though the document isn't found. - do: - catch: /searching with a specified routing is not supported because the destination index \[test\] is in time series mode/ + catch: /searching with a specified routing is not supported because the destination index \[test\] is in time.series mode/ search: index: test routing: rrrr diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/update/100_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/update/100_synthetic_source.yml index f74fde7eb2a24..f4894692b6cad 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/update/100_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/update/100_synthetic_source.yml @@ -1,3 +1,9 @@ +setup: + - requires: + cluster_features: [ "mapper.source.mode_from_index_setting" ] + reason: "Source mode configured through index setting" + +--- keyword: - requires: cluster_features: ["gte_v8.4.0"] @@ -7,9 +13,10 @@ keyword: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: kwd: type: keyword @@ -65,9 +72,10 @@ stored text: indices.create: index: test body: + settings: + index: + mapping.source.mode: synthetic mappings: - _source: - mode: synthetic properties: text: type: text diff --git a/server/build.gradle b/server/build.gradle index c3ac4d9704091..2a27e653b4453 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import org.elasticsearch.gradle.internal.info.BuildParams @@ -16,7 +17,7 @@ apply plugin: 'elasticsearch.internal-test-artifact' publishing { publications { elastic { - artifactId 'elasticsearch' + artifactId = 'elasticsearch' } } } @@ -27,24 +28,24 @@ base { dependencies { - api project(':libs:elasticsearch-core') - api project(':libs:elasticsearch-logging') - api project(':libs:elasticsearch-secure-sm') - api project(':libs:elasticsearch-x-content') - api project(":libs:elasticsearch-geo") - api project(":libs:elasticsearch-lz4") - api project(":libs:elasticsearch-plugin-api") - api project(":libs:elasticsearch-plugin-analysis-api") - api project(':libs:elasticsearch-grok') - api project(":libs:elasticsearch-tdigest") - implementation project(":libs:elasticsearch-simdvec") - - implementation project(':libs:elasticsearch-plugin-classloader') + api project(':libs:core') + api project(':libs:logging') + api project(':libs:secure-sm') + api project(':libs:x-content') + api project(":libs:geo") + api project(":libs:lz4") + api project(":libs:plugin-api") + api project(":libs:plugin-analysis-api") + api project(':libs:grok') + api project(":libs:tdigest") + implementation project(":libs:simdvec") + implementation project(":libs:entitlement") // lucene api "org.apache.lucene:lucene-core:${versions.lucene}" api "org.apache.lucene:lucene-analysis-common:${versions.lucene}" api "org.apache.lucene:lucene-backward-codecs:${versions.lucene}" + api "org.apache.lucene:lucene-facet:${versions.lucene}" api "org.apache.lucene:lucene-grouping:${versions.lucene}" api "org.apache.lucene:lucene-highlighter:${versions.lucene}" api "org.apache.lucene:lucene-join:${versions.lucene}" @@ -56,7 +57,7 @@ dependencies { api "org.apache.lucene:lucene-suggest:${versions.lucene}" // utilities - api project(":libs:elasticsearch-cli") + api project(":libs:cli") implementation 'com.carrotsearch:hppc:0.8.1' // precentil ranks aggregation @@ -67,7 +68,7 @@ dependencies { api "org.apache.logging.log4j:log4j-core:${versions.log4j}" // access to native functions - implementation project(':libs:elasticsearch-native') + implementation project(':libs:native') api "co.elastic.logging:log4j2-ecs-layout:${versions.ecsLogging}" api "co.elastic.logging:ecs-logging-core:${versions.ecsLogging}" @@ -132,7 +133,7 @@ def generatePluginsList = tasks.register("generatePluginsList") { sourceSets.main.output.dir(generatedResourcesDir) sourceSets.main.compiledBy(generateModulesList, generatePluginsList) -if (BuildParams.isSnapshotBuild() == false) { +if (buildParams.isSnapshotBuild() == false) { tasks.named("test").configure { systemProperty 'es.index_mode_feature_flag_registered', 'true' systemProperty 'es.failure_store_feature_flag_enabled', 'true' diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java index 27f8fc915cdd7..2d720c9cfc1b8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; @@ -14,6 +15,7 @@ import org.elasticsearch.action.admin.indices.cache.clear.TransportClearIndicesCacheAction; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; import org.elasticsearch.action.admin.indices.close.TransportCloseIndexAction; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction; import org.elasticsearch.action.admin.indices.flush.FlushRequest; @@ -156,9 +158,11 @@ public void setup() { for (int i = 0; i < numIndices; i++) { indices.add("test" + i); } - for (String index : indices) { - assertAcked(prepareCreate(index).addAlias(new Alias(index + "-alias"))); - } + assertAcked( + indices.stream() + .map(index -> prepareCreate(index).addAlias(new Alias(index + "-alias"))) + .toArray(CreateIndexRequestBuilder[]::new) + ); ensureGreen(); } @@ -555,11 +559,7 @@ public void testUpdateSettings() { } public void testSearchQueryThenFetch() throws Exception { - interceptTransportActions( - SearchTransportService.QUERY_ACTION_NAME, - SearchTransportService.FETCH_ID_ACTION_NAME, - SearchTransportService.FREE_CONTEXT_ACTION_NAME - ); + interceptTransportActions(SearchTransportService.QUERY_ACTION_NAME, SearchTransportService.FETCH_ID_ACTION_NAME); String[] randomIndicesOrAliases = randomIndicesOrAliases(); for (int i = 0; i < randomIndicesOrAliases.length; i++) { @@ -579,16 +579,13 @@ public void testSearchQueryThenFetch() throws Exception { SearchTransportService.QUERY_ACTION_NAME, SearchTransportService.FETCH_ID_ACTION_NAME ); - // free context messages are not necessarily sent, but if they are, check their indices - assertIndicesSubsetOptionalRequests(Arrays.asList(searchRequest.indices()), SearchTransportService.FREE_CONTEXT_ACTION_NAME); } public void testSearchDfsQueryThenFetch() throws Exception { interceptTransportActions( SearchTransportService.DFS_ACTION_NAME, SearchTransportService.QUERY_ID_ACTION_NAME, - SearchTransportService.FETCH_ID_ACTION_NAME, - SearchTransportService.FREE_CONTEXT_ACTION_NAME + SearchTransportService.FETCH_ID_ACTION_NAME ); String[] randomIndicesOrAliases = randomIndicesOrAliases(); @@ -610,8 +607,6 @@ public void testSearchDfsQueryThenFetch() throws Exception { SearchTransportService.QUERY_ID_ACTION_NAME, SearchTransportService.FETCH_ID_ACTION_NAME ); - // free context messages are not necessarily sent, but if they are, check their indices - assertIndicesSubsetOptionalRequests(Arrays.asList(searchRequest.indices()), SearchTransportService.FREE_CONTEXT_ACTION_NAME); } private static void assertSameIndices(IndicesRequest originalRequest, String... actions) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/ListenerActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/ListenerActionIT.java index 535b9b73c9dc2..8b5e014b519c8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/ListenerActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/ListenerActionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/RejectionActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/RejectionActionIT.java index a30d654900c20..15e50eca2c693 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/RejectionActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/RejectionActionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/HotThreadsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/HotThreadsIT.java index 95a5ca9157f49..8c80cee58f46c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/HotThreadsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/HotThreadsIT.java @@ -1,15 +1,16 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin; import org.apache.logging.log4j.Level; import org.apache.lucene.util.Constants; -import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.admin.cluster.node.hotthreads.NodeHotThreads; import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsRequest; import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsResponse; @@ -25,15 +26,14 @@ import org.hamcrest.Matcher; import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.ExecutionException; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; @@ -43,11 +43,10 @@ public class HotThreadsIT extends ESIntegTestCase { - public void testHotThreadsDontFail() throws InterruptedException { + public void testHotThreadsDontFail() throws InterruptedException, ExecutionException { // This test just checks if nothing crashes or gets stuck etc. createIndex("test"); final int iters = scaledRandomIntBetween(2, 20); - final AtomicBoolean hasErrors = new AtomicBoolean(false); for (int i = 0; i < iters; i++) { final NodesHotThreadsRequest request = new NodesHotThreadsRequest( Strings.EMPTY_ARRAY, @@ -66,36 +65,7 @@ public void testHotThreadsDontFail() throws InterruptedException { randomBoolean() ) ); - final CountDownLatch latch = new CountDownLatch(1); - client().execute(TransportNodesHotThreadsAction.TYPE, request, new ActionListener<>() { - @Override - public void onResponse(NodesHotThreadsResponse nodeHotThreads) { - boolean success = false; - try { - assertThat(nodeHotThreads, notNullValue()); - Map nodesMap = nodeHotThreads.getNodesMap(); - assertThat(nodeHotThreads.failures(), empty()); - assertThat(nodesMap.size(), equalTo(cluster().size())); - for (NodeHotThreads ht : nodeHotThreads.getNodes()) { - assertNotNull(ht.getHotThreads()); - } - success = true; - } finally { - if (success == false) { - hasErrors.set(true); - } - latch.countDown(); - } - } - - @Override - public void onFailure(Exception e) { - logger.error("FAILED", e); - hasErrors.set(true); - latch.countDown(); - fail(); - } - }); + final ActionFuture hotThreadsFuture = client().execute(TransportNodesHotThreadsAction.TYPE, request); indexRandom( true, @@ -104,7 +74,7 @@ public void onFailure(Exception e) { prepareIndex("test").setId("3").setSource("field1", "value3") ); ensureSearchable(); - while (latch.getCount() > 0) { + while (hotThreadsFuture.isDone() == false) { assertHitCount( prepareSearch().setQuery(matchAllQuery()) .setPostFilter( @@ -114,8 +84,15 @@ public void onFailure(Exception e) { 3L ); } - safeAwait(latch); - assertThat(hasErrors.get(), is(false)); + assertResponse(hotThreadsFuture, nodeHotThreads -> { + assertThat(nodeHotThreads, notNullValue()); + Map nodesMap = nodeHotThreads.getNodesMap(); + assertThat(nodeHotThreads.failures(), empty()); + assertThat(nodesMap.size(), equalTo(cluster().size())); + for (NodeHotThreads ht : nodeHotThreads.getNodes()) { + assertNotNull(ht.getHotThreads()); + } + }); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java index 5d4a922ec3e11..a5aa39f5feb1e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java index 245e7cede4cf5..e745bd75be510 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.allocation; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionIT.java index 54b1b08806a93..df2917c08b1b6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.allocation; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportDesiredNodesActionsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportDesiredNodesActionsIT.java index 0fb8b450ffaff..27fc7bba2eb07 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportDesiredNodesActionsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportDesiredNodesActionsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.desirednodes; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksIT.java index 69768d6efe3bd..22bba47b0c5f9 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.tasks; @@ -251,11 +252,13 @@ public void testDoNotWaitForCompletion() throws Exception { if (waitForCompletion) { assertFalse(cancelFuture.isDone()); } else { - assertBusy(() -> assertTrue(cancelFuture.isDone())); + cancelFuture.get(); } allowEntireRequest(rootRequest); waitForRootTask(mainTaskFuture, false); - cancelFuture.actionGet(); + if (waitForCompletion) { + cancelFuture.actionGet(); + } ensureBansAndCancellationsConsistency(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskStorageRetryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskStorageRetryIT.java index 603c2cb14bd62..a283c19cfacaf 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskStorageRetryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskStorageRetryIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.tasks; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java index 180bef7ea4098..1e16357a24412 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.tasks; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoIT.java index 395f8e5c67642..25678939cb375 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.remote; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/repositories/RepositoryBlocksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/repositories/RepositoryBlocksIT.java index 47ed06ed4a905..b2b3d3908fd0f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/repositories/RepositoryBlocksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/repositories/RepositoryBlocksIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.repositories; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java index eff71d0caf650..f68d6dca830ea 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateActionDisruptionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateActionDisruptionIT.java index 85dd1337204b2..c182c93966318 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateActionDisruptionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateActionDisruptionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.state; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java index 2385c42526d40..339e1f0fb3c4c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.stats; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRemoteIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRemoteIT.java new file mode 100644 index 0000000000000..6cc9824245247 --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRemoteIT.java @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.admin.cluster.stats; + +import org.elasticsearch.Version; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.test.AbstractMultiClustersTestCase; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; +import org.elasticsearch.test.InternalTestCluster; +import org.junit.Assert; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.equalToIgnoringCase; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.oneOf; + +@ClusterScope(scope = Scope.TEST, numDataNodes = 0) +public class ClusterStatsRemoteIT extends AbstractMultiClustersTestCase { + private static final String REMOTE1 = "cluster-a"; + private static final String REMOTE2 = "cluster-b"; + + private static final String INDEX_NAME = "demo"; + + @Override + protected boolean reuseClusters() { + return false; + } + + @Override + protected Collection remoteClusterAlias() { + return List.of(REMOTE1, REMOTE2); + } + + @Override + protected Map skipUnavailableForRemoteClusters() { + return Map.of(REMOTE1, false, REMOTE2, true); + } + + public void testRemoteClusterStats() throws ExecutionException, InterruptedException { + setupClusters(); + final Client client = client(LOCAL_CLUSTER); + SearchRequest searchRequest = new SearchRequest("*", "*:*"); + searchRequest.allowPartialSearchResults(false); + searchRequest.setCcsMinimizeRoundtrips(randomBoolean()); + searchRequest.source(new SearchSourceBuilder().query(new MatchAllQueryBuilder()).size(10)); + + // do a search + assertResponse(cluster(LOCAL_CLUSTER).client().search(searchRequest), Assert::assertNotNull); + // collect stats without remotes + ClusterStatsResponse response = client.admin().cluster().prepareClusterStats().get(); + assertNotNull(response.getCcsMetrics()); + var remotesUsage = response.getCcsMetrics().getByRemoteCluster(); + assertThat(remotesUsage.size(), equalTo(3)); + assertNull(response.getRemoteClustersStats()); + // collect stats with remotes + response = client.admin().cluster().execute(TransportClusterStatsAction.TYPE, new ClusterStatsRequest(true)).get(); + assertNotNull(response.getCcsMetrics()); + remotesUsage = response.getCcsMetrics().getByRemoteCluster(); + assertThat(remotesUsage.size(), equalTo(3)); + assertNotNull(response.getRemoteClustersStats()); + var remoteStats = response.getRemoteClustersStats(); + assertThat(remoteStats.size(), equalTo(2)); + for (String clusterAlias : remoteClusterAlias()) { + assertThat(remoteStats, hasKey(clusterAlias)); + assertThat(remotesUsage, hasKey(clusterAlias)); + assertThat(remoteStats.get(clusterAlias).status(), equalToIgnoringCase(ClusterHealthStatus.GREEN.name())); + assertThat(remoteStats.get(clusterAlias).indicesCount(), greaterThan(0L)); + assertThat(remoteStats.get(clusterAlias).nodesCount(), greaterThan(0L)); + assertThat(remoteStats.get(clusterAlias).shardsCount(), greaterThan(0L)); + assertThat(remoteStats.get(clusterAlias).heapBytes(), greaterThan(0L)); + assertThat(remoteStats.get(clusterAlias).memBytes(), greaterThan(0L)); + assertThat(remoteStats.get(clusterAlias).indicesBytes(), greaterThan(0L)); + assertThat(remoteStats.get(clusterAlias).versions(), hasItem(Version.CURRENT.toString())); + assertThat(remoteStats.get(clusterAlias).clusterUUID(), not(equalTo(""))); + assertThat(remoteStats.get(clusterAlias).mode(), oneOf("sniff", "proxy")); + } + assertFalse(remoteStats.get(REMOTE1).skipUnavailable()); + assertTrue(remoteStats.get(REMOTE2).skipUnavailable()); + } + + private void setupClusters() { + int numShardsLocal = randomIntBetween(2, 5); + Settings localSettings = indexSettings(numShardsLocal, randomIntBetween(0, 1)).build(); + assertAcked( + client(LOCAL_CLUSTER).admin() + .indices() + .prepareCreate(INDEX_NAME) + .setSettings(localSettings) + .setMapping("@timestamp", "type=date", "f", "type=text") + ); + indexDocs(client(LOCAL_CLUSTER)); + + int numShardsRemote = randomIntBetween(2, 10); + for (String clusterAlias : remoteClusterAlias()) { + final InternalTestCluster remoteCluster = cluster(clusterAlias); + remoteCluster.ensureAtLeastNumDataNodes(randomIntBetween(2, 3)); + assertAcked( + client(clusterAlias).admin() + .indices() + .prepareCreate(INDEX_NAME) + .setSettings(indexSettings(numShardsRemote, randomIntBetween(0, 1))) + .setMapping("@timestamp", "type=date", "f", "type=text") + ); + assertFalse( + client(clusterAlias).admin() + .cluster() + .prepareHealth(TEST_REQUEST_TIMEOUT, INDEX_NAME) + .setWaitForGreenStatus() + .setTimeout(TimeValue.timeValueSeconds(30)) + .get() + .isTimedOut() + ); + indexDocs(client(clusterAlias)); + } + + } + + private void indexDocs(Client client) { + int numDocs = between(5, 20); + for (int i = 0; i < numDocs; i++) { + client.prepareIndex(INDEX_NAME).setSource("f", "v", "@timestamp", randomNonNegativeLong()).get(); + } + client.admin().indices().prepareRefresh(INDEX_NAME).get(); + } + +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/ListTasksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/ListTasksIT.java index 2acb38e9bda1e..4efe013b40baf 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/ListTasksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/ListTasksIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.tasks; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/PendingTasksBlocksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/PendingTasksBlocksIT.java index eaf8948348684..6475e80901ea7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/PendingTasksBlocksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/PendingTasksBlocksIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.tasks; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/alias/ValidateIndicesAliasesRequestIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/alias/ValidateIndicesAliasesRequestIT.java index 7f6bb0239b730..5a49322666768 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/alias/ValidateIndicesAliasesRequestIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/alias/ValidateIndicesAliasesRequestIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.alias; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheBlocksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheBlocksIT.java index e0805148a47e3..85222c43e550f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheBlocksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheBlocksIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.cache.clear; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/AutoCreateSystemIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/AutoCreateSystemIndexIT.java index dc00c36470de2..b10e17b3f4e0f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/AutoCreateSystemIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/AutoCreateSystemIndexIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.create; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CloneIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CloneIndexIT.java index d006192579ead..deb191b4f1ce2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CloneIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CloneIndexIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.create; @@ -11,17 +12,23 @@ import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; +import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.xcontent.XContentType; +import java.util.List; + import static org.elasticsearch.action.admin.indices.create.ShrinkIndexIT.assertNoResizeSourceIndexSettings; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public class CloneIndexIT extends ESIntegTestCase { @@ -108,4 +115,92 @@ public void testCreateCloneIndex() { } + public void testResizeChangeIndexMode() { + prepareCreate("source").setSettings(indexSettings(1, 0)).setMapping("@timestamp", "type=date", "host.name", "type=keyword").get(); + updateIndexSettings(Settings.builder().put("index.blocks.write", true), "source"); + List indexSettings = List.of( + Settings.builder().put("index.mode", "logsdb").build(), + Settings.builder().put("index.mode", "time_series").put("index.routing_path", "host.name").build(), + Settings.builder().put("index.mode", "lookup").build() + ); + for (Settings settings : indexSettings) { + IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> { + indicesAdmin().prepareResizeIndex("source", "target").setResizeType(ResizeType.CLONE).setSettings(settings).get(); + }); + assertThat(error.getMessage(), equalTo("can't change setting [index.mode] during resize")); + } + } + + public void testResizeChangeSyntheticSource() { + prepareCreate("source").setSettings(indexSettings(between(1, 5), 0)) + .setMapping("@timestamp", "type=date", "host.name", "type=keyword") + .get(); + updateIndexSettings(Settings.builder().put("index.blocks.write", true), "source"); + IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> { + indicesAdmin().prepareResizeIndex("source", "target") + .setResizeType(ResizeType.CLONE) + .setSettings(Settings.builder().put("index.mapping.source.mode", "synthetic").putNull("index.blocks.write").build()) + .get(); + }); + assertThat(error.getMessage(), containsString("can't change setting [index.mapping.source.mode] during resize")); + } + + public void testResizeChangeRecoveryUseSyntheticSource() { + prepareCreate("source").setSettings( + indexSettings(between(1, 5), 0).put("index.mode", "logsdb") + .put( + "index.version.created", + IndexVersionUtils.randomVersionBetween( + random(), + IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BACKPORT, + IndexVersion.current() + ) + ) + ).setMapping("@timestamp", "type=date", "host.name", "type=keyword").get(); + updateIndexSettings(Settings.builder().put("index.blocks.write", true), "source"); + IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> { + indicesAdmin().prepareResizeIndex("source", "target") + .setResizeType(ResizeType.CLONE) + .setSettings( + Settings.builder() + .put( + "index.version.created", + IndexVersionUtils.randomVersionBetween( + random(), + IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BACKPORT, + IndexVersion.current() + ) + ) + .put("index.recovery.use_synthetic_source", true) + .put("index.mode", "logsdb") + .putNull("index.blocks.write") + .build() + ) + .get(); + }); + // The index.recovery.use_synthetic_source setting requires either index.mode or index.mapping.source.mode + // to be present in the settings. Since these are all unmodifiable settings with a non-deterministic evaluation + // order, any of them may trigger a failure first. + assertThat( + error.getMessage(), + anyOf( + containsString("can't change setting [index.mode] during resize"), + containsString("can't change setting [index.recovery.use_synthetic_source] during resize") + ) + ); + } + + public void testResizeChangeIndexSorts() { + prepareCreate("source").setSettings(indexSettings(between(1, 5), 0)) + .setMapping("@timestamp", "type=date", "host.name", "type=keyword") + .get(); + updateIndexSettings(Settings.builder().put("index.blocks.write", true), "source"); + ValidationException error = expectThrows(ValidationException.class, () -> { + indicesAdmin().prepareResizeIndex("source", "target") + .setResizeType(ResizeType.CLONE) + .setSettings(Settings.builder().putList("index.sort.field", List.of("@timestamp")).build()) + .get(); + }); + assertThat(error.getMessage(), containsString("can't override index sort when resizing an index")); + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java index 3723a415ab061..f92d1735446b0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.create; @@ -16,6 +17,8 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.IndicesOptions; @@ -154,6 +157,19 @@ public void testEmptyMappings() throws Exception { assertTrue(mappings.sourceAsMap().isEmpty()); } + public void testTwoEmptyEqualMappings() throws Exception { + assertAcked(prepareCreate("test1"), prepareCreate("test2").setMapping(XContentFactory.jsonBuilder().startObject().endObject())); + FieldCapabilitiesRequest fieldCapsReq1 = new FieldCapabilitiesRequest(); + fieldCapsReq1.indices("test1"); + fieldCapsReq1.fields("*"); + FieldCapabilitiesResponse fieldCapsResp1 = internalCluster().coordOnlyNodeClient().fieldCaps(fieldCapsReq1).actionGet(); + FieldCapabilitiesRequest fieldCapsReq2 = new FieldCapabilitiesRequest(); + fieldCapsReq2.indices("test2"); + fieldCapsReq2.fields("*"); + FieldCapabilitiesResponse fieldCapsResp2 = internalCluster().coordOnlyNodeClient().fieldCaps(fieldCapsReq2).actionGet(); + assertEquals(fieldCapsResp1.get(), fieldCapsResp2.get()); + } + public void testInvalidShardCountSettings() throws Exception { int value = randomIntBetween(-10, 0); try { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java index 84a8027418321..112ed994385ef 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.create; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java index 9c794efe3a412..4f6d24b419595 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.create; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java index 41646496c59c4..e1bf5bce6f3ae 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.create; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexBlocksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexBlocksIT.java index 415cfff459a67..c2e164b5d21e7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexBlocksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexBlocksIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.delete; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerIT.java index 6f6db1d2d3d82..5339592478fd1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.diskusage; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/flush/FlushBlocksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/flush/FlushBlocksIT.java index 4e2fade87196f..80c8aaa3381ce 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/flush/FlushBlocksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/flush/FlushBlocksIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.flush; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeBlocksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeBlocksIT.java index b5d8ef0308b91..0f96b5e27bd0e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeBlocksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeBlocksIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.forcemerge; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeIT.java index 22bc37b2fb946..14350de32d23e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.forcemerge; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/get/GetIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/get/GetIndexIT.java index cc2da878d5afb..3945c3a48f0af 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/get/GetIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/get/GetIndexIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.get; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/mapping/put/ValidateMappingRequestPluginIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/mapping/put/ValidateMappingRequestPluginIT.java index e94b0a6e0fb76..b79281c607f3a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/mapping/put/ValidateMappingRequestPluginIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/mapping/put/ValidateMappingRequestPluginIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.mapping.put; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/refresh/RefreshBlocksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/refresh/RefreshBlocksIT.java index 2067038e0fdd2..5c0021306dcb8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/refresh/RefreshBlocksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/refresh/RefreshBlocksIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.refresh; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java index becea454b7d58..1dc5e24639c97 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.rollover; @@ -648,9 +649,11 @@ public void testRolloverWithClosedIndexInAlias() { final String openNonwriteIndex = "open-index-nonwrite"; final String closedIndex = "closed-index-nonwrite"; final String writeIndexPrefix = "write-index-"; - assertAcked(prepareCreate(openNonwriteIndex).addAlias(new Alias(aliasName)).get()); - assertAcked(prepareCreate(closedIndex).addAlias(new Alias(aliasName)).get()); - assertAcked(prepareCreate(writeIndexPrefix + "000001").addAlias(new Alias(aliasName).writeIndex(true)).get()); + assertAcked( + prepareCreate(openNonwriteIndex).addAlias(new Alias(aliasName)), + prepareCreate(closedIndex).addAlias(new Alias(aliasName)), + prepareCreate(writeIndexPrefix + "000001").addAlias(new Alias(aliasName).writeIndex(true)) + ); ensureGreen(); index(closedIndex, null, "{\"foo\": \"bar\"}"); @@ -673,17 +676,18 @@ public void testRolloverWithClosedWriteIndex() throws Exception { final String openNonwriteIndex = "open-index-nonwrite"; final String closedIndex = "closed-index-nonwrite"; final String writeIndexPrefix = "write-index-"; - assertAcked(prepareCreate(openNonwriteIndex).addAlias(new Alias(aliasName)).get()); - assertAcked(prepareCreate(closedIndex).addAlias(new Alias(aliasName)).get()); - assertAcked(prepareCreate(writeIndexPrefix + "000001").addAlias(new Alias(aliasName).writeIndex(true)).get()); + assertAcked( + prepareCreate(openNonwriteIndex).addAlias(new Alias(aliasName)), + prepareCreate(closedIndex).addAlias(new Alias(aliasName)), + prepareCreate(writeIndexPrefix + "000001").addAlias(new Alias(aliasName).writeIndex(true)) + ); ensureGreen(openNonwriteIndex, closedIndex, writeIndexPrefix + "000001"); index(closedIndex, null, "{\"foo\": \"bar\"}"); index(aliasName, null, "{\"foo\": \"bar\"}"); index(aliasName, null, "{\"foo\": \"bar\"}"); refresh(aliasName); - assertAcked(indicesAdmin().prepareClose(closedIndex).get()); - assertAcked(indicesAdmin().prepareClose(writeIndexPrefix + "000001").get()); + assertAcked(indicesAdmin().prepareClose(closedIndex, writeIndexPrefix + "000001").get()); ensureGreen(aliasName); RolloverResponse rolloverResponse = indicesAdmin().prepareRolloverIndex(aliasName) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsBlocksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsBlocksIT.java index 0705e1216af43..609314d17b40d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsBlocksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsBlocksIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.segments; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsWithVectorsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsWithVectorsIT.java index 066b9877ec7ac..a4dc9f0f7a90e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsWithVectorsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsWithVectorsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.segments; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java index e6b042c059f41..b98b4c533e37f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.shards; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsBlocksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsBlocksIT.java index 95b98722a8423..06095552f16a6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsBlocksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsBlocksIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.stats; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkAfterWriteFsyncFailureIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkAfterWriteFsyncFailureIT.java index 0ed585164750a..f02adcdaaa092 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkAfterWriteFsyncFailureIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkAfterWriteFsyncFailureIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java index 38d5719287292..27302d87d8374 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; @@ -103,7 +104,11 @@ public void testBulkWithWriteIndexAndRouting() { // allowing the auto-generated timestamp to externally be set would allow making the index inconsistent with duplicate docs public void testExternallySetAutoGeneratedTimestamp() { IndexRequest indexRequest = new IndexRequest("index1").source(Collections.singletonMap("foo", "baz")); - indexRequest.autoGenerateId(); + if (randomBoolean()) { + indexRequest.autoGenerateId(); + } else { + indexRequest.autoGenerateTimeBasedId(); + } if (randomBoolean()) { indexRequest.id("test"); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessor2IT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessor2IT.java index f5fdb7b50c59c..2ba969c57004b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessor2IT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessor2IT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessor2RetryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessor2RetryIT.java index 3768daaa20ffc..8b8b62da98f97 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessor2RetryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessor2RetryIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java index d8797f3c64575..f6169c373bb14 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorIT.java index dc0ccb803eb52..2c5ee976e7c10 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorRetryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorRetryIT.java index cfdaacfae9cfb..d4ffad33d2314 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorRetryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorRetryIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkRejectionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkRejectionIT.java index f85ed227c8531..47ddd60d55ed0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkRejectionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkRejectionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java index 5251f171150b7..8f897c92c8ab7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/IncrementalBulkIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/IncrementalBulkIT.java new file mode 100644 index 0000000000000..07b3ddf696147 --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/IncrementalBulkIT.java @@ -0,0 +1,651 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.bulk; + +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.core.AbstractRefCounted; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.IndexingPressure; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.ingest.IngestClientIT; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.XContentType; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; +import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.lessThan; + +public class IncrementalBulkIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return List.of(IngestClientIT.ExtendedIngestTestPlugin.class); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal, otherSettings)) + .put(IndexingPressure.SPLIT_BULK_LOW_WATERMARK.getKey(), "512B") + .put(IndexingPressure.SPLIT_BULK_LOW_WATERMARK_SIZE.getKey(), "2048B") + .put(IndexingPressure.SPLIT_BULK_HIGH_WATERMARK.getKey(), "4KB") + .put(IndexingPressure.SPLIT_BULK_HIGH_WATERMARK_SIZE.getKey(), "1024B") + .build(); + } + + public void testSingleBulkRequest() { + String index = "test"; + createIndex(index); + + IncrementalBulkService incrementalBulkService = internalCluster().getInstance(IncrementalBulkService.class); + + IncrementalBulkService.Handler handler = incrementalBulkService.newBulkRequest(); + IndexRequest indexRequest = indexRequest(index); + + PlainActionFuture future = new PlainActionFuture<>(); + AbstractRefCounted refCounted = AbstractRefCounted.of(() -> {}); + handler.lastItems(List.of(indexRequest), refCounted::decRef, future); + + BulkResponse bulkResponse = safeGet(future); + assertNoFailures(bulkResponse); + + refresh(index); + + assertResponse(prepareSearch(index).setQuery(QueryBuilders.matchAllQuery()), searchResponse -> { + assertNoFailures(searchResponse); + assertThat(searchResponse.getHits().getTotalHits().value, equalTo((long) 1)); + }); + + assertFalse(refCounted.hasReferences()); + } + + public void testBufferedResourcesReleasedOnClose() { + String index = "test"; + createIndex(index); + + String nodeName = internalCluster().getRandomNodeName(); + IncrementalBulkService incrementalBulkService = internalCluster().getInstance(IncrementalBulkService.class, nodeName); + IndexingPressure indexingPressure = internalCluster().getInstance(IndexingPressure.class, nodeName); + + IncrementalBulkService.Handler handler = incrementalBulkService.newBulkRequest(); + IndexRequest indexRequest = indexRequest(index); + + AbstractRefCounted refCounted = AbstractRefCounted.of(() -> {}); + handler.addItems(List.of(indexRequest), refCounted::decRef, () -> {}); + + assertTrue(refCounted.hasReferences()); + assertThat(indexingPressure.stats().getCurrentCoordinatingBytes(), greaterThan(0L)); + + handler.close(); + + assertFalse(refCounted.hasReferences()); + assertThat(indexingPressure.stats().getCurrentCoordinatingBytes(), equalTo(0L)); + } + + public void testIndexingPressureRejection() { + String index = "test"; + createIndex(index); + + String nodeName = internalCluster().getRandomNodeName(); + IncrementalBulkService incrementalBulkService = internalCluster().getInstance(IncrementalBulkService.class, nodeName); + IndexingPressure indexingPressure = internalCluster().getInstance(IndexingPressure.class, nodeName); + + try (Releasable r = indexingPressure.markCoordinatingOperationStarted(1, indexingPressure.stats().getMemoryLimit(), true)) { + IncrementalBulkService.Handler handler = incrementalBulkService.newBulkRequest(); + AbstractRefCounted refCounted = AbstractRefCounted.of(() -> {}); + + if (randomBoolean()) { + AtomicBoolean nextPage = new AtomicBoolean(false); + refCounted.incRef(); + handler.addItems(List.of(indexRequest(index)), refCounted::decRef, () -> nextPage.set(true)); + assertTrue(nextPage.get()); + } + + PlainActionFuture future = new PlainActionFuture<>(); + handler.lastItems(List.of(indexRequest(index)), refCounted::decRef, future); + + expectThrows(EsRejectedExecutionException.class, future::actionGet); + assertFalse(refCounted.hasReferences()); + } + } + + public void testIncrementalBulkLowWatermarkBackOff() throws Exception { + String index = "test"; + createIndex(index); + + String nodeName = internalCluster().getRandomNodeName(); + IncrementalBulkService incrementalBulkService = internalCluster().getInstance(IncrementalBulkService.class, nodeName); + IndexingPressure indexingPressure = internalCluster().getInstance(IndexingPressure.class, nodeName); + + IncrementalBulkService.Handler handler = incrementalBulkService.newBulkRequest(); + + AbstractRefCounted refCounted = AbstractRefCounted.of(() -> {}); + AtomicBoolean nextPage = new AtomicBoolean(false); + + IndexRequest indexRequest = indexRequest(index); + long total = indexRequest.ramBytesUsed(); + long lowWaterMarkSplits = indexingPressure.stats().getLowWaterMarkSplits(); + long highWaterMarkSplits = indexingPressure.stats().getHighWaterMarkSplits(); + while (total < 2048) { + refCounted.incRef(); + handler.addItems(List.of(indexRequest), refCounted::decRef, () -> nextPage.set(true)); + assertTrue(nextPage.get()); + nextPage.set(false); + indexRequest = indexRequest(index); + total += indexRequest.ramBytesUsed(); + } + + assertThat(indexingPressure.stats().getCurrentCombinedCoordinatingAndPrimaryBytes(), greaterThan(0L)); + refCounted.incRef(); + handler.addItems(List.of(indexRequest(index)), refCounted::decRef, () -> nextPage.set(true)); + + assertBusy(() -> assertThat(indexingPressure.stats().getCurrentCombinedCoordinatingAndPrimaryBytes(), equalTo(0L))); + assertBusy(() -> assertThat(indexingPressure.stats().getLowWaterMarkSplits(), equalTo(lowWaterMarkSplits + 1))); + assertThat(indexingPressure.stats().getHighWaterMarkSplits(), equalTo(highWaterMarkSplits)); + + PlainActionFuture future = new PlainActionFuture<>(); + handler.lastItems(List.of(indexRequest), refCounted::decRef, future); + + BulkResponse bulkResponse = safeGet(future); + assertNoFailures(bulkResponse); + assertFalse(refCounted.hasReferences()); + } + + public void testIncrementalBulkHighWatermarkBackOff() throws Exception { + String index = "test"; + createIndex(index); + + String nodeName = internalCluster().getRandomNodeName(); + IncrementalBulkService incrementalBulkService = internalCluster().getInstance(IncrementalBulkService.class, nodeName); + IndexingPressure indexingPressure = internalCluster().getInstance(IndexingPressure.class, nodeName); + ThreadPool threadPool = internalCluster().getInstance(ThreadPool.class, nodeName); + long lowWaterMarkSplits = indexingPressure.stats().getLowWaterMarkSplits(); + long highWaterMarkSplits = indexingPressure.stats().getHighWaterMarkSplits(); + + AbstractRefCounted refCounted = AbstractRefCounted.of(() -> {}); + AtomicBoolean nextPage = new AtomicBoolean(false); + + ArrayList handlers = new ArrayList<>(); + for (int i = 0; i < 4; ++i) { + ArrayList> requests = new ArrayList<>(); + add512BRequests(requests, index); + IncrementalBulkService.Handler handler = incrementalBulkService.newBulkRequest(); + handlers.add(handler); + refCounted.incRef(); + handler.addItems(requests, refCounted::decRef, () -> nextPage.set(true)); + assertTrue(nextPage.get()); + nextPage.set(false); + } + + // Test that a request smaller than SPLIT_BULK_HIGH_WATERMARK_SIZE (1KB) is not throttled + ArrayList> requestsNoThrottle = new ArrayList<>(); + add512BRequests(requestsNoThrottle, index); + IncrementalBulkService.Handler handlerNoThrottle = incrementalBulkService.newBulkRequest(); + handlers.add(handlerNoThrottle); + refCounted.incRef(); + handlerNoThrottle.addItems(requestsNoThrottle, refCounted::decRef, () -> nextPage.set(true)); + assertTrue(nextPage.get()); + nextPage.set(false); + assertThat(indexingPressure.stats().getHighWaterMarkSplits(), equalTo(highWaterMarkSplits)); + assertThat(indexingPressure.stats().getLowWaterMarkSplits(), equalTo(lowWaterMarkSplits)); + + ArrayList> requestsThrottle = new ArrayList<>(); + // Test that a request larger than SPLIT_BULK_HIGH_WATERMARK_SIZE (1KB) is throttled + add512BRequests(requestsThrottle, index); + add512BRequests(requestsThrottle, index); + + CountDownLatch finishLatch = new CountDownLatch(1); + blockWritePool(threadPool, finishLatch); + IncrementalBulkService.Handler handlerThrottled = incrementalBulkService.newBulkRequest(); + refCounted.incRef(); + handlerThrottled.addItems(requestsThrottle, refCounted::decRef, () -> nextPage.set(true)); + assertFalse(nextPage.get()); + finishLatch.countDown(); + + handlers.add(handlerThrottled); + + // Wait until we are ready for the next page + assertBusy(() -> assertTrue(nextPage.get())); + assertBusy(() -> assertThat(indexingPressure.stats().getHighWaterMarkSplits(), equalTo(highWaterMarkSplits + 1))); + assertThat(indexingPressure.stats().getLowWaterMarkSplits(), equalTo(lowWaterMarkSplits)); + + for (IncrementalBulkService.Handler h : handlers) { + refCounted.incRef(); + PlainActionFuture future = new PlainActionFuture<>(); + h.lastItems(List.of(indexRequest(index)), refCounted::decRef, future); + BulkResponse bulkResponse = safeGet(future); + assertNoFailures(bulkResponse); + } + + assertBusy(() -> assertThat(indexingPressure.stats().getCurrentCombinedCoordinatingAndPrimaryBytes(), equalTo(0L))); + refCounted.decRef(); + assertFalse(refCounted.hasReferences()); + } + + public void testMultipleBulkPartsWithBackoff() { + ExecutorService executorService = Executors.newFixedThreadPool(1); + + try (Releasable ignored = executorService::shutdown;) { + String index = "test"; + createIndex(index); + + IncrementalBulkService incrementalBulkService = internalCluster().getInstance(IncrementalBulkService.class); + long docs = randomIntBetween(200, 400); + + IncrementalBulkService.Handler handler = incrementalBulkService.newBulkRequest(); + + BulkResponse bulkResponse = executeBulk(docs, index, handler, executorService); + assertNoFailures(bulkResponse); + + refresh(index); + + assertResponse(prepareSearch(index).setQuery(QueryBuilders.matchAllQuery()), searchResponse -> { + assertNoFailures(searchResponse); + assertThat(searchResponse.getHits().getTotalHits().value, equalTo(docs)); + }); + } + } + + public void testGlobalBulkFailure() throws InterruptedException { + ExecutorService executorService = Executors.newFixedThreadPool(1); + CountDownLatch blockingLatch = new CountDownLatch(1); + + try (Releasable ignored = executorService::shutdown; Releasable ignored2 = blockingLatch::countDown) { + String index = "test"; + createIndex(index); + + String randomNodeName = internalCluster().getRandomNodeName(); + IncrementalBulkService incrementalBulkService = internalCluster().getInstance(IncrementalBulkService.class, randomNodeName); + ThreadPool threadPool = internalCluster().getInstance(ThreadPool.class, randomNodeName); + + blockWritePool(threadPool, blockingLatch); + fillWriteQueue(threadPool); + + IncrementalBulkService.Handler handler = incrementalBulkService.newBulkRequest(); + if (randomBoolean()) { + expectThrows( + EsRejectedExecutionException.class, + () -> executeBulk(randomIntBetween(200, 400), index, handler, executorService) + ); + } else { + PlainActionFuture future = new PlainActionFuture<>(); + AbstractRefCounted refCounted = AbstractRefCounted.of(() -> {}); + handler.lastItems(List.of(indexRequest(index)), refCounted::decRef, future); + assertFalse(refCounted.hasReferences()); + expectThrows(EsRejectedExecutionException.class, future::actionGet); + } + } + } + + public void testBulkLevelBulkFailureAfterFirstIncrementalRequest() throws Exception { + ExecutorService executorService = Executors.newFixedThreadPool(1); + + try (Releasable ignored = executorService::shutdown) { + String index = "test"; + createIndex(index); + + String randomNodeName = internalCluster().getRandomNodeName(); + IncrementalBulkService incrementalBulkService = internalCluster().getInstance(IncrementalBulkService.class, randomNodeName); + ThreadPool threadPool = internalCluster().getInstance(ThreadPool.class, randomNodeName); + IncrementalBulkService.Handler handler = incrementalBulkService.newBulkRequest(); + AbstractRefCounted refCounted = AbstractRefCounted.of(() -> {}); + PlainActionFuture future = new PlainActionFuture<>(); + + CountDownLatch blockingLatch1 = new CountDownLatch(1); + + AtomicBoolean nextRequested = new AtomicBoolean(true); + AtomicLong hits = new AtomicLong(0); + try { + blockWritePool(threadPool, blockingLatch1); + while (nextRequested.get()) { + nextRequested.set(false); + refCounted.incRef(); + handler.addItems(List.of(indexRequest(index)), refCounted::decRef, () -> nextRequested.set(true)); + hits.incrementAndGet(); + } + } finally { + blockingLatch1.countDown(); + } + assertBusy(() -> assertTrue(nextRequested.get())); + + CountDownLatch blockingLatch2 = new CountDownLatch(1); + + try { + blockWritePool(threadPool, blockingLatch2); + fillWriteQueue(threadPool); + + handler.lastItems(List.of(indexRequest(index)), refCounted::decRef, future); + } finally { + blockingLatch2.countDown(); + } + + // Should not throw because some succeeded + BulkResponse bulkResponse = safeGet(future); + + assertTrue(bulkResponse.hasFailures()); + BulkItemResponse[] items = bulkResponse.getItems(); + assertThat(Arrays.stream(items).filter(r -> r.isFailed() == false).count(), equalTo(hits.get())); + assertThat(items[items.length - 1].getFailure().getCause(), instanceOf(EsRejectedExecutionException.class)); + + refresh(index); + + assertResponse(prepareSearch(index).setQuery(QueryBuilders.matchAllQuery()), searchResponse -> { + assertNoFailures(searchResponse); + assertThat(searchResponse.getHits().getTotalHits().value, equalTo(hits.get())); + }); + } + } + + public void testShortCircuitShardLevelFailure() throws Exception { + String index = "test"; + createIndex(index, 2, 0); + + String coordinatingOnlyNode = internalCluster().startCoordinatingOnlyNode(Settings.EMPTY); + + AbstractRefCounted refCounted = AbstractRefCounted.of(() -> {}); + IncrementalBulkService incrementalBulkService = internalCluster().getInstance(IncrementalBulkService.class, coordinatingOnlyNode); + IncrementalBulkService.Handler handler = incrementalBulkService.newBulkRequest(); + + AtomicBoolean nextRequested = new AtomicBoolean(true); + AtomicLong hits = new AtomicLong(0); + while (nextRequested.get()) { + nextRequested.set(false); + refCounted.incRef(); + handler.addItems(List.of(indexRequest(index)), refCounted::decRef, () -> nextRequested.set(true)); + hits.incrementAndGet(); + } + + assertBusy(() -> assertTrue(nextRequested.get())); + + String node = findShard(resolveIndex(index), 0); + String secondShardNode = findShard(resolveIndex(index), 1); + IndexingPressure primaryPressure = internalCluster().getInstance(IndexingPressure.class, node); + long memoryLimit = primaryPressure.stats().getMemoryLimit(); + long primaryRejections = primaryPressure.stats().getPrimaryRejections(); + try (Releasable releasable = primaryPressure.markPrimaryOperationStarted(10, memoryLimit, false)) { + while (primaryPressure.stats().getPrimaryRejections() == primaryRejections) { + while (nextRequested.get()) { + nextRequested.set(false); + refCounted.incRef(); + List> requests = new ArrayList<>(); + for (int i = 0; i < 20; ++i) { + requests.add(indexRequest(index)); + } + handler.addItems(requests, refCounted::decRef, () -> nextRequested.set(true)); + } + assertBusy(() -> assertTrue(nextRequested.get())); + } + } + + while (nextRequested.get()) { + nextRequested.set(false); + refCounted.incRef(); + handler.addItems(List.of(indexRequest(index)), refCounted::decRef, () -> nextRequested.set(true)); + } + + assertBusy(() -> assertTrue(nextRequested.get())); + + PlainActionFuture future = new PlainActionFuture<>(); + handler.lastItems(List.of(indexRequest(index)), refCounted::decRef, future); + + BulkResponse bulkResponse = safeGet(future); + assertTrue(bulkResponse.hasFailures()); + for (int i = 0; i < hits.get(); ++i) { + assertFalse(bulkResponse.getItems()[i].isFailed()); + } + + boolean shardsOnDifferentNodes = node.equals(secondShardNode) == false; + for (int i = (int) hits.get(); i < bulkResponse.getItems().length; ++i) { + BulkItemResponse item = bulkResponse.getItems()[i]; + if (item.getResponse() != null && item.getResponse().getShardId().id() == 1 && shardsOnDifferentNodes) { + assertFalse(item.isFailed()); + } else { + assertTrue(item.isFailed()); + assertThat(item.getFailure().getCause().getCause(), instanceOf(EsRejectedExecutionException.class)); + } + } + } + + public void testShortCircuitShardLevelFailureWithIngestNodeHop() throws Exception { + String dataOnlyNode = internalCluster().startDataOnlyNode(); + String index = "test1"; + + // We ensure that the index is assigned to a non-ingest node to ensure that indexing pressure does not reject at the coordinating + // level. + createIndex( + index, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.routing.allocation.require._name", dataOnlyNode) + .build() + ); + + String pipelineId = "pipeline_id"; + BytesReference pipelineSource = BytesReference.bytes( + jsonBuilder().startObject() + .field("description", "my_pipeline") + .startArray("processors") + .startObject() + .startObject("test") + .endObject() + .endObject() + .endArray() + .endObject() + ); + + client().admin().cluster().preparePutPipeline(pipelineId, pipelineSource, XContentType.JSON).get(); + + // By adding an ingest pipeline and sending the request to a coordinating node without the ingest role, we ensure that we are + // testing the serialization of shard level requests over the wire. This is because the transport bulk action will be dispatched to + // a node with the ingest role. + String coordinatingOnlyNode = internalCluster().startCoordinatingOnlyNode(Settings.EMPTY); + + AbstractRefCounted refCounted = AbstractRefCounted.of(() -> {}); + IncrementalBulkService incrementalBulkService = internalCluster().getInstance(IncrementalBulkService.class, coordinatingOnlyNode); + IncrementalBulkService.Handler handler = incrementalBulkService.newBulkRequest(); + + AtomicBoolean nextRequested = new AtomicBoolean(true); + AtomicLong hits = new AtomicLong(0); + while (nextRequested.get()) { + nextRequested.set(false); + refCounted.incRef(); + handler.addItems(List.of(indexRequest(index).setPipeline(pipelineId)), refCounted::decRef, () -> nextRequested.set(true)); + hits.incrementAndGet(); + } + + assertBusy(() -> assertTrue(nextRequested.get())); + + String node = findShard(resolveIndex(index), 0); + assertThat(node, equalTo(dataOnlyNode)); + IndexingPressure primaryPressure = internalCluster().getInstance(IndexingPressure.class, node); + long memoryLimit = primaryPressure.stats().getMemoryLimit(); + try (Releasable releasable = primaryPressure.markPrimaryOperationStarted(10, memoryLimit, false)) { + while (nextRequested.get()) { + nextRequested.set(false); + refCounted.incRef(); + handler.addItems(List.of(indexRequest(index).setPipeline(pipelineId)), refCounted::decRef, () -> nextRequested.set(true)); + } + + assertBusy(() -> assertTrue(nextRequested.get())); + } + + while (nextRequested.get()) { + nextRequested.set(false); + refCounted.incRef(); + handler.addItems(List.of(indexRequest(index).setPipeline(pipelineId)), refCounted::decRef, () -> nextRequested.set(true)); + } + + assertBusy(() -> assertTrue(nextRequested.get())); + + PlainActionFuture future = new PlainActionFuture<>(); + handler.lastItems(List.of(indexRequest(index)), refCounted::decRef, future); + + BulkResponse bulkResponse = safeGet(future); + assertTrue(bulkResponse.hasFailures()); + for (int i = 0; i < hits.get(); ++i) { + assertFalse(bulkResponse.getItems()[i].isFailed()); + } + + for (int i = (int) hits.get(); i < bulkResponse.getItems().length; ++i) { + BulkItemResponse item = bulkResponse.getItems()[i]; + assertTrue(item.isFailed()); + assertThat(item.getFailure().getCause().getCause(), instanceOf(EsRejectedExecutionException.class)); + } + } + + private static void blockWritePool(ThreadPool threadPool, CountDownLatch finishLatch) { + final var threadCount = threadPool.info(ThreadPool.Names.WRITE).getMax(); + final var startBarrier = new CyclicBarrier(threadCount + 1); + final var blockingTask = new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + fail(e); + } + + @Override + protected void doRun() { + safeAwait(startBarrier); + safeAwait(finishLatch); + } + + @Override + public boolean isForceExecution() { + return true; + } + }; + for (int i = 0; i < threadCount; i++) { + threadPool.executor(ThreadPool.Names.WRITE).execute(blockingTask); + } + safeAwait(startBarrier); + } + + private static void fillWriteQueue(ThreadPool threadPool) { + final var queueSize = Math.toIntExact(threadPool.info(ThreadPool.Names.WRITE).getQueueSize().singles()); + final var queueFilled = new AtomicBoolean(false); + final var queueFillingTask = new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + fail(e); + } + + @Override + protected void doRun() { + assertTrue("thread pool not blocked", queueFilled.get()); + } + + @Override + public boolean isForceExecution() { + return true; + } + }; + for (int i = 0; i < queueSize; i++) { + threadPool.executor(ThreadPool.Names.WRITE).execute(queueFillingTask); + } + queueFilled.set(true); + } + + private BulkResponse executeBulk(long docs, String index, IncrementalBulkService.Handler handler, ExecutorService executorService) { + ConcurrentLinkedQueue> queue = new ConcurrentLinkedQueue<>(); + for (int i = 0; i < docs; i++) { + IndexRequest indexRequest = indexRequest(index); + queue.add(indexRequest); + } + + AbstractRefCounted refCounted = AbstractRefCounted.of(() -> {}); + PlainActionFuture future = new PlainActionFuture<>(); + Runnable r = new Runnable() { + + @Override + public void run() { + int toRemove = Math.min(randomIntBetween(5, 10), queue.size()); + ArrayList> docs = new ArrayList<>(); + for (int i = 0; i < toRemove; i++) { + docs.add(queue.poll()); + } + + if (queue.isEmpty()) { + handler.lastItems(docs, refCounted::decRef, future); + } else { + refCounted.incRef(); + handler.addItems(docs, refCounted::decRef, () -> executorService.execute(this)); + } + } + }; + + executorService.execute(r); + + BulkResponse bulkResponse = future.actionGet(); + assertFalse(refCounted.hasReferences()); + return bulkResponse; + } + + private static void add512BRequests(ArrayList> requests, String index) { + long total = 0; + while (total < 512) { + IndexRequest indexRequest = indexRequest(index); + requests.add(indexRequest); + total += indexRequest.ramBytesUsed(); + } + assertThat(total, lessThan(1024L)); + } + + private static IndexRequest indexRequest(String index) { + IndexRequest indexRequest = new IndexRequest(); + indexRequest.index(index); + indexRequest.source(Map.of("field", randomAlphaOfLength(10))); + return indexRequest; + } + + protected static String findShard(Index index, int shardId) { + for (String node : internalCluster().getNodeNames()) { + var indicesService = internalCluster().getInstance(IndicesService.class, node); + IndexService indexService = indicesService.indexService(index); + if (indexService != null) { + IndexShard shard = indexService.getShardOrNull(shardId); + if (shard != null && shard.isActive() && shard.routingEntry().primary()) { + return node; + } + } + } + throw new AssertionError("IndexShard instance not found for shard " + new ShardId(index, shardId)); + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java index 573d929ee30a9..ce99af803d41d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; @@ -14,12 +15,14 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.ingest.SimulateIndexResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.compress.CompressedXContent; @@ -31,6 +34,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.concurrent.TimeUnit; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -56,7 +60,7 @@ public void testMappingValidationIndexExists() { } """; indicesAdmin().create(new CreateIndexRequest(indexName).mapping(mapping)).actionGet(); - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -86,13 +90,191 @@ public void testMappingValidationIndexExists() { assertThat(fields.size(), equalTo(1)); } + @SuppressWarnings("unchecked") + public void testMappingValidationIndexExistsTemplateSubstitutions() throws IOException { + /* + * This test simulates a BulkRequest of two documents into an existing index. Then we make sure the index contains no documents, and + * that the index's mapping in the cluster state has not been updated with the two new field. With the mapping from the template + * that was used to create the index, we would expect the second document to throw an exception because it uses a field that does + * not exist. But we substitute a new version of the component template named "test-component-template" that allows for the new + * field. + */ + String originalComponentTemplateMappingString = """ + { + "_doc":{ + "dynamic":"strict", + "properties":{ + "foo1":{ + "type":"text" + } + } + } + } + """; + CompressedXContent mapping = CompressedXContent.fromJSON(originalComponentTemplateMappingString); + Template template = new Template(Settings.EMPTY, mapping, null); + PutComponentTemplateAction.Request componentTemplateActionRequest = new PutComponentTemplateAction.Request( + "test-component-template" + ); + ComponentTemplate componentTemplate = new ComponentTemplate(template, null, null); + componentTemplateActionRequest.componentTemplate(componentTemplate); + client().execute(PutComponentTemplateAction.INSTANCE, componentTemplateActionRequest).actionGet(); + ComposableIndexTemplate composableIndexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of("my-index-*")) + .componentTemplates(List.of("test-component-template")) + .build(); + final String indexTemplateName = "test-index-template"; + TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request( + indexTemplateName + ); + request.indexTemplate(composableIndexTemplate); + client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); + + String indexName = "my-index-1"; + // First, run before the index is created: + assertMappingsUpdatedFromSubstitutions(indexName, indexTemplateName); + // Now, create the index and make sure the component template substitutions work the same: + indicesAdmin().create(new CreateIndexRequest(indexName)).actionGet(); + assertMappingsUpdatedFromSubstitutions(indexName, indexTemplateName); + // Now make sure nothing was actually changed: + indicesAdmin().refresh(new RefreshRequest(indexName)).actionGet(); + SearchResponse searchResponse = client().search(new SearchRequest(indexName)).actionGet(); + assertThat(searchResponse.getHits().getTotalHits().value, equalTo(0L)); + searchResponse.decRef(); + ClusterStateResponse clusterStateResponse = admin().cluster().state(new ClusterStateRequest(TEST_REQUEST_TIMEOUT)).actionGet(); + Map indexMapping = clusterStateResponse.getState().metadata().index(indexName).mapping().sourceAsMap(); + Map fields = (Map) indexMapping.get("properties"); + assertThat(fields.size(), equalTo(1)); + } + + private void assertMappingsUpdatedFromSubstitutions(String indexName, String indexTemplateName) { + IndexRequest indexRequest1 = new IndexRequest(indexName).source(""" + { + "foo1": "baz" + } + """, XContentType.JSON).id(randomUUID()); + IndexRequest indexRequest2 = new IndexRequest(indexName).source(""" + { + "foo3": "baz" + } + """, XContentType.JSON).id(randomUUID()); + { + // First we use the original component template, and expect a failure in the second document: + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of()); + bulkRequest.add(indexRequest1); + bulkRequest.add(indexRequest2); + BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); + assertThat(response.getItems().length, equalTo(2)); + assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[0].getResponse()).getException()); + assertThat(response.getItems()[1].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertThat( + ((SimulateIndexResponse) response.getItems()[1].getResponse()).getException().getMessage(), + containsString("mapping set to strict, dynamic introduction of") + ); + } + + { + // Now we substitute a "test-component-template" that defines both fields, so we expect no exception: + BulkRequest bulkRequest = new SimulateBulkRequest( + Map.of(), + Map.of( + "test-component-template", + Map.of( + "template", + Map.of( + "mappings", + Map.of( + "dynamic", + "strict", + "properties", + Map.of("foo1", Map.of("type", "text"), "foo3", Map.of("type", "text")) + ) + ) + ) + ), + Map.of(), + Map.of() + ); + bulkRequest.add(indexRequest1); + bulkRequest.add(indexRequest2); + BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); + assertThat(response.getItems().length, equalTo(2)); + assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[0].getResponse()).getException()); + assertThat(response.getItems()[1].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[1].getResponse()).getException()); + } + + { + /* + * Now we substitute a "test-component-template-2" that defines both fields, and an index template that pulls it in, so we + * expect no exception: + */ + BulkRequest bulkRequest = new SimulateBulkRequest( + Map.of(), + Map.of( + "test-component-template-2", + Map.of( + "template", + Map.of( + "mappings", + Map.of( + "dynamic", + "strict", + "properties", + Map.of("foo1", Map.of("type", "text"), "foo3", Map.of("type", "text")) + ) + ) + ) + ), + Map.of( + indexTemplateName, + Map.of("index_patterns", List.of(indexName), "composed_of", List.of("test-component-template-2")) + ), + Map.of() + ); + bulkRequest.add(indexRequest1); + bulkRequest.add(indexRequest2); + BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); + assertThat(response.getItems().length, equalTo(2)); + assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[0].getResponse()).getException()); + assertThat(response.getItems()[1].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[1].getResponse()).getException()); + } + + { + /* + * Now we mapping_addition that defines both fields, so we expect no exception: + */ + BulkRequest bulkRequest = new SimulateBulkRequest( + Map.of(), + Map.of(), + Map.of(), + Map.of( + "_doc", + Map.of("dynamic", "strict", "properties", Map.of("foo1", Map.of("type", "text"), "foo3", Map.of("type", "text"))) + ) + ); + bulkRequest.add(indexRequest1); + bulkRequest.add(indexRequest2); + BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); + assertThat(response.getItems().length, equalTo(2)); + assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[0].getResponse()).getException()); + assertThat(response.getItems()[1].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[1].getResponse()).getException()); + } + } + public void testMappingValidationIndexDoesNotExistsNoTemplate() { /* * This test simulates a BulkRequest of two documents into an index that does not exist. There is no template (other than the * mapping-less "random-index-template" created by the parent class), so we expect no mapping validation failure. */ String indexName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -139,7 +321,7 @@ public void testMappingValidationIndexDoesNotExistsV2Template() throws IOExcepti request.indexTemplate(composableIndexTemplate); client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -171,7 +353,7 @@ public void testMappingValidationIndexDoesNotExistsV1Template() { indicesAdmin().putTemplate( new PutIndexTemplateRequest("test-template").patterns(List.of("my-index-*")).mapping("foo1", "type=integer") ).actionGet(); - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -225,7 +407,7 @@ public void testMappingValidationIndexDoesNotExistsDataStream() throws IOExcepti client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); { // First, try with no @timestamp to make sure we're picking up data-stream-specific templates - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -236,7 +418,8 @@ public void testMappingValidationIndexDoesNotExistsDataStream() throws IOExcepti "foo3": "baz" } """, XContentType.JSON).id(randomUUID())); - BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); + BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest) + .actionGet(5, TimeUnit.SECONDS); assertThat(response.getItems().length, equalTo(2)); assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); assertThat( @@ -251,7 +434,7 @@ public void testMappingValidationIndexDoesNotExistsDataStream() throws IOExcepti } { // Now with @timestamp - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "@timestamp": "2024-08-27", diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/WriteAckDelayIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/WriteAckDelayIT.java index 29a5e491dd3fd..274cf90ec9529 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/WriteAckDelayIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/WriteAckDelayIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/CCSPointInTimeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/CCSPointInTimeIT.java index 26afe0e52bd02..ed92e7704f4ba 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/CCSPointInTimeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/CCSPointInTimeIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/LookupRuntimeFieldIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/LookupRuntimeFieldIT.java index 7eaed125156e0..2aa739eca48e9 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/LookupRuntimeFieldIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/LookupRuntimeFieldIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/PointInTimeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/PointInTimeIT.java index da2dfc50d7fe9..66323e687eefb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/PointInTimeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/PointInTimeIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java index 88d934973fc49..30291eb07f155 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchShardsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchShardsIT.java index 7da015052fe82..48f8bd143e2ff 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchShardsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchShardsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java index 5435389452a51..29e262986e0ca 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; @@ -26,6 +27,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.LongArray; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexSettings; @@ -305,11 +307,15 @@ public void testFinalReduce() throws ExecutionException, InterruptedException { public void testWaitForRefreshIndexValidation() throws Exception { int numberOfShards = randomIntBetween(3, 10); - assertAcked(prepareCreate("test1").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards))); - assertAcked(prepareCreate("test2").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards))); - assertAcked(prepareCreate("test3").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards))); - indicesAdmin().prepareAliases().addAlias("test1", "testAlias").get(); - indicesAdmin().prepareAliases().addAlias(new String[] { "test2", "test3" }, "testFailedAlias").get(); + assertAcked( + prepareCreate("test1").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards)), + prepareCreate("test2").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards)), + prepareCreate("test3").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards)) + ); + assertAcked( + indicesAdmin().prepareAliases().addAlias("test1", "testAlias"), + indicesAdmin().prepareAliases().addAlias(new String[] { "test2", "test3" }, "testFailedAlias") + ); long[] validCheckpoints = new long[numberOfShards]; Arrays.fill(validCheckpoints, SequenceNumbers.UNASSIGNED_SEQ_NO); @@ -374,8 +380,10 @@ public void testShardCountLimit() throws Exception { try { final int numPrimaries1 = randomIntBetween(2, 10); final int numPrimaries2 = randomIntBetween(1, 10); - assertAcked(prepareCreate("test1").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numPrimaries1))); - assertAcked(prepareCreate("test2").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numPrimaries2))); + assertAcked( + prepareCreate("test1").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numPrimaries1)), + prepareCreate("test2").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numPrimaries2)) + ); // no exception prepareSearch("test1").get().decRef(); @@ -668,7 +676,7 @@ public Aggregator subAggregator(String aggregatorName) { } @Override - public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws IOException { + public InternalAggregation[] buildAggregations(LongArray owningBucketOrds) { return new InternalAggregation[] { buildEmptyAggregation() }; } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/support/ActiveShardsObserverIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/support/ActiveShardsObserverIT.java index 023fa54fef9ec..6eda417cfb752 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/support/ActiveShardsObserverIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/support/ActiveShardsObserverIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/support/AutoCreateIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/support/AutoCreateIndexIT.java index e4ef0fa7f2d4f..bcaca0766e536 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/support/AutoCreateIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/support/AutoCreateIndexIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/support/WaitActiveShardCountIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/support/WaitActiveShardCountIT.java index bb970f69ead18..77c68b77da3ce 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/support/WaitActiveShardCountIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/support/WaitActiveShardCountIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/support/master/IndexingMasterFailoverIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/support/master/IndexingMasterFailoverIT.java index 1887e37cbbf47..a15e8409511c1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/support/master/IndexingMasterFailoverIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/support/master/IndexingMasterFailoverIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.master; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/support/master/TransportMasterNodeActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/support/master/TransportMasterNodeActionIT.java index 321c1c84d5cb1..3875e0dd1d739 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/support/master/TransportMasterNodeActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/support/master/TransportMasterNodeActionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.master; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionBypassCircuitBreakerOnReplicaIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionBypassCircuitBreakerOnReplicaIT.java index f9f047232e8c6..e18586117953f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionBypassCircuitBreakerOnReplicaIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionBypassCircuitBreakerOnReplicaIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.replication; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java index 81f26243fb6fb..60cf86031759d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.replication; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java index ca4bde8fd0f72..d16fbf9298552 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.termvectors; @@ -530,8 +531,10 @@ private void checkBrownFoxTermVector(Fields fields, String fieldName, boolean wi public void testDuelWithAndWithoutTermVectors() throws IOException, ExecutionException, InterruptedException { // setup indices String[] indexNames = new String[] { "with_tv", "without_tv" }; - assertAcked(prepareCreate(indexNames[0]).setMapping("field1", "type=text,term_vector=with_positions_offsets,analyzer=keyword")); - assertAcked(prepareCreate(indexNames[1]).setMapping("field1", "type=text,term_vector=no,analyzer=keyword")); + assertAcked( + prepareCreate(indexNames[1]).setMapping("field1", "type=text,term_vector=no,analyzer=keyword"), + prepareCreate(indexNames[0]).setMapping("field1", "type=text,term_vector=with_positions_offsets,analyzer=keyword") + ); ensureGreen(); // index documents with and without term vectors @@ -1073,9 +1076,7 @@ public void testWithKeywordAndNormalizer() throws IOException, ExecutionExceptio "type=text,term_vector=with_positions_offsets,analyzer=my_analyzer", "field2", "type=text,term_vector=with_positions_offsets,analyzer=keyword" - ) - ); - assertAcked( + ), prepareCreate(indexNames[1]).setSettings(builder.build()) .setMapping("field1", "type=keyword,normalizer=my_normalizer", "field2", "type=keyword") ); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/termvectors/MultiTermVectorsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/termvectors/MultiTermVectorsIT.java index 9beffa7f964d9..0017b312c4404 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/termvectors/MultiTermVectorsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/termvectors/MultiTermVectorsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.termvectors; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/aliases/IndexAliasesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/aliases/IndexAliasesIT.java index 91903fd700034..fbda057730847 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/aliases/IndexAliasesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/aliases/IndexAliasesIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aliases; @@ -348,10 +349,8 @@ public void testSearchingFilteringAliasesSingleIndex() throws Exception { } public void testSearchingFilteringAliasesTwoIndices() throws Exception { - logger.info("--> creating index [test1]"); - assertAcked(prepareCreate("test1").setMapping("name", "type=text")); - logger.info("--> creating index [test2]"); - assertAcked(prepareCreate("test2").setMapping("name", "type=text")); + logger.info("--> creating indices [test1, test2]"); + assertAcked(prepareCreate("test1").setMapping("name", "type=text"), prepareCreate("test2").setMapping("name", "type=text")); ensureGreen(); logger.info("--> adding filtering aliases to index [test1]"); @@ -558,8 +557,7 @@ public void testSearchingFilteringAliasesMultipleIndices() throws Exception { public void testDeletingByQueryFilteringAliases() throws Exception { logger.info("--> creating index [test1] and [test2"); - assertAcked(prepareCreate("test1").setMapping("name", "type=text")); - assertAcked(prepareCreate("test2").setMapping("name", "type=text")); + assertAcked(prepareCreate("test1").setMapping("name", "type=text"), prepareCreate("test2").setMapping("name", "type=text")); ensureGreen(); logger.info("--> adding filtering aliases to index [test1]"); @@ -613,8 +611,7 @@ public void testDeletingByQueryFilteringAliases() throws Exception { public void testDeleteAliases() throws Exception { logger.info("--> creating index [test1] and [test2]"); - assertAcked(prepareCreate("test1").setMapping("name", "type=text")); - assertAcked(prepareCreate("test2").setMapping("name", "type=text")); + assertAcked(prepareCreate("test1").setMapping("name", "type=text"), prepareCreate("test2").setMapping("name", "type=text")); ensureGreen(); logger.info("--> adding filtering aliases to index [test1]"); @@ -652,8 +649,7 @@ public void testDeleteAliases() throws Exception { } logger.info("--> creating index [foo_foo] and [bar_bar]"); - assertAcked(prepareCreate("foo_foo")); - assertAcked(prepareCreate("bar_bar")); + assertAcked(prepareCreate("foo_foo"), prepareCreate("bar_bar")); ensureGreen(); logger.info("--> adding [foo] alias to [foo_foo] and [bar_bar]"); @@ -1196,13 +1192,15 @@ public void testAliasesWithBlocks() { } } - public void testAliasActionRemoveIndex() throws InterruptedException, ExecutionException { - assertAcked(prepareCreate("foo_foo")); - assertAcked(prepareCreate("bar_bar")); - assertAliasesVersionIncreases(new String[] { "foo_foo", "bar_bar" }, () -> { - assertAcked(indicesAdmin().prepareAliases().addAlias("foo_foo", "foo")); - assertAcked(indicesAdmin().prepareAliases().addAlias("bar_bar", "foo")); - }); + public void testAliasActionRemoveIndex() { + assertAcked(prepareCreate("foo_foo"), prepareCreate("bar_bar")); + assertAliasesVersionIncreases( + new String[] { "foo_foo", "bar_bar" }, + () -> assertAcked( + indicesAdmin().prepareAliases().addAlias("bar_bar", "foo"), + indicesAdmin().prepareAliases().addAlias("foo_foo", "foo") + ) + ); IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, indicesAdmin().prepareAliases().removeIndex("foo")); assertEquals( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/aliases/NetNewSystemIndexAliasIT.java b/server/src/internalClusterTest/java/org/elasticsearch/aliases/NetNewSystemIndexAliasIT.java index 2e2340294d058..27392fefbf4ca 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/aliases/NetNewSystemIndexAliasIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/aliases/NetNewSystemIndexAliasIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aliases; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/aliases/SystemIndexAliasIT.java b/server/src/internalClusterTest/java/org/elasticsearch/aliases/SystemIndexAliasIT.java index 77eef5e9872bd..fbade0ac2c871 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/aliases/SystemIndexAliasIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/aliases/SystemIndexAliasIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.aliases; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/blocks/SimpleBlocksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/blocks/SimpleBlocksIT.java index 5173577827154..bb4d579f6bed2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/blocks/SimpleBlocksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/blocks/SimpleBlocksIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.blocks; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/broadcast/BroadcastActionsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/broadcast/BroadcastActionsIT.java index c45f980553431..4e7c22f0d8847 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/broadcast/BroadcastActionsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/broadcast/BroadcastActionsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.broadcast; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java index e6bd170cba179..b4dda6f033240 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.bwcompat; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterHealthIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterHealthIT.java index a190ac61bbe18..8c5cb264aafc3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterHealthIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterHealthIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java index f0801d01a70d9..853ac9b877d3f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java index ffa2cd29778ad..5b99300d99f22 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; @@ -152,7 +153,6 @@ public void testClusterStateDiffSerialization() throws Exception { } // Check routing table - assertThat(clusterStateFromDiffs.routingTable().version(), equalTo(clusterState.routingTable().version())); assertThat(clusterStateFromDiffs.routingTable().indicesRouting(), equalTo(clusterState.routingTable().indicesRouting())); // Check cluster blocks diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/DesiredNodesSnapshotsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/DesiredNodesSnapshotsIT.java index 382d7aa8eb647..83407e42a4823 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/DesiredNodesSnapshotsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/DesiredNodesSnapshotsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/DesiredNodesStatusIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/DesiredNodesStatusIT.java index 08dbd800ede5e..52ea5256b8672 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/DesiredNodesStatusIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/DesiredNodesStatusIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java index a3c7f8b77a444..d05e49abd2ae3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/NoMasterNodeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/NoMasterNodeIT.java index 6b104291693e1..13515d34ec65f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/NoMasterNodeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/NoMasterNodeIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/PrevalidateNodeRemovalIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/PrevalidateNodeRemovalIT.java index 6b5bb08f0f247..90e3196d76378 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/PrevalidateNodeRemovalIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/PrevalidateNodeRemovalIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/PrevalidateShardPathIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/PrevalidateShardPathIT.java index c7d1d49f6e451..87943dedc708b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/PrevalidateShardPathIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/PrevalidateShardPathIT.java @@ -1,15 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; import org.apache.lucene.tests.util.LuceneTestCase; -import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplanationUtils; import org.elasticsearch.action.admin.cluster.node.shutdown.NodePrevalidateShardPathResponse; import org.elasticsearch.action.admin.cluster.node.shutdown.PrevalidateShardPathRequest; import org.elasticsearch.action.admin.cluster.node.shutdown.PrevalidateShardPathResponse; @@ -19,18 +19,17 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.junit.annotations.TestIssueLogging; import java.util.HashSet; -import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; /* * We rely on the shard directory being deleted after the relocation. This removal sometimes fails @@ -41,13 +40,6 @@ @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) public class PrevalidateShardPathIT extends ESIntegTestCase { - @TestIssueLogging( - value = "org.elasticsearch.cluster.service.MasterService:DEBUG, " - + "org.elasticsearch.indices.store.IndicesStore:TRACE," - + "org.elasticsearch.indices.cluster.IndicesClusterStateService:DEBUG," - + "org.elasticsearch.indices.IndicesService:TRACE", - issueUrl = "https://github.com/elastic/elasticsearch/issues/104807" - ) public void testCheckShards() throws Exception { internalCluster().startMasterOnlyNode(); String node1 = internalCluster().startDataOnlyNode(); @@ -79,55 +71,22 @@ public void testCheckShards() throws Exception { assertThat(nodeResponse.getShardIds(), equalTo(shardIds)); } // Check that after relocation the source node doesn't have the shard path - String node3 = internalCluster().startDataOnlyNode(); + internalCluster().startDataOnlyNode(); ensureStableCluster(4); - logger.info("Relocating shards from the node {}", node2); + logger.info("--> relocating shards from the node {}", node2); updateIndexSettings(Settings.builder().put("index.routing.allocation.exclude._name", node2), indexName); ensureGreen(indexName); + logger.info("--> green after relocation"); assertBusy(() -> { try { // The excluded node should eventually delete the shards - PrevalidateShardPathRequest req2 = new PrevalidateShardPathRequest(shardIdsToCheck, node2Id); - PrevalidateShardPathResponse resp2 = client().execute(TransportPrevalidateShardPathAction.TYPE, req2).get(); - assertThat(resp2.getNodes().size(), equalTo(1)); - assertThat(resp2.getNodes().get(0).getNode().getId(), equalTo(node2Id)); - assertTrue("There should be no failures in the response", resp.failures().isEmpty()); - Set node2ShardIds = resp2.getNodes().get(0).getShardIds(); - if (node2ShardIds.size() > 0) { - logger.info( - "Relocation source node {} should have no shards after the relocation, but still got {}", - node2Id, - node2ShardIds - ); - List node2Shards = clusterService().state() - .routingTable() - .allShards() - .filter(s -> s.getIndexName().equals(indexName)) - .filter(s -> node2ShardIds.contains(s.shardId())) - .toList(); - logger.info("Found {} shards on the relocation source node {} in the cluster state", node2Shards, node2Id); - for (var node2Shard : node2Shards) { - var explanation = ClusterAllocationExplanationUtils.getClusterAllocationExplanation( - client(), - node2Shard.getIndexName(), - node2Shard.id(), - node2Shard.primary() - ); - logger.info( - "Shard: {} is still located on relocation source node: {}. Allocation explanation: {}", - node2Shard.shardId(), - node2, - Strings.toString(ChunkedToXContent.wrapAsToXContent(explanation), false, true) - ); - } - throw new AssertionError("The relocation source node should have removed the shard(s)"); - } + assertNoShards(shardIdsToCheck, node2Id); } catch (AssertionError e) { // Removal of shards which are no longer allocated to the node is attempted on every cluster state change in IndicesStore. - // If for whatever reason the removal is not triggered (e.g. not enough nodes reported that the shards are active) or it + // If for whatever reason the removal is not successful (e.g. not enough nodes reported that the shards are active) or it // temporarily failed to clean up the shard folder, we need to trigger another cluster state change for this removal to // finally succeed. - logger.info("Triggering an extra cluster state update"); + logger.info("--> Triggering an extra cluster state update: {}", e.getMessage()); updateIndexSettings( Settings.builder().put("index.routing.allocation.exclude.name", "non-existent" + randomAlphaOfLength(5)), indexName @@ -136,4 +95,24 @@ public void testCheckShards() throws Exception { } }, 30, TimeUnit.SECONDS); } + + private void assertNoShards(Set shards, String nodeId) throws Exception { + assertBusy(() -> { + PrevalidateShardPathRequest req = new PrevalidateShardPathRequest(shards, nodeId); + PrevalidateShardPathResponse resp = client().execute(TransportPrevalidateShardPathAction.TYPE, req).get(); + assertThat(resp.getNodes().size(), equalTo(1)); + assertThat(resp.getNodes().get(0).getNode().getId(), equalTo(nodeId)); + assertTrue("There should be no failures in the response", resp.failures().isEmpty()); + Set node2ShardIds = resp.getNodes().get(0).getShardIds(); + assertThat( + Strings.format( + "Relocation source node [%s] should have no shards after the relocation, but still got %s", + nodeId, + node2ShardIds + ), + node2ShardIds, + is(empty()) + ); + }); + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java index 1259650d37791..def125e5c99c3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleDataNodesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleDataNodesIT.java index 58daca22303cf..1ded83275df1a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleDataNodesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleDataNodesIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java index 8cdc49d3b12d5..09f2c04e249e3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/UpdateSettingsValidationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/UpdateSettingsValidationIT.java index 05b58ea2f8808..a59f03aa7898d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/UpdateSettingsValidationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/UpdateSettingsValidationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/AwarenessAllocationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/AwarenessAllocationIT.java index 36d903205f05c..f911569a3cfe9 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/AwarenessAllocationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/AwarenessAllocationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.allocation; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java index da585d1bb67d4..3a1f34175962b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.allocation; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/FilteringAllocationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/FilteringAllocationIT.java index abce3ce30fbad..1294a3de848dc 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/FilteringAllocationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/FilteringAllocationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.allocation; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/SimpleAllocationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/SimpleAllocationIT.java index 6a3d2f2fe5210..3440638fafba0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/SimpleAllocationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/SimpleAllocationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.allocation; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/CoordinationDiagnosticsServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/CoordinationDiagnosticsServiceIT.java index c3c838c286b12..97490771d5b16 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/CoordinationDiagnosticsServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/CoordinationDiagnosticsServiceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/InitialClusterStateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/InitialClusterStateIT.java index eebd059ed13b5..c921d068bb17e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/InitialClusterStateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/InitialClusterStateIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RareClusterStateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RareClusterStateIT.java index bd12f570e136f..d9429cdb27d73 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RareClusterStateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RareClusterStateIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RemoveCustomsCommandIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RemoveCustomsCommandIT.java index a9948367f2780..f47634baaf5cd 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RemoveCustomsCommandIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RemoveCustomsCommandIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RemoveIndexSettingsCommandIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RemoveIndexSettingsCommandIT.java index a5e445270ccc4..ffe25ffd7eeee 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RemoveIndexSettingsCommandIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RemoveIndexSettingsCommandIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RemoveSettingsCommandIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RemoveSettingsCommandIT.java index 527d8b0a62fe4..d7ed6bb47b98b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RemoveSettingsCommandIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RemoveSettingsCommandIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RestHandlerNodesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RestHandlerNodesIT.java index 5a7f4609a7d0f..b5322bada1391 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RestHandlerNodesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RestHandlerNodesIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java index 2c1ca5866fa46..57a1cf7f1685a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/VotingConfigurationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/VotingConfigurationIT.java index 6e21c3622ec45..fd03591ebb1c5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/VotingConfigurationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/VotingConfigurationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/ZenDiscoveryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/ZenDiscoveryIT.java index ea127a7352914..55c9112aad1a8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/ZenDiscoveryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/ZenDiscoveryIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/AutoExpandReplicasIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/AutoExpandReplicasIT.java index ce69a088a1739..b5c91a59021b4 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/AutoExpandReplicasIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/AutoExpandReplicasIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsServiceIT.java index 6e928a42e142b..c1e68040e075b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsServiceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; @@ -27,6 +28,7 @@ import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; import static org.hamcrest.Matchers.equalTo; @@ -41,45 +43,58 @@ public void testThatNonDynamicSettingChangesTakeEffect() throws Exception { MetadataUpdateSettingsService metadataUpdateSettingsService = internalCluster().getCurrentMasterNodeInstance( MetadataUpdateSettingsService.class ); - UpdateSettingsClusterStateUpdateRequest request = new UpdateSettingsClusterStateUpdateRequest().ackTimeout(TimeValue.ZERO); - List indices = new ArrayList<>(); + List indicesList = new ArrayList<>(); for (IndicesService indicesService : internalCluster().getInstances(IndicesService.class)) { for (IndexService indexService : indicesService) { - indices.add(indexService.index()); + indicesList.add(indexService.index()); } } - request.indices(indices.toArray(Index.EMPTY_ARRAY)); - request.settings(Settings.builder().put("index.codec", "FastDecompressionCompressingStoredFieldsData").build()); + final var indices = indicesList.toArray(Index.EMPTY_ARRAY); + + final Function requestFactory = + onStaticSetting -> new UpdateSettingsClusterStateUpdateRequest( + TEST_REQUEST_TIMEOUT, + TimeValue.ZERO, + Settings.builder().put("index.codec", "FastDecompressionCompressingStoredFieldsData").build(), + UpdateSettingsClusterStateUpdateRequest.OnExisting.OVERWRITE, + onStaticSetting, + indices + ); // First make sure it fails if reopenShards is not set on the request: AtomicBoolean expectedFailureOccurred = new AtomicBoolean(false); - metadataUpdateSettingsService.updateSettings(request, new ActionListener<>() { - @Override - public void onResponse(AcknowledgedResponse acknowledgedResponse) { - fail("Should have failed updating a non-dynamic setting without reopenShards set to true"); - } + metadataUpdateSettingsService.updateSettings( + requestFactory.apply(UpdateSettingsClusterStateUpdateRequest.OnStaticSetting.REJECT), + new ActionListener<>() { + @Override + public void onResponse(AcknowledgedResponse acknowledgedResponse) { + fail("Should have failed updating a non-dynamic setting without reopenShards set to true"); + } - @Override - public void onFailure(Exception e) { - expectedFailureOccurred.set(true); + @Override + public void onFailure(Exception e) { + expectedFailureOccurred.set(true); + } } - }); + ); assertBusy(() -> assertThat(expectedFailureOccurred.get(), equalTo(true))); // Now we set reopenShards and expect it to work: - request.reopenShards(true); AtomicBoolean success = new AtomicBoolean(false); - metadataUpdateSettingsService.updateSettings(request, new ActionListener<>() { - @Override - public void onResponse(AcknowledgedResponse acknowledgedResponse) { - success.set(true); - } + metadataUpdateSettingsService.updateSettings( + requestFactory.apply(UpdateSettingsClusterStateUpdateRequest.OnStaticSetting.REOPEN_INDICES), + new ActionListener<>() { + @Override + public void onResponse(AcknowledgedResponse acknowledgedResponse) { + success.set(true); + } - @Override - public void onFailure(Exception e) { - fail(e); + @Override + public void onFailure(Exception e) { + fail(e); + } } - }); + ); assertBusy(() -> assertThat(success.get(), equalTo(true))); // Now we look into the IndexShard objects to make sure that the code was actually updated (vs just the setting): @@ -109,16 +124,23 @@ public void testThatNonDynamicSettingChangesDoNotUnncessesarilyCauseReopens() th MetadataUpdateSettingsService metadataUpdateSettingsService = internalCluster().getCurrentMasterNodeInstance( MetadataUpdateSettingsService.class ); - UpdateSettingsClusterStateUpdateRequest request = new UpdateSettingsClusterStateUpdateRequest().ackTimeout(TimeValue.ZERO); - List indices = new ArrayList<>(); + List indicesList = new ArrayList<>(); for (IndicesService indicesService : internalCluster().getInstances(IndicesService.class)) { for (IndexService indexService : indicesService) { - indices.add(indexService.index()); + indicesList.add(indexService.index()); } } - request.indices(indices.toArray(Index.EMPTY_ARRAY)); - request.settings(Settings.builder().put("index.codec", "FastDecompressionCompressingStoredFieldsData").build()); - request.reopenShards(true); + final var indices = indicesList.toArray(Index.EMPTY_ARRAY); + + final Function requestFactory = + settings -> new UpdateSettingsClusterStateUpdateRequest( + TEST_REQUEST_TIMEOUT, + TimeValue.ZERO, + settings.build(), + UpdateSettingsClusterStateUpdateRequest.OnExisting.OVERWRITE, + UpdateSettingsClusterStateUpdateRequest.OnStaticSetting.REOPEN_INDICES, + indices + ); ClusterService clusterService = internalCluster().getInstance(ClusterService.class); AtomicBoolean shardsUnassigned = new AtomicBoolean(false); @@ -141,47 +163,49 @@ public void testThatNonDynamicSettingChangesDoNotUnncessesarilyCauseReopens() th AtomicBoolean success = new AtomicBoolean(false); // Make the first request, just to set things up: - metadataUpdateSettingsService.updateSettings(request, new ActionListener<>() { - @Override - public void onResponse(AcknowledgedResponse acknowledgedResponse) { - success.set(true); - } + metadataUpdateSettingsService.updateSettings( + requestFactory.apply(Settings.builder().put("index.codec", "FastDecompressionCompressingStoredFieldsData")), + new ActionListener<>() { + @Override + public void onResponse(AcknowledgedResponse acknowledgedResponse) { + success.set(true); + } - @Override - public void onFailure(Exception e) { - fail(e); + @Override + public void onFailure(Exception e) { + fail(e); + } } - }); + ); assertBusy(() -> assertThat(success.get(), equalTo(true))); assertBusy(() -> assertThat(expectedSettingsChangeInClusterState.get(), equalTo(true))); assertThat(shardsUnassigned.get(), equalTo(true)); assertBusy(() -> assertThat(hasUnassignedShards(clusterService.state(), indexName), equalTo(false))); - // Same request, except now we'll also set the dynamic "index.max_result_window" setting: - request.settings( - Settings.builder() - .put("index.codec", "FastDecompressionCompressingStoredFieldsData") - .put("index.max_result_window", "1500") - .build() - ); success.set(false); expectedSettingsChangeInClusterState.set(false); shardsUnassigned.set(false); expectedSetting.set("index.max_result_window"); expectedSettingValue.set("1500"); // Making this request ought to add this new setting but not unassign the shards: - metadataUpdateSettingsService.updateSettings(request, new ActionListener<>() { - @Override - public void onResponse(AcknowledgedResponse acknowledgedResponse) { - success.set(true); - } + metadataUpdateSettingsService.updateSettings( + // Same request, except now we'll also set the dynamic "index.max_result_window" setting: + requestFactory.apply( + Settings.builder().put("index.codec", "FastDecompressionCompressingStoredFieldsData").put("index.max_result_window", "1500") + ), + new ActionListener<>() { + @Override + public void onResponse(AcknowledgedResponse acknowledgedResponse) { + success.set(true); + } - @Override - public void onFailure(Exception e) { - fail(e); + @Override + public void onFailure(Exception e) { + fail(e); + } } - }); + ); assertBusy(() -> assertThat(success.get(), equalTo(true))); assertBusy(() -> assertThat(expectedSettingsChangeInClusterState.get(), equalTo(true))); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java index 8bbdaf7388699..5e51c83d54795 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/AllocationIdIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/AllocationIdIT.java index 69923c787a054..17d2ce0c43be4 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/AllocationIdIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/AllocationIdIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.routing; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java index 8a2f5d749ff21..77734cc68f40e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.routing; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java index 9a13470eea255..6688bd95db842 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.routing; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/RemoveReplicaPriorityIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/RemoveReplicaPriorityIT.java index de61c6cf566c2..1632f86f34d48 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/RemoveReplicaPriorityIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/RemoveReplicaPriorityIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.routing; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/ShardRoutingRoleIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/ShardRoutingRoleIT.java index 556836736a9f8..56a84fe4d48d0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/ShardRoutingRoleIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/ShardRoutingRoleIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.routing; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/AllocationFailuresResetIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/AllocationFailuresResetIT.java index 6b97a8b6f3ad0..d8834dede2db3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/AllocationFailuresResetIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/AllocationFailuresResetIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.routing.allocation; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorIT.java index 5509f46786a80..10378b4d61d29 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.routing.allocation; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java index 2f4b3588cf566..f893d415eb0d2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.routing.allocation; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDeciderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDeciderIT.java index 2490eade46d31..4226fdaba941b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDeciderIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDeciderIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.routing.allocation.decider; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java index 106fd9530c3a5..ac1c44de6dcf6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.routing.allocation.decider; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java index fd7e9f8fb3572..aee29b2564d92 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.routing.allocation.decider; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/UpdateShardAllocationSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/UpdateShardAllocationSettingsIT.java index be530f0bd4cb4..7fb763b5b91a3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/UpdateShardAllocationSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/UpdateShardAllocationSettingsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.routing.allocation.decider; @@ -38,8 +39,10 @@ public void testEnableRebalance() { // we test with 2 shards since otherwise it's pretty fragile if there are difference in the num or shards such that // all shards are relocated to the second node which is not what we want here. It's solely a test for the settings to take effect final int numShards = 2; - assertAcked(prepareCreate("test").setSettings(indexSettings(numShards, 0))); - assertAcked(prepareCreate("test_1").setSettings(indexSettings(numShards, 0))); + assertAcked( + prepareCreate("test").setSettings(indexSettings(numShards, 0)), + prepareCreate("test_1").setSettings(indexSettings(numShards, 0)) + ); ensureGreen(); assertAllShardsOnNodes("test", firstNode); assertAllShardsOnNodes("test_1", firstNode); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceIT.java index cab7b7df00fe9..d4294ac759433 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.routing.allocation.shards; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/service/ClusterServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/service/ClusterServiceIT.java index fde465346d4be..af35dc1a6731c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/service/ClusterServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/service/ClusterServiceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.service; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/settings/ClusterSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/settings/ClusterSettingsIT.java index a9767cce318d4..3e0abec9003ca 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/settings/ClusterSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/settings/ClusterSettingsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.settings; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/settings/ClusterSettingsUpdateWithFaultyMasterIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/settings/ClusterSettingsUpdateWithFaultyMasterIT.java index 99d431747ec7a..a53c1a773c5aa 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/settings/ClusterSettingsUpdateWithFaultyMasterIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/settings/ClusterSettingsUpdateWithFaultyMasterIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.settings; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java index 85aa7d6206a5e..c5907bc64f579 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.settings; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/shards/ClusterSearchShardsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/shards/ClusterSearchShardsIT.java index 7e9406dfcf09a..99d7b28536f7a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/shards/ClusterSearchShardsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/shards/ClusterSearchShardsIT.java @@ -1,13 +1,13 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.shards; -import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsGroup; import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest; import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse; @@ -144,17 +144,13 @@ public void testClusterSearchShardsWithBlocks() { enableIndexBlock("test-blocks", SETTING_BLOCKS_METADATA); assertBlocked( null, - asInstanceOf( + safeAwaitAndUnwrapFailure( ClusterBlockException.class, - ExceptionsHelper.unwrapCause( - safeAwaitFailure( - ClusterSearchShardsResponse.class, - l -> client().execute( - TransportClusterSearchShardsAction.TYPE, - new ClusterSearchShardsRequest(TEST_REQUEST_TIMEOUT, "test-blocks"), - l - ) - ) + ClusterSearchShardsResponse.class, + l -> client().execute( + TransportClusterSearchShardsAction.TYPE, + new ClusterSearchShardsRequest(TEST_REQUEST_TIMEOUT, "test-blocks"), + l ) ) ); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/shards/ClusterShardLimitIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/shards/ClusterShardLimitIT.java index 7a66cb3abb7cd..41b62b1101ac3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/shards/ClusterShardLimitIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/shards/ClusterShardLimitIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.shards; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/common/network/ThreadWatchdogIT.java b/server/src/internalClusterTest/java/org/elasticsearch/common/network/ThreadWatchdogIT.java index 2d17e26fd2959..f2441e43de8d8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/common/network/ThreadWatchdogIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/common/network/ThreadWatchdogIT.java @@ -1,14 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.network; -import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.Level; import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.client.Request; @@ -22,7 +23,6 @@ import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.common.util.concurrent.RunOnce; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; @@ -42,7 +42,6 @@ import java.io.IOException; import java.util.Collection; import java.util.List; -import java.util.concurrent.CountDownLatch; import java.util.function.Predicate; import java.util.function.Supplier; @@ -102,26 +101,24 @@ public void handleRequest(RestRequest request, RestChannel channel, NodeClient c } private static void blockAndWaitForWatchdogLogs() { - final var threadName = Thread.currentThread().getName(); - final var logsSeenLatch = new CountDownLatch(2); - final var warningSeen = new RunOnce(logsSeenLatch::countDown); - final var threadDumpSeen = new RunOnce(logsSeenLatch::countDown); - MockLog.assertThatLogger(() -> safeAwait(logsSeenLatch), ThreadWatchdog.class, new MockLog.LoggingExpectation() { - @Override - public void match(LogEvent event) { - final var formattedMessage = event.getMessage().getFormattedMessage(); - if (formattedMessage.contains("the following threads are active but did not make progress in the preceding [100ms]:") - && formattedMessage.contains(threadName)) { - warningSeen.run(); - } - if (formattedMessage.contains("hot threads dump due to active threads not making progress")) { - threadDumpSeen.run(); - } - } - - @Override - public void assertMatched() {} - }); + MockLog.awaitLogger( + () -> {}, + ThreadWatchdog.class, + new MockLog.SeenEventExpectation( + "warning", + ThreadWatchdog.class.getCanonicalName(), + Level.WARN, + "*the following threads are active but did not make progress in the preceding [100ms]:*" + + Thread.currentThread().getName() + + "*" + ), + new MockLog.SeenEventExpectation( + "thread dump", + ThreadWatchdog.class.getCanonicalName(), + Level.WARN, + "*hot threads dump due to active threads not making progress*" + ) + ); } public void testThreadWatchdogHttpLogging() throws IOException { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/common/settings/ConsistentSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/common/settings/ConsistentSettingsIT.java index 50fd72c851e9f..baf15801e295f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/common/settings/ConsistentSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/common/settings/ConsistentSettingsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.common.settings; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/ClusterDisruptionCleanSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/ClusterDisruptionCleanSettingsIT.java index e8234fb09512b..5e19ed8ab5e56 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/discovery/ClusterDisruptionCleanSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/discovery/ClusterDisruptionCleanSettingsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/ClusterDisruptionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/ClusterDisruptionIT.java index 6ae7e0f7e84e5..bfeee6332f075 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/discovery/ClusterDisruptionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/discovery/ClusterDisruptionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java index b512f369c76d5..328da020ce382 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/DiskDisruptionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/DiskDisruptionIT.java index 08b3c49b37acf..ceb96b5d68600 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/discovery/DiskDisruptionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/discovery/DiskDisruptionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/MasterDisruptionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/MasterDisruptionIT.java index 3e7e4d47e7a23..bf81200509691 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/discovery/MasterDisruptionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/discovery/MasterDisruptionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery; @@ -20,21 +21,15 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.disruption.BlockMasterServiceOnMaster; -import org.elasticsearch.test.disruption.IntermittentLongGCDisruption; import org.elasticsearch.test.disruption.NetworkDisruption; import org.elasticsearch.test.disruption.NetworkDisruption.TwoPartitions; import org.elasticsearch.test.disruption.ServiceDisruptionScheme; -import org.elasticsearch.test.disruption.SingleNodeDisruption; import org.elasticsearch.xcontent.XContentType; -import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.not; /** * Tests relating to the loss of the master. @@ -42,44 +37,6 @@ @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) public class MasterDisruptionIT extends AbstractDisruptionTestCase { - /** - * Test that cluster recovers from a long GC on master that causes other nodes to elect a new one - */ - public void testMasterNodeGCs() throws Exception { - List nodes = startCluster(3); - // NOTE: this assume must happen after starting the cluster, so that cleanup will have something to cleanup. - assumeFalse("jdk20 removed thread suspend/resume", Runtime.version().feature() >= 20); - - String oldMasterNode = internalCluster().getMasterName(); - // a very long GC, but it's OK as we remove the disruption when it has had an effect - SingleNodeDisruption masterNodeDisruption = new IntermittentLongGCDisruption(random(), oldMasterNode, 100, 200, 30000, 60000); - internalCluster().setDisruptionScheme(masterNodeDisruption); - masterNodeDisruption.startDisrupting(); - - Set oldNonMasterNodesSet = new HashSet<>(nodes); - oldNonMasterNodesSet.remove(oldMasterNode); - - List oldNonMasterNodes = new ArrayList<>(oldNonMasterNodesSet); - - logger.info("waiting for nodes to de-elect master [{}]", oldMasterNode); - for (String node : oldNonMasterNodesSet) { - assertDifferentMaster(node, oldMasterNode); - } - - logger.info("waiting for nodes to elect a new master"); - ensureStableCluster(2, oldNonMasterNodes.get(0)); - - // restore GC - masterNodeDisruption.stopDisrupting(); - final TimeValue waitTime = new TimeValue(DISRUPTION_HEALING_OVERHEAD.millis() + masterNodeDisruption.expectedTimeToHeal().millis()); - ensureStableCluster(3, waitTime, false, oldNonMasterNodes.get(0)); - - // make sure all nodes agree on master - String newMaster = internalCluster().getMasterName(); - assertThat(newMaster, not(equalTo(oldMasterNode))); - assertMaster(newMaster, nodes); - } - /** * This test isolates the master from rest of the cluster, waits for a new master to be elected, restores the partition * and verifies that all node agree on the new cluster state diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/SettingsBasedSeedHostsProviderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/SettingsBasedSeedHostsProviderIT.java index fa40215c4ad26..4eae2224cee92 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/discovery/SettingsBasedSeedHostsProviderIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/discovery/SettingsBasedSeedHostsProviderIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/SnapshotDisruptionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/SnapshotDisruptionIT.java index 6a8806ca26526..5bb4d98308289 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/discovery/SnapshotDisruptionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/discovery/SnapshotDisruptionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/StableMasterDisruptionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/StableMasterDisruptionIT.java index 601266b50d237..48db23635220c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/discovery/StableMasterDisruptionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/discovery/StableMasterDisruptionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery; @@ -13,33 +14,26 @@ import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.coordination.CoordinationDiagnosticsService; import org.elasticsearch.cluster.coordination.Coordinator; import org.elasticsearch.cluster.coordination.FollowersChecker; import org.elasticsearch.cluster.coordination.LeaderChecker; import org.elasticsearch.cluster.coordination.MasterHistoryService; -import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.core.Tuple; import org.elasticsearch.health.GetHealthAction; import org.elasticsearch.health.HealthStatus; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.disruption.LongGCDisruption; import org.elasticsearch.test.disruption.NetworkDisruption; import org.elasticsearch.test.disruption.NetworkDisruption.NetworkLinkDisruptionType; import org.elasticsearch.test.disruption.NetworkDisruption.TwoPartitions; -import org.elasticsearch.test.disruption.SingleNodeDisruption; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.ToXContent; @@ -49,17 +43,12 @@ import org.junit.Before; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; -import java.util.Objects; import java.util.Set; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import static java.util.Collections.singleton; @@ -226,266 +215,6 @@ private void testFollowerCheckerAfterMasterReelection(NetworkLinkDisruptionType ensureStableCluster(3); } - /** - * Tests that emulates a frozen elected master node that unfreezes and pushes its cluster state to other nodes that already are - * following another elected master node. These nodes should reject this cluster state and prevent them from following the stale master. - */ - public void testStaleMasterNotHijackingMajority() throws Exception { - assumeFalse("jdk20 removed thread suspend/resume", Runtime.version().feature() >= 20); - final List nodes = internalCluster().startNodes( - 3, - Settings.builder() - .put(LeaderChecker.LEADER_CHECK_TIMEOUT_SETTING.getKey(), "1s") - .put(Coordinator.PUBLISH_TIMEOUT_SETTING.getKey(), "1s") - .build() - ); - ensureStableCluster(3); - - // Save the current master node as old master node, because that node will get frozen - final String oldMasterNode = internalCluster().getMasterName(); - - // Simulating a painful gc by suspending all threads for a long time on the current elected master node. - SingleNodeDisruption masterNodeDisruption = new LongGCDisruption(random(), oldMasterNode); - - // Save the majority side - final List majoritySide = new ArrayList<>(nodes); - majoritySide.remove(oldMasterNode); - - // Keeps track of the previous and current master when a master node transition took place on each node on the majority side: - final Map>> masters = Collections.synchronizedMap(new HashMap<>()); - for (final String node : majoritySide) { - masters.put(node, new ArrayList<>()); - internalCluster().getInstance(ClusterService.class, node).addListener(event -> { - DiscoveryNode previousMaster = event.previousState().nodes().getMasterNode(); - DiscoveryNode currentMaster = event.state().nodes().getMasterNode(); - if (Objects.equals(previousMaster, currentMaster) == false) { - logger.info( - "--> node {} received new cluster state: {} \n and had previous cluster state: {}", - node, - event.state(), - event.previousState() - ); - String previousMasterNodeName = previousMaster != null ? previousMaster.getName() : null; - String currentMasterNodeName = currentMaster != null ? currentMaster.getName() : null; - masters.get(node).add(new Tuple<>(previousMasterNodeName, currentMasterNodeName)); - } - }); - } - - final CountDownLatch oldMasterNodeSteppedDown = new CountDownLatch(1); - internalCluster().getInstance(ClusterService.class, oldMasterNode).addListener(event -> { - if (event.state().nodes().getMasterNodeId() == null) { - oldMasterNodeSteppedDown.countDown(); - } - }); - - internalCluster().setDisruptionScheme(masterNodeDisruption); - logger.info("--> freezing node [{}]", oldMasterNode); - masterNodeDisruption.startDisrupting(); - - // Wait for majority side to elect a new master - assertBusy(() -> { - for (final Map.Entry>> entry : masters.entrySet()) { - final List> transitions = entry.getValue(); - assertTrue(entry.getKey() + ": " + transitions, transitions.stream().anyMatch(transition -> transition.v2() != null)); - } - }); - - // The old master node is frozen, but here we submit a cluster state update task that doesn't get executed, but will be queued and - // once the old master node un-freezes it gets executed. The old master node will send this update + the cluster state where it is - // flagged as master to the other nodes that follow the new master. These nodes should ignore this update. - internalCluster().getInstance(ClusterService.class, oldMasterNode) - .submitUnbatchedStateUpdateTask("sneaky-update", new ClusterStateUpdateTask(Priority.IMMEDIATE) { - @Override - public ClusterState execute(ClusterState currentState) { - return ClusterState.builder(currentState).build(); - } - - @Override - public void onFailure(Exception e) { - logger.warn("failure [sneaky-update]", e); - } - }); - - // Save the new elected master node - final String newMasterNode = internalCluster().getMasterName(majoritySide.get(0)); - logger.info("--> new detected master node [{}]", newMasterNode); - - // Stop disruption - logger.info("--> unfreezing node [{}]", oldMasterNode); - masterNodeDisruption.stopDisrupting(); - - oldMasterNodeSteppedDown.await(30, TimeUnit.SECONDS); - logger.info("--> [{}] stepped down as master", oldMasterNode); - ensureStableCluster(3); - - assertThat(masters.size(), equalTo(2)); - for (Map.Entry>> entry : masters.entrySet()) { - String nodeName = entry.getKey(); - List> transitions = entry.getValue(); - assertTrue( - "[" + nodeName + "] should not apply state from old master [" + oldMasterNode + "] but it did: " + transitions, - transitions.stream().noneMatch(t -> oldMasterNode.equals(t.v2())) - ); - } - assertGreenMasterStability(internalCluster().client()); - } - - /** - * This helper method creates a 3-node cluster where all nodes are master-eligible, and then simulates a long GC on the master node 5 - * times (forcing another node to be elected master 5 times). It then asserts that the master stability health indicator status is - * YELLOW, and that expectedMasterStabilitySymptomSubstring is contained in the symptom. - * @param expectedMasterStabilitySymptomSubstring A string to expect in the master stability health indicator symptom - * @throws Exception - */ - public void testRepeatedMasterChanges(String expectedMasterStabilitySymptomSubstring) throws Exception { - assumeFalse("jdk20 removed thread suspend/resume", Runtime.version().feature() >= 20); - final List nodes = internalCluster().startNodes( - 3, - Settings.builder() - .put(LeaderChecker.LEADER_CHECK_TIMEOUT_SETTING.getKey(), "1s") - .put(Coordinator.PUBLISH_TIMEOUT_SETTING.getKey(), "1s") - .put(CoordinationDiagnosticsService.IDENTITY_CHANGES_THRESHOLD_SETTING.getKey(), 1) - .put(CoordinationDiagnosticsService.NO_MASTER_TRANSITIONS_THRESHOLD_SETTING.getKey(), 100) - .build() - ); - ensureStableCluster(3); - String firstMaster = internalCluster().getMasterName(); - // Force the master to change 2 times: - for (int i = 0; i < 2; i++) { - // Save the current master node as old master node, because that node will get frozen - final String oldMasterNode = internalCluster().getMasterName(); - - // Simulating a painful gc by suspending all threads for a long time on the current elected master node. - SingleNodeDisruption masterNodeDisruption = new LongGCDisruption(random(), oldMasterNode); - - // Save the majority side - final List majoritySide = new ArrayList<>(nodes); - majoritySide.remove(oldMasterNode); - - // Keeps track of the previous and current master when a master node transition took place on each node on the majority side: - final Map>> masters = Collections.synchronizedMap(new HashMap<>()); - for (final String node : majoritySide) { - masters.put(node, new ArrayList<>()); - internalCluster().getInstance(ClusterService.class, node).addListener(event -> { - DiscoveryNode previousMaster = event.previousState().nodes().getMasterNode(); - DiscoveryNode currentMaster = event.state().nodes().getMasterNode(); - if (Objects.equals(previousMaster, currentMaster) == false) { - logger.info( - "--> node {} received new cluster state: {} \n and had previous cluster state: {}", - node, - event.state(), - event.previousState() - ); - String previousMasterNodeName = previousMaster != null ? previousMaster.getName() : null; - String currentMasterNodeName = currentMaster != null ? currentMaster.getName() : null; - masters.get(node).add(new Tuple<>(previousMasterNodeName, currentMasterNodeName)); - } - }); - } - - final CountDownLatch oldMasterNodeSteppedDown = new CountDownLatch(1); - internalCluster().getInstance(ClusterService.class, oldMasterNode).addListener(event -> { - if (event.state().nodes().getMasterNodeId() == null) { - oldMasterNodeSteppedDown.countDown(); - } - }); - internalCluster().clearDisruptionScheme(); - internalCluster().setDisruptionScheme(masterNodeDisruption); - logger.info("--> freezing node [{}]", oldMasterNode); - masterNodeDisruption.startDisrupting(); - - // Wait for majority side to elect a new master - assertBusy(() -> { - for (final Map.Entry>> entry : masters.entrySet()) { - final List> transitions = entry.getValue(); - assertTrue(entry.getKey() + ": " + transitions, transitions.stream().anyMatch(transition -> transition.v2() != null)); - } - }); - - // Save the new elected master node - final String newMasterNode = internalCluster().getMasterName(majoritySide.get(0)); - logger.info("--> new detected master node [{}]", newMasterNode); - - // Stop disruption - logger.info("--> unfreezing node [{}]", oldMasterNode); - masterNodeDisruption.stopDisrupting(); - - oldMasterNodeSteppedDown.await(30, TimeUnit.SECONDS); - logger.info("--> [{}] stepped down as master", oldMasterNode); - ensureStableCluster(3); - - assertThat(masters.size(), equalTo(2)); - } - List nodeNamesExceptFirstMaster = Arrays.stream(internalCluster().getNodeNames()) - .filter(name -> name.equals(firstMaster) == false) - .toList(); - /* - * It is possible that the first node that became master got re-elected repeatedly. And since it was in a simulated GC when the - * other node(s) were master, it only saw itself as master. So we want to check with another node. - */ - Client client = internalCluster().client(randomFrom(nodeNamesExceptFirstMaster)); - assertMasterStability(client, HealthStatus.YELLOW, containsString(expectedMasterStabilitySymptomSubstring)); - } - - public void testRepeatedNullMasterRecognizedAsGreenIfMasterDoesNotKnowItIsUnstable() throws Exception { - assumeFalse("jdk20 removed thread suspend/resume", Runtime.version().feature() >= 20); - /* - * In this test we have a single master-eligible node. We pause it repeatedly (simulating a long GC pause for example) so that - * other nodes decide it is no longer the master. However since there is no other master-eligible node, another node is never - * elected master. And the master node never recognizes that it had a problem. So when we run the master stability check on one - * of the data nodes, it will see that there is a problem (the master has gone null repeatedly), but when it checks with the - * master, the master says everything is fine. So we expect a GREEN status. - */ - final List masterNodes = internalCluster().startMasterOnlyNodes( - 1, - Settings.builder() - .put(LeaderChecker.LEADER_CHECK_TIMEOUT_SETTING.getKey(), "1s") - .put(Coordinator.PUBLISH_TIMEOUT_SETTING.getKey(), "1s") - .put(CoordinationDiagnosticsService.NO_MASTER_TRANSITIONS_THRESHOLD_SETTING.getKey(), 1) - .build() - ); - int nullTransitionsThreshold = 1; - final List dataNodes = internalCluster().startDataOnlyNodes( - 2, - Settings.builder() - .put(LeaderChecker.LEADER_CHECK_TIMEOUT_SETTING.getKey(), "1s") - .put(Coordinator.PUBLISH_TIMEOUT_SETTING.getKey(), "1s") - .put(CoordinationDiagnosticsService.NO_MASTER_TRANSITIONS_THRESHOLD_SETTING.getKey(), nullTransitionsThreshold) - .put(CoordinationDiagnosticsService.NODE_HAS_MASTER_LOOKUP_TIMEFRAME_SETTING.getKey(), new TimeValue(60, TimeUnit.SECONDS)) - .build() - ); - ensureStableCluster(3); - for (int i = 0; i < nullTransitionsThreshold + 1; i++) { - final String masterNode = masterNodes.get(0); - - // Simulating a painful gc by suspending all threads for a long time on the current elected master node. - SingleNodeDisruption masterNodeDisruption = new LongGCDisruption(random(), masterNode); - - final CountDownLatch dataNodeMasterSteppedDown = new CountDownLatch(2); - internalCluster().getInstance(ClusterService.class, dataNodes.get(0)).addListener(event -> { - if (event.state().nodes().getMasterNodeId() == null) { - dataNodeMasterSteppedDown.countDown(); - } - }); - internalCluster().getInstance(ClusterService.class, dataNodes.get(1)).addListener(event -> { - if (event.state().nodes().getMasterNodeId() == null) { - dataNodeMasterSteppedDown.countDown(); - } - }); - internalCluster().clearDisruptionScheme(); - internalCluster().setDisruptionScheme(masterNodeDisruption); - logger.info("--> freezing node [{}]", masterNode); - masterNodeDisruption.startDisrupting(); - dataNodeMasterSteppedDown.await(30, TimeUnit.SECONDS); - // Stop disruption - logger.info("--> unfreezing node [{}]", masterNode); - masterNodeDisruption.stopDisrupting(); - ensureStableCluster(3, TimeValue.timeValueSeconds(30), false, randomFrom(dataNodes)); - } - assertGreenMasterStability(internalCluster().client(randomFrom(dataNodes))); - } - public void testNoMasterEligibleNodes() throws Exception { /* * In this test we have a single master-eligible node. We then stop the master. We set the master lookup threshold very low on the diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryIT.java index 445cbda9feb6a..10f13f6ab152f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.discovery.single; @@ -159,7 +160,7 @@ public boolean innerMatch(final LogEvent event) { other.beforeTest(random()); final ClusterState first = internalCluster().getInstance(ClusterService.class).state(); assertThat(first.nodes().getSize(), equalTo(1)); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); } finally { other.close(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/document/AliasedIndexDocumentActionsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/document/AliasedIndexDocumentActionsIT.java index a928b1a2eaecc..fb4351f5522c0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/document/AliasedIndexDocumentActionsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/document/AliasedIndexDocumentActionsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.document; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/document/DocumentActionsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/document/DocumentActionsIT.java index 709f6b866ba28..eb10877f5892d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/document/DocumentActionsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/document/DocumentActionsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.document; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/document/ShardInfoIT.java b/server/src/internalClusterTest/java/org/elasticsearch/document/ShardInfoIT.java index eeda0257fb0e4..ef3225f84c672 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/document/ShardInfoIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/document/ShardInfoIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.document; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java b/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java index 00a4f170cf7eb..f813932ebe924 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.env; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/env/NodeRepurposeCommandIT.java b/server/src/internalClusterTest/java/org/elasticsearch/env/NodeRepurposeCommandIT.java index e6839044e2100..d947b093ec70d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/env/NodeRepurposeCommandIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/env/NodeRepurposeCommandIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.env; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/explain/ExplainActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/explain/ExplainActionIT.java index cdb418182fff2..0b4304a6df31e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/explain/ExplainActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/explain/ExplainActionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.explain; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/features/ClusterFeaturesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/features/ClusterFeaturesIT.java index a695c46bcbfa2..74fd945ed3779 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/features/ClusterFeaturesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/features/ClusterFeaturesIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.features; @@ -30,6 +31,7 @@ public void testClusterHasFeatures() { FeatureService service = internalCluster().getCurrentMasterNodeInstance(FeatureService.class); assertThat(service.getNodeFeatures(), hasKey(FeatureService.FEATURES_SUPPORTED.id())); + assertThat(service.getNodeFeatures(), hasKey(FeatureService.TEST_FEATURES_ENABLED.id())); // check the nodes all have a feature in their cluster state (there should always be features_supported) var response = clusterAdmin().state(new ClusterStateRequest(TEST_REQUEST_TIMEOUT).clear().nodes(true)).actionGet(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/GatewayIndexStateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/GatewayIndexStateIT.java index 92c1e9729b460..00bd350fe2b84 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/GatewayIndexStateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/gateway/GatewayIndexStateIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gateway; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/GatewayServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/GatewayServiceIT.java index 6ac91bd483614..8202a1cd104a2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/GatewayServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/gateway/GatewayServiceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gateway; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/MetadataNodesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/MetadataNodesIT.java index 94824db66d152..d6ccdf3dc0399 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/MetadataNodesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/gateway/MetadataNodesIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gateway; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/QuorumGatewayIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/QuorumGatewayIT.java index f1b06e4efc97d..891130bb1c406 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/QuorumGatewayIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/gateway/QuorumGatewayIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gateway; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java index 4281562b64791..ab246f106eafc 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gateway; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java index 193f025e6843b..3561ce2e8cf00 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gateway; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java index 02c566f49e2b3..fd20736548772 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gateway; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java index 13886cba9084c..c3ffd60e630e1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.gateway; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/get/GetActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/get/GetActionIT.java index 76c501df1fd29..6c7754932af68 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/get/GetActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/get/GetActionIT.java @@ -1,14 +1,17 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.get; import org.apache.lucene.index.DirectoryReader; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.bulk.BulkRequestBuilder; @@ -36,6 +39,7 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalSettingsPlugin; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentType; import org.junit.Before; @@ -49,6 +53,8 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; +import java.util.function.UnaryOperator; import static java.util.Collections.singleton; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -103,25 +109,27 @@ public void testSimpleGet() { ); ensureGreen(); - GetResponse response = client().prepareGet(indexOrAlias(), "1").get(); + final Function, GetResponse> docGetter = op -> getDocument(indexOrAlias(), "1", op); + + GetResponse response = docGetter.apply(UnaryOperator.identity()); assertThat(response.isExists(), equalTo(false)); logger.info("--> index doc 1"); prepareIndex("test").setId("1").setSource("field1", "value1", "field2", "value2").get(); logger.info("--> non realtime get 1"); - response = client().prepareGet(indexOrAlias(), "1").setRealtime(false).get(); + response = docGetter.apply(r -> r.setRealtime(false)); assertThat(response.isExists(), equalTo(false)); logger.info("--> realtime get 1"); - response = client().prepareGet(indexOrAlias(), "1").get(); + response = docGetter.apply(UnaryOperator.identity()); assertThat(response.isExists(), equalTo(true)); assertThat(response.getIndex(), equalTo("test")); assertThat(response.getSourceAsMap().get("field1").toString(), equalTo("value1")); assertThat(response.getSourceAsMap().get("field2").toString(), equalTo("value2")); logger.info("--> realtime get 1 (no source, implicit)"); - response = client().prepareGet(indexOrAlias(), "1").setStoredFields(Strings.EMPTY_ARRAY).get(); + response = docGetter.apply(r -> r.setStoredFields(Strings.EMPTY_ARRAY)); assertThat(response.isExists(), equalTo(true)); assertThat(response.getIndex(), equalTo("test")); Set fields = new HashSet<>(response.getFields().keySet()); @@ -129,7 +137,7 @@ public void testSimpleGet() { assertThat(response.getSourceAsBytesRef(), nullValue()); logger.info("--> realtime get 1 (no source, explicit)"); - response = client().prepareGet(indexOrAlias(), "1").setFetchSource(false).get(); + response = docGetter.apply(r -> r.setFetchSource(false)); assertThat(response.isExists(), equalTo(true)); assertThat(response.getIndex(), equalTo("test")); fields = new HashSet<>(response.getFields().keySet()); @@ -137,14 +145,14 @@ public void testSimpleGet() { assertThat(response.getSourceAsBytesRef(), nullValue()); logger.info("--> realtime get 1 (no type)"); - response = client().prepareGet(indexOrAlias(), "1").get(); + response = docGetter.apply(UnaryOperator.identity()); assertThat(response.isExists(), equalTo(true)); assertThat(response.getIndex(), equalTo("test")); assertThat(response.getSourceAsMap().get("field1").toString(), equalTo("value1")); assertThat(response.getSourceAsMap().get("field2").toString(), equalTo("value2")); logger.info("--> realtime fetch of field"); - response = client().prepareGet(indexOrAlias(), "1").setStoredFields("field1").get(); + response = docGetter.apply(r -> r.setStoredFields("field1")); assertThat(response.isExists(), equalTo(true)); assertThat(response.getIndex(), equalTo("test")); assertThat(response.getSourceAsBytesRef(), nullValue()); @@ -152,7 +160,7 @@ public void testSimpleGet() { assertThat(response.getField("field2"), nullValue()); logger.info("--> realtime fetch of field & source"); - response = client().prepareGet(indexOrAlias(), "1").setStoredFields("field1").setFetchSource("field1", null).get(); + response = docGetter.apply(r -> r.setStoredFields("field1").setFetchSource("field1", null)); assertThat(response.isExists(), equalTo(true)); assertThat(response.getIndex(), equalTo("test")); assertThat(response.getSourceAsMap(), hasKey("field1")); @@ -161,7 +169,7 @@ public void testSimpleGet() { assertThat(response.getField("field2"), nullValue()); logger.info("--> realtime get 1"); - response = client().prepareGet(indexOrAlias(), "1").get(); + response = docGetter.apply(UnaryOperator.identity()); assertThat(response.isExists(), equalTo(true)); assertThat(response.getIndex(), equalTo("test")); assertThat(response.getSourceAsMap().get("field1").toString(), equalTo("value1")); @@ -171,14 +179,14 @@ public void testSimpleGet() { refresh(); logger.info("--> non realtime get 1 (loaded from index)"); - response = client().prepareGet(indexOrAlias(), "1").setRealtime(false).get(); + response = docGetter.apply(r -> r.setRealtime(false)); assertThat(response.isExists(), equalTo(true)); assertThat(response.getIndex(), equalTo("test")); assertThat(response.getSourceAsMap().get("field1").toString(), equalTo("value1")); assertThat(response.getSourceAsMap().get("field2").toString(), equalTo("value2")); logger.info("--> realtime fetch of field (loaded from index)"); - response = client().prepareGet(indexOrAlias(), "1").setStoredFields("field1").get(); + response = docGetter.apply(r -> r.setStoredFields("field1")); assertThat(response.isExists(), equalTo(true)); assertThat(response.getIndex(), equalTo("test")); assertThat(response.getSourceAsBytesRef(), nullValue()); @@ -186,7 +194,7 @@ public void testSimpleGet() { assertThat(response.getField("field2"), nullValue()); logger.info("--> realtime fetch of field & source (loaded from index)"); - response = client().prepareGet(indexOrAlias(), "1").setStoredFields("field1").setFetchSource(true).get(); + response = docGetter.apply(r -> r.setStoredFields("field1").setFetchSource(true)); assertThat(response.isExists(), equalTo(true)); assertThat(response.getIndex(), equalTo("test")); assertThat(response.getSourceAsBytesRef(), not(nullValue())); @@ -197,7 +205,7 @@ public void testSimpleGet() { prepareIndex("test").setId("1").setSource("field1", "value1_1", "field2", "value2_1").get(); logger.info("--> realtime get 1"); - response = client().prepareGet(indexOrAlias(), "1").get(); + response = docGetter.apply(UnaryOperator.identity()); assertThat(response.isExists(), equalTo(true)); assertThat(response.getIndex(), equalTo("test")); assertThat(response.getSourceAsMap().get("field1").toString(), equalTo("value1_1")); @@ -206,7 +214,7 @@ public void testSimpleGet() { logger.info("--> update doc 1 again"); prepareIndex("test").setId("1").setSource("field1", "value1_2", "field2", "value2_2").get(); - response = client().prepareGet(indexOrAlias(), "1").get(); + response = docGetter.apply(UnaryOperator.identity()); assertThat(response.isExists(), equalTo(true)); assertThat(response.getIndex(), equalTo("test")); assertThat(response.getSourceAsMap().get("field1").toString(), equalTo("value1_2")); @@ -215,7 +223,7 @@ public void testSimpleGet() { DeleteResponse deleteResponse = client().prepareDelete("test", "1").get(); assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult()); - response = client().prepareGet(indexOrAlias(), "1").get(); + response = docGetter.apply(UnaryOperator.identity()); assertThat(response.isExists(), equalTo(false)); } @@ -231,14 +239,34 @@ public void testGetWithAliasPointingToMultipleIndices() { DocWriteResponse indexResponse = prepareIndex("index1").setId("id").setSource(Collections.singletonMap("foo", "bar")).get(); assertThat(indexResponse.status().getStatus(), equalTo(RestStatus.CREATED.getStatus())); - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, client().prepareGet("alias1", "_alias_id")); - assertThat(exception.getMessage(), endsWith("can't execute a single index op")); + assertThat( + asInstanceOf(IllegalArgumentException.class, getDocumentFailure("alias1", "_alias_id", r -> r)).getMessage(), + endsWith("can't execute a single index op") + ); } static String indexOrAlias() { return randomBoolean() ? "test" : "alias"; } + private static GetResponse getDocument(String index, String id, UnaryOperator requestOperator) { + return safeAwait(l -> getDocumentAsync(index, id, requestOperator, l)); + } + + private static Throwable getDocumentFailure(String index, String id, UnaryOperator requestOperator) { + return ExceptionsHelper.unwrapCause(safeAwaitFailure(GetResponse.class, l -> getDocumentAsync(index, id, requestOperator, l))); + } + + private static void getDocumentAsync( + String index, + String id, + UnaryOperator requestOperator, + ActionListener listener + ) { + requestOperator.apply(client().prepareGet(index, id)) + .execute(ActionListener.runBefore(listener, () -> ThreadPool.assertCurrentThreadPool(ThreadPool.Names.GET))); + } + public void testSimpleMultiGet() throws Exception { assertAcked( prepareCreate("test").addAlias(new Alias("alias").writeIndex(randomFrom(true, false, null))) @@ -310,13 +338,14 @@ public void testGetDocWithMultivaluedFields() throws Exception { assertAcked(prepareCreate("test").setMapping(mapping1)); ensureGreen(); - GetResponse response = client().prepareGet("test", "1").get(); - assertThat(response.isExists(), equalTo(false)); + final Function, GetResponse> docGetter = op -> getDocument("test", "1", op); + + GetResponse response = docGetter.apply(UnaryOperator.identity()); assertThat(response.isExists(), equalTo(false)); prepareIndex("test").setId("1").setSource(jsonBuilder().startObject().array("field", "1", "2").endObject()).get(); - response = client().prepareGet("test", "1").setStoredFields("field").get(); + response = docGetter.apply(r -> r.setStoredFields("field")); assertThat(response.isExists(), equalTo(true)); assertThat(response.getId(), equalTo("1")); Set fields = new HashSet<>(response.getFields().keySet()); @@ -327,7 +356,7 @@ public void testGetDocWithMultivaluedFields() throws Exception { // Now test values being fetched from stored fields. refresh(); - response = client().prepareGet("test", "1").setStoredFields("field").get(); + response = docGetter.apply(r -> r.setStoredFields("field")); assertThat(response.isExists(), equalTo(true)); assertThat(response.getId(), equalTo("1")); fields = new HashSet<>(response.getFields().keySet()); @@ -341,7 +370,9 @@ public void testGetWithVersion() { assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSettings(Settings.builder().put("index.refresh_interval", -1))); ensureGreen(); - GetResponse response = client().prepareGet("test", "1").get(); + final Function, GetResponse> docGetter = op -> getDocument(indexOrAlias(), "1", op); + + GetResponse response = docGetter.apply(UnaryOperator.identity()); assertThat(response.isExists(), equalTo(false)); logger.info("--> index doc 1"); @@ -349,64 +380,52 @@ public void testGetWithVersion() { // From translog: - response = client().prepareGet(indexOrAlias(), "1").setVersion(Versions.MATCH_ANY).get(); + response = docGetter.apply(r -> r.setVersion(Versions.MATCH_ANY)); assertThat(response.isExists(), equalTo(true)); assertThat(response.getId(), equalTo("1")); assertThat(response.getVersion(), equalTo(1L)); - response = client().prepareGet(indexOrAlias(), "1").setVersion(1).get(); + response = docGetter.apply(r -> r.setVersion(1)); assertThat(response.isExists(), equalTo(true)); assertThat(response.getId(), equalTo("1")); assertThat(response.getVersion(), equalTo(1L)); - try { - client().prepareGet(indexOrAlias(), "1").setVersion(2).get(); - fail(); - } catch (VersionConflictEngineException e) { - // all good - } + assertThat(getDocumentFailure(indexOrAlias(), "1", r -> r.setVersion(2)), instanceOf(VersionConflictEngineException.class)); // From Lucene index: refresh(); - response = client().prepareGet(indexOrAlias(), "1").setVersion(Versions.MATCH_ANY).setRealtime(false).get(); + response = docGetter.apply(r -> r.setVersion(Versions.MATCH_ANY).setRealtime(false)); assertThat(response.isExists(), equalTo(true)); assertThat(response.getId(), equalTo("1")); assertThat(response.getIndex(), equalTo("test")); assertThat(response.getVersion(), equalTo(1L)); - response = client().prepareGet(indexOrAlias(), "1").setVersion(1).setRealtime(false).get(); + response = docGetter.apply(r -> r.setVersion(1).setRealtime(false)); assertThat(response.isExists(), equalTo(true)); assertThat(response.getId(), equalTo("1")); assertThat(response.getIndex(), equalTo("test")); assertThat(response.getVersion(), equalTo(1L)); - try { - client().prepareGet(indexOrAlias(), "1").setVersion(2).setRealtime(false).get(); - fail(); - } catch (VersionConflictEngineException e) { - // all good - } + assertThat( + getDocumentFailure(indexOrAlias(), "1", r -> r.setVersion(2).setRealtime(false)), + instanceOf(VersionConflictEngineException.class) + ); logger.info("--> index doc 1 again, so increasing the version"); prepareIndex("test").setId("1").setSource("field1", "value1", "field2", "value2").get(); // From translog: - response = client().prepareGet(indexOrAlias(), "1").setVersion(Versions.MATCH_ANY).get(); + response = docGetter.apply(r -> r.setVersion(Versions.MATCH_ANY)); assertThat(response.isExists(), equalTo(true)); assertThat(response.getId(), equalTo("1")); assertThat(response.getIndex(), equalTo("test")); assertThat(response.getVersion(), equalTo(2L)); - try { - client().prepareGet(indexOrAlias(), "1").setVersion(1).get(); - fail(); - } catch (VersionConflictEngineException e) { - // all good - } + assertThat(getDocumentFailure(indexOrAlias(), "1", r -> r.setVersion(1)), instanceOf(VersionConflictEngineException.class)); - response = client().prepareGet(indexOrAlias(), "1").setVersion(2).get(); + response = docGetter.apply(r -> r.setVersion(2)); assertThat(response.isExists(), equalTo(true)); assertThat(response.getId(), equalTo("1")); assertThat(response.getIndex(), equalTo("test")); @@ -415,20 +434,18 @@ public void testGetWithVersion() { // From Lucene index: refresh(); - response = client().prepareGet(indexOrAlias(), "1").setVersion(Versions.MATCH_ANY).setRealtime(false).get(); + response = docGetter.apply(r -> r.setVersion(Versions.MATCH_ANY).setRealtime(false)); assertThat(response.isExists(), equalTo(true)); assertThat(response.getId(), equalTo("1")); assertThat(response.getIndex(), equalTo("test")); assertThat(response.getVersion(), equalTo(2L)); - try { - client().prepareGet(indexOrAlias(), "1").setVersion(1).setRealtime(false).get(); - fail(); - } catch (VersionConflictEngineException e) { - // all good - } + assertThat( + getDocumentFailure(indexOrAlias(), "1", r -> r.setVersion(1).setRealtime(false)), + instanceOf(VersionConflictEngineException.class) + ); - response = client().prepareGet(indexOrAlias(), "1").setVersion(2).setRealtime(false).get(); + response = docGetter.apply(r -> r.setVersion(2).setRealtime(false)); assertThat(response.isExists(), equalTo(true)); assertThat(response.getId(), equalTo("1")); assertThat(response.getIndex(), equalTo("test")); @@ -571,15 +588,15 @@ public void testGetFieldsNonLeafField() throws Exception { .setSource(jsonBuilder().startObject().startObject("field1").field("field2", "value1").endObject().endObject()) .get(); - IllegalArgumentException exc = expectThrows( + IllegalArgumentException exc = asInstanceOf( IllegalArgumentException.class, - client().prepareGet(indexOrAlias(), "1").setStoredFields("field1") + getDocumentFailure(indexOrAlias(), "1", r -> r.setStoredFields("field1")) ); assertThat(exc.getMessage(), equalTo("field [field1] isn't a leaf field")); flush(); - exc = expectThrows(IllegalArgumentException.class, client().prepareGet(indexOrAlias(), "1").setStoredFields("field1")); + exc = asInstanceOf(IllegalArgumentException.class, getDocumentFailure(indexOrAlias(), "1", r -> r.setStoredFields("field1"))); assertThat(exc.getMessage(), equalTo("field [field1] isn't a leaf field")); } @@ -649,13 +666,13 @@ public void testGetFieldsComplexField() throws Exception { logger.info("checking real time retrieval"); String field = "field1.field2.field3.field4"; - GetResponse getResponse = client().prepareGet("my-index", "1").setStoredFields(field).get(); + GetResponse getResponse = getDocument("my-index", "1", r -> r.setStoredFields(field)); assertThat(getResponse.isExists(), equalTo(true)); assertThat(getResponse.getField(field).getValues().size(), equalTo(2)); assertThat(getResponse.getField(field).getValues().get(0).toString(), equalTo("value1")); assertThat(getResponse.getField(field).getValues().get(1).toString(), equalTo("value2")); - getResponse = client().prepareGet("my-index", "1").setStoredFields(field).get(); + getResponse = getDocument("my-index", "1", r -> r.setStoredFields(field)); assertThat(getResponse.isExists(), equalTo(true)); assertThat(getResponse.getField(field).getValues().size(), equalTo(2)); assertThat(getResponse.getField(field).getValues().get(0).toString(), equalTo("value1")); @@ -680,7 +697,7 @@ public void testGetFieldsComplexField() throws Exception { logger.info("checking post-flush retrieval"); - getResponse = client().prepareGet("my-index", "1").setStoredFields(field).get(); + getResponse = getDocument("my-index", "1", r -> r.setStoredFields(field)); assertThat(getResponse.isExists(), equalTo(true)); assertThat(getResponse.getField(field).getValues().size(), equalTo(2)); assertThat(getResponse.getField(field).getValues().get(0).toString(), equalTo("value1")); @@ -853,7 +870,7 @@ public void testAvoidWrappingSearcherInMultiGet() { // start tracking translog locations in the live version map { index("test", "0", Map.of("f", "empty")); - client().prepareGet("test", "0").setRealtime(true).get(); + getDocument("test", "0", r -> r.setRealtime(true)); refresh("test"); } Map indexedDocs = new HashMap<>(); @@ -908,7 +925,7 @@ public void testAvoidWrappingSearcherInMultiGet() { } public void testGetRemoteIndex() { - IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, client().prepareGet("cluster:index", "id")); + IllegalArgumentException iae = asInstanceOf(IllegalArgumentException.class, getDocumentFailure("cluster:index", "id", r -> r)); assertEquals( "Cross-cluster calls are not supported in this context but remote indices were requested: [cluster:index]", iae.getMessage() @@ -983,10 +1000,12 @@ private GetResponse multiGetDocument(String index, String docId, String field, @ } private GetResponse getDocument(String index, String docId, String field, @Nullable String routing) { - GetRequestBuilder getRequestBuilder = client().prepareGet().setIndex(index).setId(docId).setStoredFields(field); - if (routing != null) { - getRequestBuilder.setRouting(routing); - } - return getRequestBuilder.get(); + return getDocument(index, docId, r -> { + r.setStoredFields(field); + if (routing != null) { + r.setRouting(routing); + } + return r; + }); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/get/GetFromTranslogActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/get/GetFromTranslogActionIT.java index a77c01e199942..86ea3443d16b7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/get/GetFromTranslogActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/get/GetFromTranslogActionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.get; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/get/ShardMultiGetFomTranslogActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/get/ShardMultiGetFomTranslogActionIT.java index 6da2802ce7cc8..f5c1becada286 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/get/ShardMultiGetFomTranslogActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/get/ShardMultiGetFomTranslogActionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.get; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/health/GetHealthActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/health/GetHealthActionIT.java index 798f0e9bfb09f..d869902198c27 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/health/GetHealthActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/health/GetHealthActionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.health; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/health/GetHealthCancellationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/health/GetHealthCancellationIT.java index eda8a4eb9e459..5d0ad02f4306e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/health/GetHealthCancellationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/health/GetHealthCancellationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.health; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/health/HealthMetadataServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/health/HealthMetadataServiceIT.java index 30fc7e263a4c8..bcfcf26486d0a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/health/HealthMetadataServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/health/HealthMetadataServiceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.health; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/health/HealthServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/health/HealthServiceIT.java index a5931f29b9ff0..9da8789c6e401 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/health/HealthServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/health/HealthServiceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.health; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/health/UpdateHealthInfoCacheIT.java b/server/src/internalClusterTest/java/org/elasticsearch/health/UpdateHealthInfoCacheIT.java index b6477a7e1a6c8..3d6d11bc9c3d7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/health/UpdateHealthInfoCacheIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/health/UpdateHealthInfoCacheIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.health; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/health/node/DiskHealthIndicatorServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/health/node/DiskHealthIndicatorServiceIT.java index 2f9ef3c80d23c..8cf79082d6958 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/health/node/DiskHealthIndicatorServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/health/node/DiskHealthIndicatorServiceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.health.node; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/health/node/ShardsCapacityHealthIndicatorServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/health/node/ShardsCapacityHealthIndicatorServiceIT.java index 0a9d946ddd96e..82454213d9dba 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/health/node/ShardsCapacityHealthIndicatorServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/health/node/ShardsCapacityHealthIndicatorServiceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.health.node; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java index 216d5e25218e3..b21b51f8e5fd1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/HiddenIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/HiddenIndexIT.java index f225674215bb2..d116c091fbe4a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/HiddenIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/HiddenIndexIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/IndexRequestBuilderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/IndexRequestBuilderIT.java index 71f0c75efa026..bb9b09271fdd7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/IndexRequestBuilderIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/IndexRequestBuilderIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/IndexSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/IndexSettingsIT.java index 234b387e77a07..6ed9b6373dfe2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/IndexSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/IndexSettingsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/IndexSortIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/IndexSortIT.java index aca13ecb3b0e7..57bad0bc1a091 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/IndexSortIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/IndexSortIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/IndexingPressureIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/IndexingPressureIT.java index 97dd3d9723d5f..1ee5dc8fc9ac0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/IndexingPressureIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/IndexingPressureIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index; @@ -247,7 +248,7 @@ public void testWriteCanBeRejectedAtCoordinatingLevel() throws Exception { final long bulkRequestSize = bulkRequest.ramBytesUsed(); final long bulkShardRequestSize = totalRequestSize; restartNodesWithSettings( - Settings.builder().put(IndexingPressure.MAX_INDEXING_BYTES.getKey(), (long) (bulkShardRequestSize * 1.5) + "B").build() + Settings.builder().put(IndexingPressure.MAX_COORDINATING_BYTES.getKey(), (long) (bulkShardRequestSize * 1.5) + "B").build() ); assertAcked(prepareCreate(INDEX_NAME, indexSettings(1, 1))); @@ -311,7 +312,7 @@ public void testWriteCanBeRejectedAtPrimaryLevel() throws Exception { } final long bulkShardRequestSize = totalRequestSize; restartNodesWithSettings( - Settings.builder().put(IndexingPressure.MAX_INDEXING_BYTES.getKey(), (long) (bulkShardRequestSize * 1.5) + "B").build() + Settings.builder().put(IndexingPressure.MAX_PRIMARY_BYTES.getKey(), (long) (bulkShardRequestSize * 1.5) + "B").build() ); assertAcked(prepareCreate(INDEX_NAME, indexSettings(1, 1))); @@ -357,7 +358,12 @@ public void testWriteCanBeRejectedAtPrimaryLevel() throws Exception { } public void testWritesWillSucceedIfBelowThreshold() throws Exception { - restartNodesWithSettings(Settings.builder().put(IndexingPressure.MAX_INDEXING_BYTES.getKey(), "1MB").build()); + restartNodesWithSettings( + Settings.builder() + .put(IndexingPressure.MAX_COORDINATING_BYTES.getKey(), "1MB") + .put(IndexingPressure.MAX_PRIMARY_BYTES.getKey(), "1MB") + .build() + ); assertAcked(prepareCreate(INDEX_NAME, indexSettings(1, 1))); ensureGreen(INDEX_NAME); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java new file mode 100644 index 0000000000000..e9a4b8a10e78e --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java @@ -0,0 +1,219 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index; + +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction; +import org.elasticsearch.action.admin.indices.shrink.ResizeAction; +import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; +import org.elasticsearch.action.admin.indices.shrink.ResizeType; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesIndexResponse; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.test.ESIntegTestCase; + +import java.util.Map; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; + +public class LookupIndexModeIT extends ESIntegTestCase { + + @Override + protected int numberOfShards() { + return 1; + } + + public void testBasic() { + internalCluster().ensureAtLeastNumDataNodes(1); + Settings.Builder lookupSettings = Settings.builder().put("index.mode", "lookup"); + if (randomBoolean()) { + lookupSettings.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1); + } + CreateIndexRequest createRequest = new CreateIndexRequest("hosts"); + createRequest.settings(lookupSettings); + createRequest.simpleMapping("ip", "type=ip", "os", "type=keyword"); + assertAcked(client().admin().indices().execute(TransportCreateIndexAction.TYPE, createRequest)); + Settings settings = client().admin().indices().prepareGetSettings("hosts").get().getIndexToSettings().get("hosts"); + assertThat(settings.get("index.mode"), equalTo("lookup")); + assertThat(settings.get("index.auto_expand_replicas"), equalTo("0-all")); + Map allHosts = Map.of( + "192.168.1.2", + "Windows", + "192.168.1.3", + "MacOS", + "192.168.1.4", + "Linux", + "192.168.1.5", + "Android", + "192.168.1.6", + "iOS", + "192.168.1.7", + "Windows", + "192.168.1.8", + "MacOS", + "192.168.1.9", + "Linux", + "192.168.1.10", + "Linux", + "192.168.1.11", + "Windows" + ); + for (Map.Entry e : allHosts.entrySet()) { + client().prepareIndex("hosts").setSource("ip", e.getKey(), "os", e.getValue()).get(); + } + refresh("hosts"); + assertAcked(client().admin().indices().prepareCreate("events").setSettings(Settings.builder().put("index.mode", "logsdb")).get()); + int numDocs = between(1, 10); + for (int i = 0; i < numDocs; i++) { + String ip = randomFrom(allHosts.keySet()); + String message = randomFrom("login", "logout", "shutdown", "restart"); + client().prepareIndex("events").setSource("@timestamp", "2024-01-01", "ip", ip, "message", message).get(); + } + refresh("events"); + // _search + { + SearchResponse resp = prepareSearch("events", "hosts").setQuery(new MatchQueryBuilder("_index_mode", "lookup")) + .setSize(10000) + .get(); + for (SearchHit hit : resp.getHits()) { + assertThat(hit.getIndex(), equalTo("hosts")); + } + assertHitCount(resp, allHosts.size()); + resp.decRef(); + } + // field_caps + { + FieldCapabilitiesRequest request = new FieldCapabilitiesRequest(); + request.indices("events", "hosts"); + request.fields("*"); + request.setMergeResults(false); + request.indexFilter(new MatchQueryBuilder("_index_mode", "lookup")); + var resp = client().fieldCaps(request).actionGet(); + assertThat(resp.getIndexResponses(), hasSize(1)); + FieldCapabilitiesIndexResponse indexResponse = resp.getIndexResponses().get(0); + assertThat(indexResponse.getIndexMode(), equalTo(IndexMode.LOOKUP)); + assertThat(indexResponse.getIndexName(), equalTo("hosts")); + } + } + + public void testRejectMoreThanOneShard() { + int numberOfShards = between(2, 5); + IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> { + client().admin() + .indices() + .prepareCreate("hosts") + .setSettings(Settings.builder().put("index.mode", "lookup").put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards)) + .setMapping("ip", "type=ip", "os", "type=keyword") + .get(); + }); + assertThat( + error.getMessage(), + equalTo("index with [lookup] mode must have [index.number_of_shards] set to 1 or unset; provided " + numberOfShards) + ); + } + + public void testResizeLookupIndex() { + Settings.Builder createSettings = Settings.builder().put("index.mode", "lookup"); + if (randomBoolean()) { + createSettings.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1); + } + CreateIndexRequest createIndexRequest = new CreateIndexRequest("lookup-1").settings(createSettings); + assertAcked(client().admin().indices().execute(TransportCreateIndexAction.TYPE, createIndexRequest)); + client().admin().indices().prepareAddBlock(IndexMetadata.APIBlock.WRITE, "lookup-1").get(); + + ResizeRequest clone = new ResizeRequest("lookup-2", "lookup-1"); + clone.setResizeType(ResizeType.CLONE); + assertAcked(client().admin().indices().execute(ResizeAction.INSTANCE, clone).actionGet()); + Settings settings = client().admin().indices().prepareGetSettings("lookup-2").get().getIndexToSettings().get("lookup-2"); + assertThat(settings.get("index.mode"), equalTo("lookup")); + assertThat(settings.get("index.number_of_shards"), equalTo("1")); + assertThat(settings.get("index.auto_expand_replicas"), equalTo("0-all")); + + ResizeRequest split = new ResizeRequest("lookup-3", "lookup-1"); + split.setResizeType(ResizeType.SPLIT); + split.getTargetIndexRequest().settings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3)); + IllegalArgumentException error = expectThrows( + IllegalArgumentException.class, + () -> client().admin().indices().execute(ResizeAction.INSTANCE, split).actionGet() + ); + assertThat( + error.getMessage(), + equalTo("index with [lookup] mode must have [index.number_of_shards] set to 1 or unset; provided 3") + ); + } + + public void testResizeRegularIndexToLookup() { + String dataNode = internalCluster().startDataOnlyNode(); + assertAcked( + client().admin() + .indices() + .prepareCreate("regular-1") + .setSettings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2) + .put("index.routing.allocation.require._name", dataNode) + ) + .setMapping("ip", "type=ip", "os", "type=keyword") + .get() + ); + client().admin().indices().prepareAddBlock(IndexMetadata.APIBlock.WRITE, "regular-1").get(); + client().admin() + .indices() + .prepareUpdateSettings("regular-1") + .setSettings(Settings.builder().put("index.number_of_replicas", 0)) + .get(); + + ResizeRequest clone = new ResizeRequest("lookup-3", "regular-1"); + clone.setResizeType(ResizeType.CLONE); + clone.getTargetIndexRequest().settings(Settings.builder().put("index.mode", "lookup")); + IllegalArgumentException error = expectThrows( + IllegalArgumentException.class, + () -> client().admin().indices().execute(ResizeAction.INSTANCE, clone).actionGet() + ); + assertThat( + error.getMessage(), + equalTo("index with [lookup] mode must have [index.number_of_shards] set to 1 or unset; provided 2") + ); + + ResizeRequest shrink = new ResizeRequest("lookup-4", "regular-1"); + shrink.setResizeType(ResizeType.SHRINK); + shrink.getTargetIndexRequest() + .settings(Settings.builder().put("index.mode", "lookup").put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)); + + error = expectThrows( + IllegalArgumentException.class, + () -> client().admin().indices().execute(ResizeAction.INSTANCE, shrink).actionGet() + ); + assertThat(error.getMessage(), equalTo("can't change setting [index.mode] during resize")); + } + + public void testDoNotOverrideAutoExpandReplicas() { + internalCluster().ensureAtLeastNumDataNodes(1); + Settings.Builder createSettings = Settings.builder().put("index.mode", "lookup"); + if (randomBoolean()) { + createSettings.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1); + } + createSettings.put("index.auto_expand_replicas", "3-5"); + CreateIndexRequest createRequest = new CreateIndexRequest("hosts"); + createRequest.settings(createSettings); + createRequest.simpleMapping("ip", "type=ip", "os", "type=keyword"); + assertAcked(client().admin().indices().execute(TransportCreateIndexAction.TYPE, createRequest)); + Settings settings = client().admin().indices().prepareGetSettings("hosts").get().getIndexToSettings().get("hosts"); + assertThat(settings.get("index.mode"), equalTo("lookup")); + assertThat(settings.get("index.auto_expand_replicas"), equalTo("3-5")); + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/SettingsListenerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/SettingsListenerIT.java index 89107c54ee820..5636ee132bd5a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/SettingsListenerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/SettingsListenerIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/WaitUntilRefreshIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/WaitUntilRefreshIT.java index ca2d2c60e23e1..650641cf4ef9d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/WaitUntilRefreshIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/WaitUntilRefreshIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/engine/InternalEngineMergeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/engine/InternalEngineMergeIT.java index b7d4ce18b15fe..80de2ffcaa7ac 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/engine/InternalEngineMergeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/engine/InternalEngineMergeIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.engine; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/engine/MaxDocsLimitIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/engine/MaxDocsLimitIT.java index be7610e55b8e6..7b7433e3aa4c3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/engine/MaxDocsLimitIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/engine/MaxDocsLimitIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.engine; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/fielddata/FieldDataLoadingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/fielddata/FieldDataLoadingIT.java index 55e90d4398201..e77c1fcf19135 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/fielddata/FieldDataLoadingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/fielddata/FieldDataLoadingIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.fielddata; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/CopyToMapperIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/CopyToMapperIntegrationIT.java index c1f06aeceebde..81a0e0ede7cd3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/CopyToMapperIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/CopyToMapperIntegrationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java index 3f79d7723beb3..f7bf775bc4f8b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper; @@ -62,6 +63,8 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.oneOf; public class DynamicMappingIT extends ESIntegTestCase { @@ -189,6 +192,35 @@ private Map indexConcurrently(int numberOfFieldsToCreate, Settin return properties; } + public void testConcurrentDynamicMappingsWithConflictingType() throws Throwable { + int numberOfDocsToCreate = 16; + indicesAdmin().prepareCreate("index").setSettings(Settings.builder()).get(); + ensureGreen("index"); + final AtomicReference error = new AtomicReference<>(); + startInParallel(numberOfDocsToCreate, i -> { + try { + assertEquals( + DocWriteResponse.Result.CREATED, + prepareIndex("index").setId(Integer.toString(i)).setSource("field" + i, 0, "field" + (i + 1), 0.1).get().getResult() + ); + } catch (Exception e) { + error.compareAndSet(null, e); + } + }); + if (error.get() != null) { + throw error.get(); + } + client().admin().indices().prepareRefresh("index").get(); + for (int i = 0; i < numberOfDocsToCreate; ++i) { + assertTrue(client().prepareGet("index", Integer.toString(i)).get().isExists()); + } + Map index = indicesAdmin().prepareGetMappings("index").get().getMappings().get("index").getSourceAsMap(); + for (int i = 0, j = 1; i < numberOfDocsToCreate; i++, j++) { + assertThat(new WriteField("properties.field" + i + ".type", () -> index).get(null), is(oneOf("long", "float"))); + assertThat(new WriteField("properties.field" + j + ".type", () -> index).get(null), is(oneOf("long", "float"))); + } + } + public void testPreflightCheckAvoidsMaster() throws InterruptedException, IOException { // can't use INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING nor INDEX_MAPPING_DEPTH_LIMIT_SETTING as a check here, as that is already // checked at parse time, see testTotalFieldsLimitForDynamicMappingsUpdateCheckedAtDocumentParseTime diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/IgnoredMetadataFieldIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/IgnoredMetadataFieldIT.java index cfe5a2b69c6da..04f26a967e635 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/IgnoredMetadataFieldIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/IgnoredMetadataFieldIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java index a22910ab9c4eb..fea5d256b1993 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.mapper; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/query/plugin/CustomQueryParserIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/query/plugin/CustomQueryParserIT.java index 66a35328954e1..7d249b1044456 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/query/plugin/CustomQueryParserIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/query/plugin/CustomQueryParserIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.query.plugin; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/search/MatchPhraseQueryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/search/MatchPhraseQueryIT.java index 70983e5abfb96..55b140f2f48af 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/search/MatchPhraseQueryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/search/MatchPhraseQueryIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.search; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/BackgroundRetentionLeaseSyncActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/BackgroundRetentionLeaseSyncActionIT.java index 0bab5be245ecf..6758cc6532d65 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/BackgroundRetentionLeaseSyncActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/BackgroundRetentionLeaseSyncActionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.seqno; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncIT.java index 749cf73e822ec..dcc5d53f13fd7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.seqno; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/PeerRecoveryRetentionLeaseCreationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/PeerRecoveryRetentionLeaseCreationIT.java index 429e5db879bec..07f9d9ee7b6c3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/PeerRecoveryRetentionLeaseCreationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/PeerRecoveryRetentionLeaseCreationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.seqno; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/RetentionLeaseIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/RetentionLeaseIT.java index 5a4785252bf98..70689dc689673 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/RetentionLeaseIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/RetentionLeaseIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.seqno; @@ -335,7 +336,7 @@ public void testRetentionLeasesSyncOnRecovery() throws Exception { .getShardOrNull(new ShardId(resolveIndex("index"), 0)); final int length = randomIntBetween(1, 8); final Map currentRetentionLeases = new LinkedHashMap<>(); - logger.info("adding retention [{}}] leases", length); + logger.info("adding retention [{}] leases", length); for (int i = 0; i < length; i++) { final String id = randomValueOtherThanMany(currentRetentionLeases.keySet()::contains, () -> randomAlphaOfLength(8)); final long retainingSequenceNumber = randomLongBetween(0, Long.MAX_VALUE); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/RetentionLeaseSyncActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/RetentionLeaseSyncActionIT.java index 2d8f455792172..b9611973e5c63 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/RetentionLeaseSyncActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/RetentionLeaseSyncActionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.seqno; @@ -91,7 +92,7 @@ private static Releasable fullyAllocatePrimaryIndexingCapacityOnNode(String targ return internalCluster().getInstance(IndexingPressure.class, targetNode) .markPrimaryOperationStarted( 1, - IndexingPressure.MAX_INDEXING_BYTES.get(internalCluster().getInstance(Settings.class, targetNode)).getBytes() + 1, + IndexingPressure.MAX_PRIMARY_BYTES.get(internalCluster().getInstance(Settings.class, targetNode)).getBytes() + 1, true ); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/GlobalCheckpointListenersIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/GlobalCheckpointListenersIT.java index 256bdd45afbf7..ade7dff9677e8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/GlobalCheckpointListenersIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/GlobalCheckpointListenersIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.shard; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java index 5d996e44c6868..870947db5bd85 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.shard; @@ -714,7 +715,15 @@ public void testShardChangesWithDefaultDocType() throws Exception { } IndexShard shard = indexService.getShard(0); try ( - Translog.Snapshot luceneSnapshot = shard.newChangesSnapshot("test", 0, numOps - 1, true, randomBoolean(), randomBoolean()); + Translog.Snapshot luceneSnapshot = shard.newChangesSnapshot( + "test", + 0, + numOps - 1, + true, + randomBoolean(), + randomBoolean(), + randomLongBetween(1, ByteSizeValue.ofMb(32).getBytes()) + ); Translog.Snapshot translogSnapshot = getTranslog(shard).newSnapshot() ) { List opsFromLucene = TestTranslog.drainSnapshot(luceneSnapshot, true); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java index b7dbcf42630e1..bd58526c61432 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.shard; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/SearchIdleIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/SearchIdleIT.java index d9100e1e631db..dda65ac02a695 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/SearchIdleIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/SearchIdleIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.shard; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedFileIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedFileIT.java index f3b888022127f..e6fc4c45219cd 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedFileIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedFileIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.store; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedTranslogIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedTranslogIT.java index 0c0ece4bf5227..7e3fb4d8bf5e1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedTranslogIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedTranslogIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.store; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/store/ExceptionRetryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/store/ExceptionRetryIT.java index 423e5c14c472a..03afabaae1d0d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/store/ExceptionRetryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/store/ExceptionRetryIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.store; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java index 45b9091ab2552..5cb468da70996 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.index.suggest.stats; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indexing/IndexActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indexing/IndexActionIT.java index bc2f0ec94f0ab..62c5f934ec8b6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indexing/IndexActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indexing/IndexActionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indexing; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indexlifecycle/IndexLifecycleActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indexlifecycle/IndexLifecycleActionIT.java index 8ea707c0a26b3..14e5e8c42cecc 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indexlifecycle/IndexLifecycleActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indexlifecycle/IndexLifecycleActionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indexlifecycle; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java index 69e982a30b354..c1f103434a998 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndexingMemoryControllerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndexingMemoryControllerIT.java index 7e057c19ea82e..74ccdce19d3ad 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndexingMemoryControllerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndexingMemoryControllerIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesLifecycleListenerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesLifecycleListenerIT.java index 325cd27f0090f..b78b6def94d28 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesLifecycleListenerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesLifecycleListenerIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java index 267af6cc1fb01..2a097e7ce980c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices; @@ -286,7 +287,7 @@ public void testWildcardBehaviour() throws Exception { verify(indicesStats(indices), false); verify(forceMerge(indices), false); verify(refreshBuilder(indices), false); - verify(validateQuery(indices), true); + verify(validateQuery(indices), false); verify(getAliases(indices), false); verify(getFieldMapping(indices), false); verify(getMapping(indices), false); @@ -337,7 +338,7 @@ public void testWildcardBehaviour() throws Exception { verify(indicesStats(indices), false); verify(forceMerge(indices), false); verify(refreshBuilder(indices), false); - verify(validateQuery(indices), true); + verify(validateQuery(indices), false); verify(getAliases(indices), false); verify(getFieldMapping(indices), false); verify(getMapping(indices), false); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesRequestCacheIT.java index 08ce9af14ab13..7c58ff616476c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesRequestCacheIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesRequestCacheIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices; @@ -320,9 +321,11 @@ public void testQueryRewriteDates() throws Exception { public void testQueryRewriteDatesWithNow() throws Exception { Client client = client(); Settings settings = indexSettings(1, 0).put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true).build(); - assertAcked(indicesAdmin().prepareCreate("index-1").setMapping("d", "type=date").setSettings(settings).get()); - assertAcked(indicesAdmin().prepareCreate("index-2").setMapping("d", "type=date").setSettings(settings).get()); - assertAcked(indicesAdmin().prepareCreate("index-3").setMapping("d", "type=date").setSettings(settings).get()); + assertAcked( + indicesAdmin().prepareCreate("index-1").setMapping("d", "type=date").setSettings(settings), + indicesAdmin().prepareCreate("index-2").setMapping("d", "type=date").setSettings(settings), + indicesAdmin().prepareCreate("index-3").setMapping("d", "type=date").setSettings(settings) + ); ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); DateFormatter formatter = DateFormatter.forPattern("strict_date_optional_time"); indexRandom( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java index 486aa8d1f5084..fbc4e08b6b78a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java index 6eaf8776916a4..cc05bb4c82ceb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptorAllowsTemplates.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptorAllowsTemplates.java index 1cfbea473ce6f..72aa3d35d96fc 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptorAllowsTemplates.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptorAllowsTemplates.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexPlugin.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexPlugin.java index e11e85e610f6e..412be48884832 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexPlugin.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexPlugin.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzerIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzerIntegrationIT.java index 0e6cd665b6951..48aef0d348045 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzerIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzerIntegrationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.analysis; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java index e0b13d727caa3..50bff915d587c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.analyze; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ResolveClusterIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ResolveClusterIT.java index 1ca8fb315b09f..1a6674edc5147 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ResolveClusterIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ResolveClusterIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.cluster; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ShardLockFailureIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ShardLockFailureIT.java index d263a9d658891..0ce3ca53e1c1f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ShardLockFailureIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ShardLockFailureIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.cluster; @@ -71,9 +72,9 @@ public void testShardLockFailure() throws Exception { var ignored1 = internalCluster().getInstance(NodeEnvironment.class, node).shardLock(shardId, "blocked for test"); var mockLog = MockLog.capture(IndicesClusterStateService.class); ) { - final CountDownLatch countDownLatch = new CountDownLatch(1); mockLog.addExpectation(new MockLog.LoggingExpectation() { + private final CountDownLatch countDownLatch = new CountDownLatch(1); int debugMessagesSeen = 0; int warnMessagesSeen = 0; @@ -100,14 +101,20 @@ public synchronized void match(LogEvent event) { } @Override - public void assertMatched() {} + public void assertMatched() { + fail("unused"); + } + + @Override + public void awaitMatched(long millis) throws InterruptedException { + assertTrue(countDownLatch.await(millis, TimeUnit.MILLISECONDS)); + } }); updateIndexSettings(Settings.builder().putNull(IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_PREFIX + "._name"), indexName); ensureYellow(indexName); - assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); + mockLog.awaitAllExpectationsMatched(); assertEquals(ClusterHealthStatus.YELLOW, clusterAdmin().prepareHealth(TEST_REQUEST_TIMEOUT, indexName).get().getStatus()); - mockLog.assertAllExpectationsMatched(); } ensureGreen(indexName); @@ -152,7 +159,7 @@ public void testShardLockTimeout() throws Exception { ); updateIndexSettings(Settings.builder().putNull(IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_PREFIX + "._name"), indexName); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); final var clusterHealthResponse = clusterAdmin().prepareHealth(TEST_REQUEST_TIMEOUT, indexName) .setWaitForEvents(Priority.LANGUID) .setTimeout(TimeValue.timeValueSeconds(10)) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/flush/FlushIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/flush/FlushIT.java index aef37161cb948..18d81911b86aa 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/flush/FlushIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/flush/FlushIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.flush; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateIT.java index ec5d9876b7703..a096bfa188f0d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.mapping; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/DedicatedMasterGetFieldMappingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/DedicatedMasterGetFieldMappingIT.java index eb83b98b2467b..186d4b430a03f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/DedicatedMasterGetFieldMappingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/DedicatedMasterGetFieldMappingIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.mapping; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/MalformedDynamicTemplateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/MalformedDynamicTemplateIT.java index 937addb473f8b..abe9bb71d3365 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/MalformedDynamicTemplateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/MalformedDynamicTemplateIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.mapping; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java index 984082ec65193..71fcb25c2e0b2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.mapping; @@ -80,9 +81,10 @@ private XContentBuilder getMappingForType() throws IOException { } public void testGetFieldMappings() throws Exception { - - assertAcked(prepareCreate("indexa").setMapping(getMappingForType())); - assertAcked(indicesAdmin().prepareCreate("indexb").setMapping(getMappingForType())); + assertAcked( + prepareCreate("indexa").setMapping(getMappingForType()), + indicesAdmin().prepareCreate("indexb").setMapping(getMappingForType()) + ); // Get mappings by full name GetFieldMappingsResponse response = indicesAdmin().prepareGetFieldMappings("indexa").setFields("field1", "obj.subfield").get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetMappingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetMappingsIT.java index 57f09e1ed2bb1..20e59fab3bd0c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetMappingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetMappingsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.mapping; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java index 3d240627cf23f..6f6e488d46b23 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.mapping; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerNoopIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerNoopIT.java index dd29823f8076f..eb3d404b408db 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerNoopIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerNoopIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.memory.breaker; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java index e547ae5a46deb..0da05f54bc5bb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.memory.breaker; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/HierarchyCircuitBreakerTelemetryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/HierarchyCircuitBreakerTelemetryIT.java index 56fcb3c1d123d..3ab2e62cc49c1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/HierarchyCircuitBreakerTelemetryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/HierarchyCircuitBreakerTelemetryIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.memory.breaker; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java index 22f987cc855cc..005f7c7dbb264 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.memory.breaker; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/DanglingIndicesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/DanglingIndicesIT.java index a51f4bb10dc00..fd323e9c1db8b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/DanglingIndicesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/DanglingIndicesIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.recovery; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java index 64f594c488a5e..581145d949cf9 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.recovery; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java index abeaf8422748d..6c5b34e3fdc45 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.recovery; @@ -146,7 +147,6 @@ import static org.elasticsearch.action.support.ActionTestUtils.assertNoFailureListener; import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING; import static org.elasticsearch.index.seqno.SequenceNumbers.NO_OPS_PERFORMED; -import static org.elasticsearch.node.RecoverySettingsChunkSizePlugin.CHUNK_SIZE_SETTING; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.hamcrest.Matchers.empty; @@ -246,7 +246,7 @@ private void assertOnGoingRecoveryState( public Settings.Builder createRecoverySettingsChunkPerSecond(long chunkSizeBytes) { return Settings.builder() // Set the chunk size in bytes - .put(CHUNK_SIZE_SETTING.getKey(), new ByteSizeValue(chunkSizeBytes, ByteSizeUnit.BYTES)) + .put(RecoverySettings.INDICES_RECOVERY_CHUNK_SIZE.getKey(), new ByteSizeValue(chunkSizeBytes, ByteSizeUnit.BYTES)) // Set one chunk of bytes per second. .put(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey(), chunkSizeBytes, ByteSizeUnit.BYTES); } @@ -269,7 +269,7 @@ private void unthrottleRecovery() { Settings.builder() // 200mb is an arbitrary number intended to be large enough to avoid more throttling. .put(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey(), "200mb") - .put(CHUNK_SIZE_SETTING.getKey(), RecoverySettings.DEFAULT_CHUNK_SIZE) + .put(RecoverySettings.INDICES_RECOVERY_CHUNK_SIZE.getKey(), RecoverySettings.DEFAULT_CHUNK_SIZE) ); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/ReplicaToPrimaryPromotionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/ReplicaToPrimaryPromotionIT.java index 25ac384a22917..7a7661d5885ad 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/ReplicaToPrimaryPromotionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/ReplicaToPrimaryPromotionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.recovery; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/TaskRecoveryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/TaskRecoveryIT.java index 16905dc5b4d37..73cf0432c92a8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/TaskRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/TaskRecoveryIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.recovery; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/plan/ShardSnapshotsServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/plan/ShardSnapshotsServiceIT.java index d117373b58f05..bf3665968d2ac 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/plan/ShardSnapshotsServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/plan/ShardSnapshotsServiceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.recovery.plan; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/GetSettingsBlocksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/GetSettingsBlocksIT.java index 45ba836ee2449..49639f80cbf42 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/GetSettingsBlocksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/GetSettingsBlocksIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.settings; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/InternalSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/InternalSettingsIT.java index 3c62f859b3f31..289ce736a8a11 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/InternalSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/InternalSettingsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.settings; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/PrivateSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/PrivateSettingsIT.java index 0ebd276511795..993c2272f69cf 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/PrivateSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/PrivateSettingsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.settings; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/UpdateNumberOfReplicasIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/UpdateNumberOfReplicasIT.java index 606b694cbfeb9..69bd3e6804fa8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/UpdateNumberOfReplicasIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/UpdateNumberOfReplicasIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.settings; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java index 20089cd463bff..4dacc6fbb6bcc 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.settings; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseIndexDisableCloseAllIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseIndexDisableCloseAllIT.java index 5501b88d8a267..cf3136d5dacf7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseIndexDisableCloseAllIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseIndexDisableCloseAllIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.state; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseIndexIT.java index 3947ae6d2b577..aaa78cc681a1f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseIndexIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.state; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseWhileRelocatingShardsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseWhileRelocatingShardsIT.java index 6647356f070ae..a6b168af5268d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseWhileRelocatingShardsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseWhileRelocatingShardsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.state; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java index 9ec7ebcf91535..e0e17a2a8ee0e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.state; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/ReopenWhileClosingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/ReopenWhileClosingIT.java index 1b0b71b86f074..3e1df75921871 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/ReopenWhileClosingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/ReopenWhileClosingIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.state; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/SimpleIndexStateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/SimpleIndexStateIT.java index c62f1776178ba..5b65cf3e299c6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/SimpleIndexStateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/SimpleIndexStateIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.state; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java index 083823523afaa..ab91d92927217 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.stats; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java index 58c1a30cb5513..a83fd6d869359 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.store; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java index c97548942fc7d..e23e342b91d65 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.template; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/IndexTemplateBlocksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/IndexTemplateBlocksIT.java index 48958e3e39b9b..2aa834b98987a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/IndexTemplateBlocksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/IndexTemplateBlocksIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.template; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java index 4051dba84588a..7894d904ecf49 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.indices.template; @@ -880,7 +881,7 @@ public void testPartitionedTemplate() throws Exception { ); assertThat( eBadSettings.getMessage(), - containsString("partition size [6] should be a positive number less than the number of shards [5]") + containsString("partition size [6] should be a positive number less than the number of routing shards [5]") ); // provide an invalid mapping for a partitioned index @@ -912,7 +913,7 @@ public void testPartitionedTemplate() throws Exception { assertThat( eBadIndex.getMessage(), - containsString("partition size [6] should be a positive number less than the number of shards [5]") + containsString("partition size [6] should be a positive number less than the number of routing shards [5]") ); // finally, create a valid index diff --git a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestAsyncProcessorIT.java b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestAsyncProcessorIT.java index 2e515b07b59a0..3adcb1026c50d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestAsyncProcessorIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestAsyncProcessorIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java index 9fd7aaabaf2f5..7cb610ba26e3e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestFileSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestFileSettingsIT.java index 0fa1ef1208593..d495737c0d8da 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestFileSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestFileSettingsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java index 5f036681f8492..6a871211db155 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestStatsNamesAndTypesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestStatsNamesAndTypesIT.java index 86e1d2e332f36..7c5f078bb8a3c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestStatsNamesAndTypesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestStatsNamesAndTypesIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.ingest; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/mget/SimpleMgetIT.java b/server/src/internalClusterTest/java/org/elasticsearch/mget/SimpleMgetIT.java index 03e8edbf1173d..03e5dd0510c1f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/mget/SimpleMgetIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/mget/SimpleMgetIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.mget; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java new file mode 100644 index 0000000000000..c3f4c79909168 --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java @@ -0,0 +1,452 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.monitor.metrics; + +import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.telemetry.Measurement; +import org.elasticsearch.telemetry.TestTelemetryPlugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.FailingFieldPlugin; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.hamcrest.Matcher; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.index.mapper.DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasSize; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0) +public class IndicesMetricsIT extends ESIntegTestCase { + + public static class TestAPMInternalSettings extends Plugin { + @Override + public List> getSettings() { + return List.of( + Setting.timeSetting("telemetry.agent.metrics_interval", TimeValue.timeValueSeconds(0), Setting.Property.NodeScope) + ); + } + } + + @Override + protected Collection> nodePlugins() { + return List.of(TestTelemetryPlugin.class, TestAPMInternalSettings.class, FailingFieldPlugin.class); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal, otherSettings)) + .put("telemetry.agent.metrics_interval", TimeValue.timeValueSeconds(0)) // disable metrics cache refresh delay + .build(); + } + + static final String STANDARD_INDEX_COUNT = "es.indices.standard.total"; + static final String STANDARD_BYTES_SIZE = "es.indices.standard.size"; + static final String STANDARD_DOCS_COUNT = "es.indices.standard.docs.total"; + static final String STANDARD_QUERY_COUNT = "es.indices.standard.query.total"; + static final String STANDARD_QUERY_TIME = "es.indices.standard.query.time"; + static final String STANDARD_QUERY_FAILURE = "es.indices.standard.query.failure.total"; + static final String STANDARD_FETCH_COUNT = "es.indices.standard.fetch.total"; + static final String STANDARD_FETCH_TIME = "es.indices.standard.fetch.time"; + static final String STANDARD_FETCH_FAILURE = "es.indices.standard.fetch.failure.total"; + static final String STANDARD_INDEXING_COUNT = "es.indices.standard.indexing.total"; + static final String STANDARD_INDEXING_TIME = "es.indices.standard.indexing.time"; + static final String STANDARD_INDEXING_FAILURE = "es.indices.standard.indexing.failure.total"; + + static final String TIME_SERIES_INDEX_COUNT = "es.indices.time_series.total"; + static final String TIME_SERIES_BYTES_SIZE = "es.indices.time_series.size"; + static final String TIME_SERIES_DOCS_COUNT = "es.indices.time_series.docs.total"; + static final String TIME_SERIES_QUERY_COUNT = "es.indices.time_series.query.total"; + static final String TIME_SERIES_QUERY_TIME = "es.indices.time_series.query.time"; + static final String TIME_SERIES_QUERY_FAILURE = "es.indices.time_series.query.failure.total"; + static final String TIME_SERIES_FETCH_COUNT = "es.indices.time_series.fetch.total"; + static final String TIME_SERIES_FETCH_TIME = "es.indices.time_series.fetch.time"; + static final String TIME_SERIES_FETCH_FAILURE = "es.indices.time_series.fetch.failure.total"; + static final String TIME_SERIES_INDEXING_COUNT = "es.indices.time_series.indexing.total"; + static final String TIME_SERIES_INDEXING_TIME = "es.indices.time_series.indexing.time"; + static final String TIME_SERIES_INDEXING_FAILURE = "es.indices.time_series.indexing.failure.total"; + + static final String LOGSDB_INDEX_COUNT = "es.indices.logsdb.total"; + static final String LOGSDB_BYTES_SIZE = "es.indices.logsdb.size"; + static final String LOGSDB_DOCS_COUNT = "es.indices.logsdb.docs.total"; + static final String LOGSDB_QUERY_COUNT = "es.indices.logsdb.query.total"; + static final String LOGSDB_QUERY_TIME = "es.indices.logsdb.query.time"; + static final String LOGSDB_QUERY_FAILURE = "es.indices.logsdb.query.failure.total"; + static final String LOGSDB_FETCH_COUNT = "es.indices.logsdb.fetch.total"; + static final String LOGSDB_FETCH_TIME = "es.indices.logsdb.fetch.time"; + static final String LOGSDB_FETCH_FAILURE = "es.indices.logsdb.fetch.failure.total"; + static final String LOGSDB_INDEXING_COUNT = "es.indices.logsdb.indexing.total"; + static final String LOGSDB_INDEXING_TIME = "es.indices.logsdb.indexing.time"; + static final String LOGSDB_INDEXING_FAILURE = "es.indices.logsdb.indexing.failure.total"; + + public void testIndicesMetrics() { + String indexNode = internalCluster().startNode(); + ensureStableCluster(1); + TestTelemetryPlugin telemetry = internalCluster().getInstance(PluginsService.class, indexNode) + .filterPlugins(TestTelemetryPlugin.class) + .findFirst() + .orElseThrow(); + IndicesService indicesService = internalCluster().getInstance(IndicesService.class, indexNode); + var indexing0 = indicesService.stats(CommonStatsFlags.ALL, false).getIndexing().getTotal(); + telemetry.resetMeter(); + long numStandardIndices = randomIntBetween(1, 5); + long numStandardDocs = populateStandardIndices(numStandardIndices); + var indexing1 = indicesService.stats(CommonStatsFlags.ALL, false).getIndexing().getTotal(); + collectThenAssertMetrics( + telemetry, + 1, + Map.of( + STANDARD_INDEX_COUNT, + equalTo(numStandardIndices), + STANDARD_DOCS_COUNT, + equalTo(numStandardDocs), + STANDARD_BYTES_SIZE, + greaterThan(0L), + + STANDARD_INDEXING_COUNT, + equalTo(numStandardDocs), + STANDARD_INDEXING_TIME, + greaterThanOrEqualTo(0L), + STANDARD_INDEXING_FAILURE, + equalTo(indexing1.getIndexFailedCount() - indexing0.getIndexCount()) + ) + ); + + long numTimeSeriesIndices = randomIntBetween(1, 5); + long numTimeSeriesDocs = populateTimeSeriesIndices(numTimeSeriesIndices); + var indexing2 = indicesService.stats(CommonStatsFlags.ALL, false).getIndexing().getTotal(); + collectThenAssertMetrics( + telemetry, + 2, + Map.of( + TIME_SERIES_INDEX_COUNT, + equalTo(numTimeSeriesIndices), + TIME_SERIES_DOCS_COUNT, + equalTo(numTimeSeriesDocs), + TIME_SERIES_BYTES_SIZE, + greaterThan(20L), + + TIME_SERIES_INDEXING_COUNT, + equalTo(numTimeSeriesDocs), + TIME_SERIES_INDEXING_TIME, + greaterThanOrEqualTo(0L), + TIME_SERIES_INDEXING_FAILURE, + equalTo(indexing2.getIndexFailedCount() - indexing1.getIndexFailedCount()) + ) + ); + + long numLogsdbIndices = randomIntBetween(1, 5); + long numLogsdbDocs = populateLogsdbIndices(numLogsdbIndices); + var indexing3 = indicesService.stats(CommonStatsFlags.ALL, false).getIndexing().getTotal(); + collectThenAssertMetrics( + telemetry, + 3, + Map.of( + LOGSDB_INDEX_COUNT, + equalTo(numLogsdbIndices), + LOGSDB_DOCS_COUNT, + equalTo(numLogsdbDocs), + LOGSDB_BYTES_SIZE, + greaterThan(0L), + LOGSDB_INDEXING_COUNT, + equalTo(numLogsdbDocs), + LOGSDB_INDEXING_TIME, + greaterThanOrEqualTo(0L), + LOGSDB_INDEXING_FAILURE, + equalTo(indexing3.getIndexFailedCount() - indexing2.getIndexFailedCount()) + ) + ); + // already collected indexing stats + collectThenAssertMetrics( + telemetry, + 4, + Map.of( + STANDARD_INDEXING_COUNT, + equalTo(0L), + STANDARD_INDEXING_TIME, + equalTo(0L), + STANDARD_INDEXING_FAILURE, + equalTo(0L), + + TIME_SERIES_INDEXING_COUNT, + equalTo(0L), + TIME_SERIES_INDEXING_TIME, + equalTo(0L), + TIME_SERIES_INDEXING_FAILURE, + equalTo(0L), + + LOGSDB_INDEXING_COUNT, + equalTo(0L), + LOGSDB_INDEXING_TIME, + equalTo(0L), + LOGSDB_INDEXING_FAILURE, + equalTo(0L) + ) + ); + String searchNode = internalCluster().startDataOnlyNode(); + indicesService = internalCluster().getInstance(IndicesService.class, searchNode); + telemetry = internalCluster().getInstance(PluginsService.class, searchNode) + .filterPlugins(TestTelemetryPlugin.class) + .findFirst() + .orElseThrow(); + ensureGreen("st*", "log*", "time*"); + // search and fetch + String preference = "_only_local"; + client(searchNode).prepareSearch("standard*").setPreference(preference).setSize(100).get().decRef(); + var search1 = indicesService.stats(CommonStatsFlags.ALL, false).getSearch().getTotal(); + collectThenAssertMetrics( + telemetry, + 1, + Map.of( + STANDARD_QUERY_COUNT, + equalTo(numStandardIndices), + STANDARD_QUERY_TIME, + equalTo(search1.getQueryTimeInMillis()), + STANDARD_FETCH_COUNT, + equalTo(search1.getFetchCount()), + STANDARD_FETCH_TIME, + equalTo(search1.getFetchTimeInMillis()), + + TIME_SERIES_QUERY_COUNT, + equalTo(0L), + TIME_SERIES_QUERY_TIME, + equalTo(0L), + + LOGSDB_QUERY_COUNT, + equalTo(0L), + LOGSDB_QUERY_TIME, + equalTo(0L) + ) + ); + + client(searchNode).prepareSearch("time*").setPreference(preference).setSize(100).get().decRef(); + var search2 = indicesService.stats(CommonStatsFlags.ALL, false).getSearch().getTotal(); + collectThenAssertMetrics( + telemetry, + 2, + Map.of( + STANDARD_QUERY_COUNT, + equalTo(0L), + STANDARD_QUERY_TIME, + equalTo(0L), + + TIME_SERIES_QUERY_COUNT, + equalTo(numTimeSeriesIndices), + TIME_SERIES_QUERY_TIME, + equalTo(search2.getQueryTimeInMillis() - search1.getQueryTimeInMillis()), + TIME_SERIES_FETCH_COUNT, + equalTo(search2.getFetchCount() - search1.getFetchCount()), + TIME_SERIES_FETCH_TIME, + equalTo(search2.getFetchTimeInMillis() - search1.getFetchTimeInMillis()), + + LOGSDB_QUERY_COUNT, + equalTo(0L), + LOGSDB_QUERY_TIME, + equalTo(0L) + ) + ); + client(searchNode).prepareSearch("logs*").setPreference(preference).setSize(100).get().decRef(); + var search3 = indicesService.stats(CommonStatsFlags.ALL, false).getSearch().getTotal(); + collectThenAssertMetrics( + telemetry, + 3, + Map.of( + STANDARD_QUERY_COUNT, + equalTo(0L), + STANDARD_QUERY_TIME, + equalTo(0L), + + TIME_SERIES_QUERY_COUNT, + equalTo(0L), + TIME_SERIES_QUERY_TIME, + equalTo(0L), + + LOGSDB_QUERY_COUNT, + equalTo(numLogsdbIndices), + LOGSDB_QUERY_TIME, + equalTo(search3.getQueryTimeInMillis() - search2.getQueryTimeInMillis()), + LOGSDB_FETCH_COUNT, + equalTo(search3.getFetchCount() - search2.getFetchCount()), + LOGSDB_FETCH_TIME, + equalTo(search3.getFetchTimeInMillis() - search2.getFetchTimeInMillis()) + ) + ); + // search failures + expectThrows( + Exception.class, + () -> { client(searchNode).prepareSearch("logs*").setPreference(preference).setRuntimeMappings(parseMapping(""" + { + "fail_me": { + "type": "long", + "script": {"source": "<>", "lang": "failing_field"} + } + } + """)).setQuery(new RangeQueryBuilder("fail_me").gte(0)).setAllowPartialSearchResults(true).get(); } + ); + collectThenAssertMetrics( + telemetry, + 4, + Map.of( + STANDARD_QUERY_FAILURE, + equalTo(0L), + STANDARD_FETCH_FAILURE, + equalTo(0L), + TIME_SERIES_QUERY_FAILURE, + equalTo(0L), + TIME_SERIES_FETCH_FAILURE, + equalTo(0L), + LOGSDB_QUERY_FAILURE, + equalTo(numLogsdbIndices), + LOGSDB_FETCH_FAILURE, + equalTo(0L) + ) + ); + + verifyStatsPerIndexMode( + Map.of(IndexMode.STANDARD, numStandardDocs, IndexMode.LOGSDB, numLogsdbDocs, IndexMode.TIME_SERIES, numTimeSeriesDocs) + ); + } + + void collectThenAssertMetrics(TestTelemetryPlugin telemetry, int times, Map> matchers) { + telemetry.collect(); + for (Map.Entry> e : matchers.entrySet()) { + String name = e.getKey(); + List measurements = telemetry.getLongGaugeMeasurement(name); + assertThat(name, measurements, hasSize(times)); + assertThat(name, measurements.get(measurements.size() - 1).getLong(), e.getValue()); + } + } + + int populateStandardIndices(long numIndices) { + int totalDocs = 0; + for (int i = 0; i < numIndices; i++) { + String indexName = "standard-" + i; + createIndex(indexName, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build()); + int numDocs = between(1, 5); + for (int d = 0; d < numDocs; d++) { + indexDoc(indexName, Integer.toString(d), "f", Integer.toString(d)); + } + totalDocs += numDocs; + flush(indexName); + } + return totalDocs; + } + + int populateTimeSeriesIndices(long numIndices) { + int totalDocs = 0; + for (int i = 0; i < numIndices; i++) { + String indexName = "time_series-" + i; + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("mode", "time_series") + .putList("routing_path", List.of("host")) + .build(); + client().admin() + .indices() + .prepareCreate(indexName) + .setSettings(settings) + .setMapping( + "@timestamp", + "type=date", + "host", + "type=keyword,time_series_dimension=true", + "cpu", + "type=long,time_series_metric=gauge" + ) + .get(); + long timestamp = DEFAULT_DATE_TIME_FORMATTER.parseMillis("2024-04-15T00:00:00Z"); + int numDocs = between(1, 5); + for (int d = 0; d < numDocs; d++) { + timestamp += between(1, 5) * 1000L; + client().prepareIndex(indexName) + .setSource("@timestamp", timestamp, "host", randomFrom("prod", "qa"), "cpu", randomIntBetween(1, 100)) + .get(); + } + totalDocs += numDocs; + flush(indexName); + refresh(indexName); + } + return totalDocs; + } + + int populateLogsdbIndices(long numIndices) { + int totalDocs = 0; + for (int i = 0; i < numIndices; i++) { + String indexName = "logsdb-" + i; + Settings settings = Settings.builder().put("mode", "logsdb").put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build(); + client().admin() + .indices() + .prepareCreate(indexName) + .setSettings(settings) + .setMapping("@timestamp", "type=date", "host.name", "type=keyword", "cpu", "type=long") + .get(); + long timestamp = DEFAULT_DATE_TIME_FORMATTER.parseMillis("2024-04-15T00:00:00Z"); + int numDocs = between(1, 5); + for (int d = 0; d < numDocs; d++) { + timestamp += between(1, 5) * 1000L; + client().prepareIndex(indexName) + .setSource("@timestamp", timestamp, "host.name", randomFrom("prod", "qa"), "cpu", randomIntBetween(1, 100)) + .get(); + } + int numFailures = between(0, 2); + for (int d = 0; d < numFailures; d++) { + expectThrows(Exception.class, () -> { + client().prepareIndex(indexName) + .setSource( + "@timestamp", + "malformed-timestamp", + "host.name", + randomFrom("prod", "qa"), + "cpu", + randomIntBetween(1, 100) + ) + .get(); + }); + } + totalDocs += numDocs; + flush(indexName); + refresh(indexName); + } + return totalDocs; + } + + private void verifyStatsPerIndexMode(Map expectedDocs) { + var nodes = clusterService().state().nodes().stream().toArray(DiscoveryNode[]::new); + var request = new IndexModeStatsActionType.StatsRequest(nodes); + var resp = client().execute(IndexModeStatsActionType.TYPE, request).actionGet(); + var stats = resp.stats(); + for (Map.Entry e : expectedDocs.entrySet()) { + assertThat(stats.get(e.getKey()).numDocs(), equalTo(e.getValue())); + } + } + + private Map parseMapping(String mapping) throws IOException { + try (XContentParser parser = createParser(JsonXContent.jsonXContent, mapping)) { + return parser.map(); + } + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/NodeIndexingMetricsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/NodeIndexingMetricsIT.java index 7b26cc5edf1bc..1635a08e1768b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/NodeIndexingMetricsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/NodeIndexingMetricsIT.java @@ -1,41 +1,56 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.monitor.metrics; +import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.bulk.IncrementalBulkService; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.core.AbstractRefCounted; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.telemetry.Measurement; import org.elasticsearch.telemetry.TestTelemetryPlugin; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; -import static org.elasticsearch.index.IndexingPressure.MAX_INDEXING_BYTES; +import static org.elasticsearch.index.IndexingPressure.MAX_COORDINATING_BYTES; +import static org.elasticsearch.index.IndexingPressure.MAX_PRIMARY_BYTES; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThan; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0) public class NodeIndexingMetricsIT extends ESIntegTestCase { @@ -197,7 +212,7 @@ public void testNodeIndexingMetricsArePublishing() { public void testCoordinatingRejectionMetricsArePublishing() { // lower Indexing Pressure limits to trigger coordinating rejections - final String dataNode = internalCluster().startNode(Settings.builder().put(MAX_INDEXING_BYTES.getKey(), "1KB")); + final String dataNode = internalCluster().startNode(Settings.builder().put(MAX_COORDINATING_BYTES.getKey(), "1KB")); ensureStableCluster(1); final TestTelemetryPlugin plugin = internalCluster().getInstance(PluginsService.class, dataNode) @@ -238,7 +253,7 @@ public void testCoordinatingRejectionMetricsArePublishing() { public void testCoordinatingRejectionMetricsSpiking() throws Exception { // lower Indexing Pressure limits to trigger coordinating rejections - final String dataNode = internalCluster().startNode(Settings.builder().put(MAX_INDEXING_BYTES.getKey(), "1KB")); + final String dataNode = internalCluster().startNode(Settings.builder().put(MAX_COORDINATING_BYTES.getKey(), "1KB")); ensureStableCluster(1); final TestTelemetryPlugin plugin = internalCluster().getInstance(PluginsService.class, dataNode) @@ -307,10 +322,10 @@ public void testCoordinatingRejectionMetricsSpiking() throws Exception { public void testPrimaryDocumentRejectionMetricsArePublishing() { // setting low Indexing Pressure limits to trigger primary rejections - final String dataNode = internalCluster().startNode(Settings.builder().put(MAX_INDEXING_BYTES.getKey(), "2KB").build()); + final String dataNode = internalCluster().startNode(Settings.builder().put(MAX_PRIMARY_BYTES.getKey(), "2KB").build()); // setting high Indexing Pressure limits to pass coordinating checks final String coordinatingNode = internalCluster().startCoordinatingOnlyNode( - Settings.builder().put(MAX_INDEXING_BYTES.getKey(), "10MB").build() + Settings.builder().put(MAX_COORDINATING_BYTES.getKey(), "10MB").build() ); ensureStableCluster(2); @@ -321,8 +336,10 @@ public void testPrimaryDocumentRejectionMetricsArePublishing() { plugin.resetMeter(); final int numberOfShards = randomIntBetween(1, 5); - assertAcked(prepareCreate("test-one", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards)).get()); - assertAcked(prepareCreate("test-two", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)).get()); + assertAcked( + prepareCreate("test-one", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards)), + prepareCreate("test-two", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)) + ); final BulkRequest bulkRequestOne = new BulkRequest(); final int batchCountOne = randomIntBetween(50, 100); @@ -374,16 +391,18 @@ public void testPrimaryDocumentRejectionMetricsArePublishing() { public void testPrimaryDocumentRejectionMetricsFluctuatingOverTime() throws Exception { // setting low Indexing Pressure limits to trigger primary rejections - final String dataNode = internalCluster().startNode(Settings.builder().put(MAX_INDEXING_BYTES.getKey(), "4KB").build()); + final String dataNode = internalCluster().startNode(Settings.builder().put(MAX_PRIMARY_BYTES.getKey(), "4KB").build()); // setting high Indexing Pressure limits to pass coordinating checks final String coordinatingNode = internalCluster().startCoordinatingOnlyNode( - Settings.builder().put(MAX_INDEXING_BYTES.getKey(), "100MB").build() + Settings.builder().put(MAX_COORDINATING_BYTES.getKey(), "100MB").build() ); ensureStableCluster(2); // for simplicity do not mix small and big documents in single index/shard - assertAcked(prepareCreate("test-index-one", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)).get()); - assertAcked(prepareCreate("test-index-two", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)).get()); + assertAcked( + prepareCreate("test-index-one", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)), + prepareCreate("test-index-two", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)) + ); final TestTelemetryPlugin plugin = internalCluster().getInstance(PluginsService.class, dataNode) .filterPlugins(TestTelemetryPlugin.class) @@ -451,6 +470,211 @@ public void testPrimaryDocumentRejectionMetricsFluctuatingOverTime() throws Exce } } + // Borrowed this test from IncrementalBulkIT and added test for metrics to it + public void testIncrementalBulkLowWatermarkSplitMetrics() throws Exception { + final String nodeName = internalCluster().startNode( + Settings.builder() + .put(IndexingPressure.SPLIT_BULK_LOW_WATERMARK.getKey(), "512B") + .put(IndexingPressure.SPLIT_BULK_LOW_WATERMARK_SIZE.getKey(), "2048B") + .put(IndexingPressure.SPLIT_BULK_HIGH_WATERMARK.getKey(), "4KB") + .put(IndexingPressure.SPLIT_BULK_HIGH_WATERMARK_SIZE.getKey(), "1024B") + .build() + ); + ensureStableCluster(1); + + String index = "test"; + createIndex(index); + + IncrementalBulkService incrementalBulkService = internalCluster().getInstance(IncrementalBulkService.class, nodeName); + IndexingPressure indexingPressure = internalCluster().getInstance(IndexingPressure.class, nodeName); + final TestTelemetryPlugin testTelemetryPlugin = internalCluster().getInstance(PluginsService.class, nodeName) + .filterPlugins(TestTelemetryPlugin.class) + .findFirst() + .orElseThrow(); + testTelemetryPlugin.resetMeter(); + + IncrementalBulkService.Handler handler = incrementalBulkService.newBulkRequest(); + + AbstractRefCounted refCounted = AbstractRefCounted.of(() -> {}); + AtomicBoolean nextPage = new AtomicBoolean(false); + + IndexRequest indexRequest = indexRequest(index); + long total = indexRequest.ramBytesUsed(); + while (total < 2048) { + refCounted.incRef(); + handler.addItems(List.of(indexRequest), refCounted::decRef, () -> nextPage.set(true)); + assertTrue(nextPage.get()); + nextPage.set(false); + indexRequest = indexRequest(index); + total += indexRequest.ramBytesUsed(); + } + + assertThat(indexingPressure.stats().getCurrentCombinedCoordinatingAndPrimaryBytes(), greaterThan(0L)); + assertThat(indexingPressure.stats().getLowWaterMarkSplits(), equalTo(0L)); + + testTelemetryPlugin.collect(); + assertThat( + getSingleRecordedMetric( + testTelemetryPlugin::getLongAsyncCounterMeasurement, + "es.indexing.coordinating.low_watermark_splits.total" + ).getLong(), + equalTo(0L) + ); + assertThat( + getSingleRecordedMetric( + testTelemetryPlugin::getLongAsyncCounterMeasurement, + "es.indexing.coordinating.high_watermark_splits.total" + ).getLong(), + equalTo(0L) + ); + + refCounted.incRef(); + handler.addItems(List.of(indexRequest(index)), refCounted::decRef, () -> nextPage.set(true)); + + assertBusy(() -> assertThat(indexingPressure.stats().getCurrentCombinedCoordinatingAndPrimaryBytes(), equalTo(0L))); + assertBusy(() -> assertThat(indexingPressure.stats().getLowWaterMarkSplits(), equalTo(1L))); + assertThat(indexingPressure.stats().getHighWaterMarkSplits(), equalTo(0L)); + + testTelemetryPlugin.collect(); + assertThat( + getLatestRecordedMetric( + testTelemetryPlugin::getLongAsyncCounterMeasurement, + "es.indexing.coordinating.low_watermark_splits.total" + ).getLong(), + equalTo(1L) + ); + assertThat( + getLatestRecordedMetric( + testTelemetryPlugin::getLongAsyncCounterMeasurement, + "es.indexing.coordinating.high_watermark_splits.total" + ).getLong(), + equalTo(0L) + ); + + PlainActionFuture future = new PlainActionFuture<>(); + handler.lastItems(List.of(indexRequest), refCounted::decRef, future); + + BulkResponse bulkResponse = safeGet(future); + assertNoFailures(bulkResponse); + assertFalse(refCounted.hasReferences()); + } + + // Borrowed this test from IncrementalBulkIT and added test for metrics to it + public void testIncrementalBulkHighWatermarkSplitMetrics() throws Exception { + final String nodeName = internalCluster().startNode( + Settings.builder() + .put(IndexingPressure.SPLIT_BULK_LOW_WATERMARK.getKey(), "512B") + .put(IndexingPressure.SPLIT_BULK_LOW_WATERMARK_SIZE.getKey(), "2048B") + .put(IndexingPressure.SPLIT_BULK_HIGH_WATERMARK.getKey(), "4KB") + .put(IndexingPressure.SPLIT_BULK_HIGH_WATERMARK_SIZE.getKey(), "1024B") + .build() + ); + ensureStableCluster(1); + + String index = "test"; + createIndex(index); + + IncrementalBulkService incrementalBulkService = internalCluster().getInstance(IncrementalBulkService.class, nodeName); + IndexingPressure indexingPressure = internalCluster().getInstance(IndexingPressure.class, nodeName); + ThreadPool threadPool = internalCluster().getInstance(ThreadPool.class, nodeName); + final TestTelemetryPlugin testTelemetryPlugin = internalCluster().getInstance(PluginsService.class, nodeName) + .filterPlugins(TestTelemetryPlugin.class) + .findFirst() + .orElseThrow(); + testTelemetryPlugin.resetMeter(); + + AbstractRefCounted refCounted = AbstractRefCounted.of(() -> {}); + AtomicBoolean nextPage = new AtomicBoolean(false); + + ArrayList handlers = new ArrayList<>(); + for (int i = 0; i < 4; ++i) { + ArrayList> requests = new ArrayList<>(); + add512BRequests(requests, index); + IncrementalBulkService.Handler handler = incrementalBulkService.newBulkRequest(); + handlers.add(handler); + refCounted.incRef(); + handler.addItems(requests, refCounted::decRef, () -> nextPage.set(true)); + assertTrue(nextPage.get()); + nextPage.set(false); + } + + // Test that a request smaller than SPLIT_BULK_HIGH_WATERMARK_SIZE (1KB) is not throttled + ArrayList> requestsNoThrottle = new ArrayList<>(); + add512BRequests(requestsNoThrottle, index); + IncrementalBulkService.Handler handlerNoThrottle = incrementalBulkService.newBulkRequest(); + handlers.add(handlerNoThrottle); + refCounted.incRef(); + handlerNoThrottle.addItems(requestsNoThrottle, refCounted::decRef, () -> nextPage.set(true)); + assertTrue(nextPage.get()); + nextPage.set(false); + assertThat(indexingPressure.stats().getHighWaterMarkSplits(), equalTo(0L)); + + testTelemetryPlugin.collect(); + assertThat( + getSingleRecordedMetric( + testTelemetryPlugin::getLongAsyncCounterMeasurement, + "es.indexing.coordinating.low_watermark_splits.total" + ).getLong(), + equalTo(0L) + ); + assertThat( + getSingleRecordedMetric( + testTelemetryPlugin::getLongAsyncCounterMeasurement, + "es.indexing.coordinating.high_watermark_splits.total" + ).getLong(), + equalTo(0L) + ); + + ArrayList> requestsThrottle = new ArrayList<>(); + // Test that a request larger than SPLIT_BULK_HIGH_WATERMARK_SIZE (1KB) is throttled + add512BRequests(requestsThrottle, index); + add512BRequests(requestsThrottle, index); + + CountDownLatch finishLatch = new CountDownLatch(1); + blockWritePool(threadPool, finishLatch); + IncrementalBulkService.Handler handlerThrottled = incrementalBulkService.newBulkRequest(); + refCounted.incRef(); + handlerThrottled.addItems(requestsThrottle, refCounted::decRef, () -> nextPage.set(true)); + assertFalse(nextPage.get()); + finishLatch.countDown(); + + handlers.add(handlerThrottled); + + // Wait until we are ready for the next page + assertBusy(() -> assertTrue(nextPage.get())); + assertBusy(() -> assertThat(indexingPressure.stats().getHighWaterMarkSplits(), equalTo(1L))); + assertThat(indexingPressure.stats().getLowWaterMarkSplits(), equalTo(0L)); + + testTelemetryPlugin.collect(); + assertThat( + getLatestRecordedMetric( + testTelemetryPlugin::getLongAsyncCounterMeasurement, + "es.indexing.coordinating.low_watermark_splits.total" + ).getLong(), + equalTo(0L) + ); + assertThat( + getLatestRecordedMetric( + testTelemetryPlugin::getLongAsyncCounterMeasurement, + "es.indexing.coordinating.high_watermark_splits.total" + ).getLong(), + equalTo(1L) + ); + + for (IncrementalBulkService.Handler h : handlers) { + refCounted.incRef(); + PlainActionFuture future = new PlainActionFuture<>(); + h.lastItems(List.of(indexRequest(index)), refCounted::decRef, future); + BulkResponse bulkResponse = safeGet(future); + assertNoFailures(bulkResponse); + } + + assertBusy(() -> assertThat(indexingPressure.stats().getCurrentCombinedCoordinatingAndPrimaryBytes(), equalTo(0L))); + refCounted.decRef(); + assertFalse(refCounted.hasReferences()); + testTelemetryPlugin.collect(); + } + private static Measurement getSingleRecordedMetric(Function> metricGetter, String name) { final List measurements = metricGetter.apply(name); assertFalse("Indexing metric is not recorded", measurements.isEmpty()); @@ -468,4 +692,47 @@ private static boolean doublesEquals(double expected, double actual) { final double eps = .0000001; return Math.abs(expected - actual) < eps; } + + private static IndexRequest indexRequest(String index) { + IndexRequest indexRequest = new IndexRequest(); + indexRequest.index(index); + indexRequest.source(Map.of("field", randomAlphaOfLength(10))); + return indexRequest; + } + + private static void add512BRequests(ArrayList> requests, String index) { + long total = 0; + while (total < 512) { + IndexRequest indexRequest = indexRequest(index); + requests.add(indexRequest); + total += indexRequest.ramBytesUsed(); + } + assertThat(total, lessThan(1024L)); + } + + private static void blockWritePool(ThreadPool threadPool, CountDownLatch finishLatch) { + final var threadCount = threadPool.info(ThreadPool.Names.WRITE).getMax(); + final var startBarrier = new CyclicBarrier(threadCount + 1); + final var blockingTask = new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + fail(e); + } + + @Override + protected void doRun() { + safeAwait(startBarrier); + safeAwait(finishLatch); + } + + @Override + public boolean isForceExecution() { + return true; + } + }; + for (int i = 0; i < threadCount; i++) { + threadPool.executor(ThreadPool.Names.WRITE).execute(blockingTask); + } + safeAwait(startBarrier); + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/nodescapabilities/SimpleNodesCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/nodescapabilities/SimpleNodesCapabilitiesIT.java index eec90241fd902..494996f7f34f3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/nodescapabilities/SimpleNodesCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/nodescapabilities/SimpleNodesCapabilitiesIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nodescapabilities; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java b/server/src/internalClusterTest/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java index e4c83c81a7684..bfba0611c2f08 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.nodesinfo; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/operateAllIndices/DestructiveOperationsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/operateAllIndices/DestructiveOperationsIT.java index a851ecb11c798..6e57f320b00ba 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/operateAllIndices/DestructiveOperationsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/operateAllIndices/DestructiveOperationsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.operateAllIndices; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/persistent/PersistentTaskCreationFailureIT.java b/server/src/internalClusterTest/java/org/elasticsearch/persistent/PersistentTaskCreationFailureIT.java new file mode 100644 index 0000000000000..8a4d1ceda784b --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/persistent/PersistentTaskCreationFailureIT.java @@ -0,0 +1,228 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.persistent; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Priority; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.SettingsModule; +import org.elasticsearch.plugins.PersistentTaskPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.test.ClusterServiceUtils; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.StreamSupport; + +import static org.hamcrest.Matchers.lessThanOrEqualTo; + +public class PersistentTaskCreationFailureIT extends ESIntegTestCase { + @Override + protected Collection> nodePlugins() { + return List.of(FailingCreationPersistentTasksPlugin.class); + } + + private static boolean hasPersistentTask(ClusterState clusterState) { + return findTasks(clusterState, FailingCreationPersistentTaskExecutor.TASK_NAME).isEmpty() == false; + } + + public void testPersistentTasksThatFailDuringCreationAreRemovedFromClusterState() { + + final var masterClusterService = internalCluster().getCurrentMasterNodeInstance(ClusterService.class); + final var plugins = StreamSupport.stream(internalCluster().getInstances(PluginsService.class).spliterator(), false) + .flatMap(ps -> ps.filterPlugins(FailingCreationPersistentTasksPlugin.class)) + .toList(); + plugins.forEach(plugin -> plugin.hasFailedToCreateTask.set(false)); + + final var taskCreatedListener = ClusterServiceUtils.addTemporaryStateListener( + masterClusterService, + PersistentTaskCreationFailureIT::hasPersistentTask + ); + + taskCreatedListener.andThenAccept(v -> { + // enqueue some higher-priority cluster state updates to check that they do not cause retries of the failing task creation step + for (int i = 0; i < 5; i++) { + masterClusterService.submitUnbatchedStateUpdateTask("test", new ClusterStateUpdateTask(Priority.IMMEDIATE) { + @Override + public ClusterState execute(ClusterState currentState) { + assertTrue(hasPersistentTask(currentState)); + + assertTrue(waitUntil(() -> { + final var completePersistentTaskPendingTasksCount = masterClusterService.getMasterService() + .pendingTasks() + .stream() + .filter( + pendingClusterTask -> pendingClusterTask.getSource().string().equals("finish persistent task (failed)") + ) + .count(); + assertThat(completePersistentTaskPendingTasksCount, lessThanOrEqualTo(1L)); + return completePersistentTaskPendingTasksCount == 1L; + })); + + return currentState.copyAndUpdateMetadata( + mdb -> mdb.putCustom( + PersistentTasksCustomMetadata.TYPE, + PersistentTasksCustomMetadata.builder( + PersistentTasksCustomMetadata.getPersistentTasksCustomMetadata(currentState) + ) + // create and remove a fake task just to force a change in lastAllocationId so that + // PersistentTasksNodeService checks for changes and potentially retries + .addTask("test", "test", null, PersistentTasksCustomMetadata.INITIAL_ASSIGNMENT) + .removeTask("test") + .build() + ) + ); + } + + @Override + public void onFailure(Exception e) { + fail(e); + } + }); + } + }); + + safeAwait( + l -> internalCluster().getInstance(PersistentTasksService.class) + .sendStartRequest( + UUIDs.base64UUID(), + FailingCreationPersistentTaskExecutor.TASK_NAME, + new FailingCreationTaskParams(), + null, + l.map(ignored -> null) + ) + ); + + safeAwait( + taskCreatedListener.andThen( + (l, v) -> ClusterServiceUtils.addTemporaryStateListener( + masterClusterService, + clusterState -> hasPersistentTask(clusterState) == false + ).addListener(l) + ) + ); + + assertEquals(1L, plugins.stream().filter(plugin -> plugin.hasFailedToCreateTask.get()).count()); + } + + public static class FailingCreationPersistentTasksPlugin extends Plugin implements PersistentTaskPlugin { + + private final AtomicBoolean hasFailedToCreateTask = new AtomicBoolean(); + + @Override + public List> getPersistentTasksExecutor( + ClusterService clusterService, + ThreadPool threadPool, + Client client, + SettingsModule settingsModule, + IndexNameExpressionResolver expressionResolver + ) { + return List.of(new FailingCreationPersistentTaskExecutor(hasFailedToCreateTask)); + } + + @Override + public List getNamedWriteables() { + return List.of( + new NamedWriteableRegistry.Entry( + PersistentTaskParams.class, + FailingCreationPersistentTaskExecutor.TASK_NAME, + FailingCreationTaskParams::new + ) + ); + } + + @Override + public List getNamedXContent() { + return List.of( + new NamedXContentRegistry.Entry( + PersistentTaskParams.class, + new ParseField(FailingCreationPersistentTaskExecutor.TASK_NAME), + p -> { + p.skipChildren(); + return new FailingCreationTaskParams(); + } + ) + ); + } + } + + public static class FailingCreationTaskParams implements PersistentTaskParams { + public FailingCreationTaskParams() {} + + public FailingCreationTaskParams(StreamInput in) {} + + @Override + public String getWriteableName() { + return FailingCreationPersistentTaskExecutor.TASK_NAME; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersion.current(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException {} + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.endObject(); + return builder; + } + } + + static class FailingCreationPersistentTaskExecutor extends PersistentTasksExecutor { + static final String TASK_NAME = "cluster:admin/persistent/test_creation_failure"; + + private final AtomicBoolean hasFailedToCreateTask; + + FailingCreationPersistentTaskExecutor(AtomicBoolean hasFailedToCreateTask) { + super(TASK_NAME, r -> fail("execution is unexpected")); + this.hasFailedToCreateTask = hasFailedToCreateTask; + } + + @Override + protected AllocatedPersistentTask createTask( + long id, + String type, + String action, + TaskId parentTaskId, + PersistentTasksCustomMetadata.PersistentTask taskInProgress, + Map headers + ) { + assertTrue("already failed before", hasFailedToCreateTask.compareAndSet(false, true)); + throw new RuntimeException("simulated"); + } + + @Override + protected void nodeOperation(AllocatedPersistentTask task, FailingCreationTaskParams params, PersistentTaskState state) { + fail("execution is unexpected"); + } + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/persistent/PersistentTaskInitializationFailureIT.java b/server/src/internalClusterTest/java/org/elasticsearch/persistent/PersistentTaskInitializationFailureIT.java index ec193a37eeab7..2f08969258591 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/persistent/PersistentTaskInitializationFailureIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/persistent/PersistentTaskInitializationFailureIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.persistent; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java b/server/src/internalClusterTest/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java index 73c9495a2cd2f..b710a05ec3780 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.persistent; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java b/server/src/internalClusterTest/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java index d71718f3f3a6b..203ad831bd2e9 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.persistent; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/persistent/decider/EnableAssignmentDeciderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/persistent/decider/EnableAssignmentDeciderIT.java index caaea0a8a3846..7038dca992493 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/persistent/decider/EnableAssignmentDeciderIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/persistent/decider/EnableAssignmentDeciderIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.persistent.decider; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java index 21aa8a8770359..08482d691947a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/plugins/internal/XContentMeteringParserDecoratorIT.java b/server/src/internalClusterTest/java/org/elasticsearch/plugins/internal/XContentMeteringParserDecoratorIT.java index 16fb618e97dfc..f11c145f71f23 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/plugins/internal/XContentMeteringParserDecoratorIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/plugins/internal/XContentMeteringParserDecoratorIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.plugins.internal; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/readiness/ReadinessClusterIT.java b/server/src/internalClusterTest/java/org/elasticsearch/readiness/ReadinessClusterIT.java index 6be1612c32ad8..11d9466aaf9db 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/readiness/ReadinessClusterIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/readiness/ReadinessClusterIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.readiness; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/recovery/FullRollingRestartIT.java b/server/src/internalClusterTest/java/org/elasticsearch/recovery/FullRollingRestartIT.java index adc9db1cb3ed1..9c4e823e2ae07 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/recovery/FullRollingRestartIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/recovery/FullRollingRestartIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.recovery; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java b/server/src/internalClusterTest/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java index c066e3098df6f..30072960b39b5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.recovery; @@ -15,8 +16,8 @@ import org.elasticsearch.action.support.broadcast.BroadcastResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.OperationRouting; import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.util.CollectionUtils; @@ -364,11 +365,10 @@ private void iterateAssertCount(final int numberOfShards, final int iterations, ); } - ClusterService clusterService = clusterService(); - final ClusterState state = clusterService.state(); + final ClusterState state = clusterService().state(); for (int shard = 0; shard < numberOfShards; shard++) { for (String id : ids) { - ShardId docShard = clusterService.operationRouting().shardId(state, "test", id, null); + ShardId docShard = OperationRouting.shardId(state, "test", id, null); if (docShard.id() == shard) { final IndexShardRoutingTable indexShardRoutingTable = state.routingTable().shardRoutingTable("test", shard); for (int copy = 0; copy < indexShardRoutingTable.size(); copy++) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/recovery/RelocationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/recovery/RelocationIT.java index 52a95b2065866..fb1fabfd198e6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/recovery/RelocationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/recovery/RelocationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.recovery; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/recovery/RestartInactiveAutoExpandReplicaNotStaleIT.java b/server/src/internalClusterTest/java/org/elasticsearch/recovery/RestartInactiveAutoExpandReplicaNotStaleIT.java index a1aecab66bbfc..85f8cbc91a369 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/recovery/RestartInactiveAutoExpandReplicaNotStaleIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/recovery/RestartInactiveAutoExpandReplicaNotStaleIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.recovery; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/recovery/SimpleRecoveryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/recovery/SimpleRecoveryIT.java index baa721cbbabd2..32554cf86d034 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/recovery/SimpleRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/recovery/SimpleRecoveryIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.recovery; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java index 28c56e0cdc916..ca2ff69ac9b17 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.recovery; @@ -18,14 +19,19 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.recovery.PeerRecoveryTargetService; import org.elasticsearch.indices.recovery.RecoveryFileChunkRequest; -import org.elasticsearch.node.RecoverySettingsChunkSizePlugin; +import org.elasticsearch.indices.recovery.RecoveryFilesInfoRequest; +import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.transport.TransportService; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -33,8 +39,8 @@ import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; -import static org.elasticsearch.node.RecoverySettingsChunkSizePlugin.CHUNK_SIZE_SETTING; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -45,7 +51,7 @@ public class TruncatedRecoveryIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - return Arrays.asList(MockTransportService.TestPlugin.class, RecoverySettingsChunkSizePlugin.class); + return Arrays.asList(MockTransportService.TestPlugin.class); } /** @@ -56,7 +62,11 @@ protected Collection> nodePlugins() { */ public void testCancelRecoveryAndResume() throws Exception { updateClusterSettings( - Settings.builder().put(CHUNK_SIZE_SETTING.getKey(), new ByteSizeValue(randomIntBetween(50, 300), ByteSizeUnit.BYTES)) + Settings.builder() + .put( + RecoverySettings.INDICES_RECOVERY_CHUNK_SIZE.getKey(), + new ByteSizeValue(randomIntBetween(50, 300), ByteSizeUnit.BYTES) + ) ); NodesStatsResponse nodeStats = clusterAdmin().prepareNodesStats().get(); @@ -71,16 +81,14 @@ public void testCancelRecoveryAndResume() throws Exception { // we use 2 nodes a lucky and unlucky one // the lucky one holds the primary // the unlucky one gets the replica and the truncated leftovers - NodeStats primariesNode = dataNodeStats.get(0); - NodeStats unluckyNode = dataNodeStats.get(1); + String primariesNode = dataNodeStats.get(0).getNode().getName(); + String unluckyNode = dataNodeStats.get(1).getNode().getName(); // create the index and prevent allocation on any other nodes than the lucky one // we have no replicas so far and make sure that we allocate the primary on the lucky node assertAcked( prepareCreate("test").setMapping("field1", "type=text", "the_id", "type=text") - .setSettings( - indexSettings(numberOfShards(), 0).put("index.routing.allocation.include._name", primariesNode.getNode().getName()) - ) + .setSettings(indexSettings(numberOfShards(), 0).put("index.routing.allocation.include._name", primariesNode)) ); // only allocate on the lucky node // index some docs and check if they are coming back @@ -101,20 +109,54 @@ public void testCancelRecoveryAndResume() throws Exception { indicesAdmin().prepareFlush().setForce(true).get(); // double flush to create safe commit in case of async durability indicesAdmin().prepareForceMerge().setMaxNumSegments(1).setFlush(true).get(); + // We write some garbage into the shard directory so that we can verify that it is cleaned up before we resend. + // Cleanup helps prevent recovery from failing due to lack of space from garbage left over from a previous + // recovery that crashed during file transmission. #104473 + // We can't look for the presence of the recovery temp files themselves because they are automatically + // cleaned up on clean shutdown by MultiFileWriter. + final String GARBAGE_PREFIX = "recovery.garbage."; + final CountDownLatch latch = new CountDownLatch(1); final AtomicBoolean truncate = new AtomicBoolean(true); + + IndicesService unluckyIndices = internalCluster().getInstance(IndicesService.class, unluckyNode); + Function getUnluckyIndexPath = (shardId) -> unluckyIndices.indexService(shardId.getIndex()) + .getShard(shardId.getId()) + .shardPath() + .resolveIndex(); + for (NodeStats dataNode : dataNodeStats) { MockTransportService.getInstance(dataNode.getNode().getName()) .addSendBehavior( - internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), + internalCluster().getInstance(TransportService.class, unluckyNode), (connection, requestId, action, request, options) -> { if (action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) { RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request; logger.info("file chunk [{}] lastChunk: {}", req, req.lastChunk()); + // During the first recovery attempt (when truncate is set), write an extra garbage file once for each + // file transmitted. We get multiple chunks per file but only one is the last. + if (truncate.get() && req.lastChunk()) { + final var shardPath = getUnluckyIndexPath.apply(req.shardId()); + final var garbagePath = Files.createTempFile(shardPath, GARBAGE_PREFIX, null); + logger.info("writing garbage at: {}", garbagePath); + } if ((req.name().endsWith("cfs") || req.name().endsWith("fdt")) && req.lastChunk() && truncate.get()) { latch.countDown(); throw new RuntimeException("Caused some truncated files for fun and profit"); } + } else if (action.equals(PeerRecoveryTargetService.Actions.FILES_INFO)) { + // verify there are no garbage files present at the FILES_INFO stage of recovery. This precedes FILES_CHUNKS + // and so will run before garbage has been introduced on the first attempt, and before post-transfer cleanup + // has been performed on the second. + final var shardPath = getUnluckyIndexPath.apply(((RecoveryFilesInfoRequest) request).shardId()); + try (var list = Files.list(shardPath).filter(path -> path.getFileName().startsWith(GARBAGE_PREFIX))) { + final var garbageFiles = list.toArray(); + assertArrayEquals( + "garbage files should have been cleaned before file transmission", + new Path[0], + garbageFiles + ); + } } connection.sendRequest(requestId, action, request, options); } @@ -127,14 +169,14 @@ public void testCancelRecoveryAndResume() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) .put( "index.routing.allocation.include._name", // now allow allocation on all nodes - primariesNode.getNode().getName() + "," + unluckyNode.getNode().getName() + primariesNode + "," + unluckyNode ), "test" ); latch.await(); - // at this point we got some truncated left overs on the replica on the unlucky node + // at this point we got some truncated leftovers on the replica on the unlucky node // now we are allowing the recovery to allocate again and finish to see if we wipe the truncated files truncate.compareAndSet(true, false); ensureGreen("test"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/repositories/IndexSnapshotsServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/repositories/IndexSnapshotsServiceIT.java index f9dc42cb7abe8..c10217b93c868 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/repositories/IndexSnapshotsServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/repositories/IndexSnapshotsServiceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/repositories/InvalidRepositoryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/repositories/InvalidRepositoryIT.java index d2f567567957c..fc817d02c3c0e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/repositories/InvalidRepositoryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/repositories/InvalidRepositoryIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/repositories/RepositoriesServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/repositories/RepositoriesServiceIT.java index 1b536aa9be982..ecf910a7ab05f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/repositories/RepositoriesServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/repositories/RepositoriesServiceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreCorruptionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreCorruptionIT.java index 702eba9b2bfb8..dccf1aced6fb1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreCorruptionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreCorruptionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.blobstore; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreDynamicSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreDynamicSettingsIT.java index ba87532cedd6f..a480081dad49a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreDynamicSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreDynamicSettingsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.blobstore; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryCleanupIT.java b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryCleanupIT.java index b8ada92c9033b..3c6cb96f53185 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryCleanupIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryCleanupIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.blobstore; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryOperationPurposeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryOperationPurposeIT.java index a9f8e0563c1ff..b25fe455aa7ad 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryOperationPurposeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryOperationPurposeIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.blobstore; @@ -35,7 +36,6 @@ import java.io.InputStream; import java.io.OutputStream; import java.util.Collection; -import java.util.Iterator; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -135,11 +135,6 @@ public BlobContainer blobContainer(BlobPath path) { return new AssertingBlobContainer(delegateBlobStore.blobContainer(path)); } - @Override - public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobNames) throws IOException { - delegateBlobStore.deleteBlobsIgnoringIfNotExists(purpose, blobNames); - } - @Override public void close() throws IOException { delegateBlobStore.close(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreSizeLimitIT.java b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreSizeLimitIT.java index efc534043ac1e..a516a158c0062 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreSizeLimitIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreSizeLimitIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.blobstore; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/repositories/fs/FsBlobStoreRepositoryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/repositories/fs/FsBlobStoreRepositoryIT.java index ca8b232521178..d536632b85394 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/repositories/fs/FsBlobStoreRepositoryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/repositories/fs/FsBlobStoreRepositoryIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.repositories.fs; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/ComponentTemplatesFileSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/ComponentTemplatesFileSettingsIT.java index 82b6ba15930b2..a4ad0f8084e0a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/ComponentTemplatesFileSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/ComponentTemplatesFileSettingsIT.java @@ -1,13 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reservedstate.service; +import org.apache.lucene.tests.util.LuceneTestCase; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.template.get.GetComponentTemplateAction; @@ -25,16 +27,12 @@ import org.elasticsearch.cluster.metadata.ReservedStateMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.Strings; import org.elasticsearch.core.Tuple; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xcontent.XContentParserConfiguration; import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; @@ -53,6 +51,7 @@ import static org.hamcrest.Matchers.notNullValue; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false) +@LuceneTestCase.SuppressFileSystems("*") public class ComponentTemplatesFileSettingsIT extends ESIntegTestCase { private static AtomicLong versionCounter = new AtomicLong(1); @@ -128,8 +127,7 @@ public class ComponentTemplatesFileSettingsIT extends ESIntegTestCase { "type": "keyword" }, "created_at": { - "type": "date", - "format": "EEE MMM dd HH:mm:ss Z yyyy" + "type": "date" } } }, @@ -159,8 +157,7 @@ public class ComponentTemplatesFileSettingsIT extends ESIntegTestCase { "type": "keyword" }, "created_at": { - "type": "date", - "format": "EEE MMM dd HH:mm:ss Z yyyy" + "type": "date" } } }, @@ -190,8 +187,7 @@ public class ComponentTemplatesFileSettingsIT extends ESIntegTestCase { "type": "keyword" }, "created_at": { - "type": "date", - "format": "EEE MMM dd HH:mm:ss Z yyyy" + "type": "date" } } }, @@ -259,8 +255,7 @@ public class ComponentTemplatesFileSettingsIT extends ESIntegTestCase { "type": "keyword" }, "created_at": { - "type": "date", - "format": "EEE MMM dd HH:mm:ss Z yyyy" + "type": "date" } } }, @@ -290,8 +285,7 @@ public class ComponentTemplatesFileSettingsIT extends ESIntegTestCase { "type": "keyword" }, "created_at": { - "type": "date", - "format": "EEE MMM dd HH:mm:ss Z yyyy" + "type": "date" } } }, @@ -335,8 +329,7 @@ public class ComponentTemplatesFileSettingsIT extends ESIntegTestCase { "type": "keyword" }, "created_at": { - "type": "date", - "format": "EEE MMM dd HH:mm:ss Z yyyy" + "type": "date" } } }, @@ -364,15 +357,7 @@ private void assertMasterNode(Client client, String node) throws ExecutionExcept } private void writeJSONFile(String node, String json) throws Exception { - long version = versionCounter.incrementAndGet(); - - FileSettingsService fileSettingsService = internalCluster().getInstance(FileSettingsService.class, node); - - Files.createDirectories(fileSettingsService.watchedFileDir()); - Path tempFilePath = createTempFile(); - - Files.write(tempFilePath, Strings.format(json, version).getBytes(StandardCharsets.UTF_8)); - Files.move(tempFilePath, fileSettingsService.watchedFile(), StandardCopyOption.ATOMIC_MOVE); + FileSettingsServiceIT.writeJSONFile(node, json, logger, versionCounter.incrementAndGet()); } private Tuple setupClusterStateListener(String node) { @@ -665,8 +650,7 @@ private TransportPutComposableIndexTemplateAction.Request sampleIndexTemplateRes "type": "keyword" }, "created_at": { - "type": "date", - "format": "EEE MMM dd HH:mm:ss Z yyyy" + "type": "date" } } }, @@ -707,8 +691,7 @@ private TransportPutComposableIndexTemplateAction.Request sampleIndexTemplateRes "type": "keyword" }, "created_at": { - "type": "date", - "format": "EEE MMM dd HH:mm:ss Z yyyy" + "type": "date" } } }, diff --git a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java index 049a58b633556..90326abb381d0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java @@ -1,14 +1,16 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reservedstate.service; import org.apache.logging.log4j.Logger; +import org.apache.lucene.tests.util.LuceneTestCase; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; @@ -19,13 +21,15 @@ import org.elasticsearch.cluster.metadata.ReservedStateHandlerMetadata; import org.elasticsearch.cluster.metadata.ReservedStateMetadata; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Randomness; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Strings; import org.elasticsearch.core.Tuple; import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction; import org.elasticsearch.test.ESIntegTestCase; +import org.junit.Before; -import java.nio.charset.StandardCharsets; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; @@ -39,6 +43,7 @@ import static org.elasticsearch.test.NodeRoles.dataOnlyNode; import static org.elasticsearch.test.NodeRoles.masterNode; import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; @@ -47,9 +52,15 @@ import static org.hamcrest.Matchers.nullValue; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false) +@LuceneTestCase.SuppressFileSystems("*") public class FileSettingsServiceIT extends ESIntegTestCase { - private static final AtomicLong versionCounter = new AtomicLong(1); + private final AtomicLong versionCounter = new AtomicLong(1); + + @Before + public void resetVersionCounter() { + versionCounter.set(1); + } private static final String testJSON = """ { @@ -101,6 +112,19 @@ public class FileSettingsServiceIT extends ESIntegTestCase { } }"""; + private static final String testOtherErrorJSON = """ + { + "metadata": { + "version": "%s", + "compatibility": "8.4.0" + }, + "state": { + "bad_cluster_settings": { + "search.allow_expensive_queries": "false" + } + } + }"""; + private void assertMasterNode(Client client, String node) { assertThat( client.admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).get().getState().nodes().getMasterNode().getName(), @@ -108,19 +132,37 @@ private void assertMasterNode(Client client, String node) { ); } - public static void writeJSONFile(String node, String json, AtomicLong versionCounter, Logger logger) throws Exception { - long version = versionCounter.incrementAndGet(); - + public static void writeJSONFile(String node, String json, Logger logger, Long version) throws Exception { FileSettingsService fileSettingsService = internalCluster().getInstance(FileSettingsService.class, node); Files.createDirectories(fileSettingsService.watchedFileDir()); Path tempFilePath = createTempFile(); - String settingsFileContent = Strings.format(json, version); - Files.write(tempFilePath, settingsFileContent.getBytes(StandardCharsets.UTF_8)); - logger.info("--> Before writing new settings file with version [{}]", version); - Files.move(tempFilePath, fileSettingsService.watchedFile(), StandardCopyOption.ATOMIC_MOVE); - logger.info("--> After writing new settings file: [{}]", settingsFileContent); + String jsonWithVersion = Strings.format(json, version); + logger.info("--> before writing JSON config to node {} with path {}", node, tempFilePath); + logger.info(jsonWithVersion); + + Files.writeString(tempFilePath, jsonWithVersion); + int retryCount = 0; + do { + try { + // this can fail on Windows because of timing + Files.move(tempFilePath, fileSettingsService.watchedFile(), StandardCopyOption.ATOMIC_MOVE); + logger.info("--> after writing JSON config to node {} with path {}", node, tempFilePath); + return; + } catch (IOException e) { + logger.info("--> retrying writing a settings file [{}]", retryCount); + if (retryCount == 4) { // retry 5 times + throw e; + } + Thread.sleep(retryDelay(retryCount)); + retryCount++; + } + } while (true); + } + + private static long retryDelay(int retryCount) { + return 100 * (1 << retryCount) + Randomness.get().nextInt(10); } private Tuple setupCleanupClusterStateListener(String node) { @@ -170,7 +212,10 @@ public void clusterChanged(ClusterChangedEvent event) { private void assertClusterStateSaveOK(CountDownLatch savedClusterState, AtomicLong metadataVersion, String expectedBytesPerSec) throws Exception { assertTrue(savedClusterState.await(20, TimeUnit.SECONDS)); + assertExpectedRecoveryBytesSettingAndVersion(metadataVersion, expectedBytesPerSec); + } + private static void assertExpectedRecoveryBytesSettingAndVersion(AtomicLong metadataVersion, String expectedBytesPerSec) { final ClusterStateResponse clusterStateResponse = clusterAdmin().state( new ClusterStateRequest(TEST_REQUEST_TIMEOUT).waitForMetadataVersion(metadataVersion.get()) ).actionGet(); @@ -211,7 +256,7 @@ public void testSettingsApplied() throws Exception { assertTrue(masterFileSettingsService.watching()); assertFalse(dataFileSettingsService.watching()); - writeJSONFile(masterNode, testJSON, versionCounter, logger); + writeJSONFile(masterNode, testJSON, logger, versionCounter.incrementAndGet()); assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2(), "50mb"); } @@ -226,7 +271,7 @@ public void testSettingsAppliedOnStart() throws Exception { // In internal cluster tests, the nodes share the config directory, so when we write with the data node path // the master will pick it up on start - writeJSONFile(dataNode, testJSON, versionCounter, logger); + writeJSONFile(dataNode, testJSON, logger, versionCounter.incrementAndGet()); logger.info("--> start master node"); final String masterNode = internalCluster().startMasterOnlyNode(); @@ -254,7 +299,7 @@ public void testReservedStatePersistsOnRestart() throws Exception { assertBusy(() -> assertTrue(masterFileSettingsService.watching())); logger.info("--> write some settings"); - writeJSONFile(masterNode, testJSON, versionCounter, logger); + writeJSONFile(masterNode, testJSON, logger, versionCounter.incrementAndGet()); assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2(), "50mb"); logger.info("--> restart master"); @@ -332,10 +377,81 @@ public void testErrorSaved() throws Exception { assertTrue(masterFileSettingsService.watching()); assertFalse(dataFileSettingsService.watching()); - writeJSONFile(masterNode, testErrorJSON, versionCounter, logger); + writeJSONFile(masterNode, testErrorJSON, logger, versionCounter.incrementAndGet()); assertClusterStateNotSaved(savedClusterState.v1(), savedClusterState.v2()); } + public void testErrorCanRecoverOnRestart() throws Exception { + internalCluster().setBootstrapMasterNodeIndex(0); + logger.info("--> start data node / non master node"); + String dataNode = internalCluster().startNode(Settings.builder().put(dataOnlyNode()).put("discovery.initial_state_timeout", "1s")); + FileSettingsService dataFileSettingsService = internalCluster().getInstance(FileSettingsService.class, dataNode); + + assertFalse(dataFileSettingsService.watching()); + + logger.info("--> start master node"); + final String masterNode = internalCluster().startMasterOnlyNode( + Settings.builder().put(INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s").build() + ); + assertMasterNode(internalCluster().nonMasterClient(), masterNode); + var savedClusterState = setupClusterStateListenerForError(masterNode); + + FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode); + + assertTrue(masterFileSettingsService.watching()); + assertFalse(dataFileSettingsService.watching()); + + writeJSONFile(masterNode, testErrorJSON, logger, versionCounter.incrementAndGet()); + AtomicLong metadataVersion = savedClusterState.v2(); + assertClusterStateNotSaved(savedClusterState.v1(), metadataVersion); + assertHasErrors(metadataVersion, "not_cluster_settings"); + + // write valid json without version increment to simulate ES being able to process settings after a restart (usually, this would be + // due to a code change) + writeJSONFile(masterNode, testJSON, logger, versionCounter.get()); + internalCluster().restartNode(masterNode); + ensureGreen(); + + // we don't know the exact metadata version to wait for so rely on an assertBusy instead + assertBusy(() -> assertExpectedRecoveryBytesSettingAndVersion(metadataVersion, "50mb")); + assertBusy(() -> assertNoErrors(metadataVersion)); + } + + public void testNewErrorOnRestartReprocessing() throws Exception { + internalCluster().setBootstrapMasterNodeIndex(0); + logger.info("--> start data node / non master node"); + String dataNode = internalCluster().startNode(Settings.builder().put(dataOnlyNode()).put("discovery.initial_state_timeout", "1s")); + FileSettingsService dataFileSettingsService = internalCluster().getInstance(FileSettingsService.class, dataNode); + + assertFalse(dataFileSettingsService.watching()); + + logger.info("--> start master node"); + final String masterNode = internalCluster().startMasterOnlyNode( + Settings.builder().put(INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s").build() + ); + assertMasterNode(internalCluster().nonMasterClient(), masterNode); + var savedClusterState = setupClusterStateListenerForError(masterNode); + + FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode); + + assertTrue(masterFileSettingsService.watching()); + assertFalse(dataFileSettingsService.watching()); + + writeJSONFile(masterNode, testErrorJSON, logger, versionCounter.incrementAndGet()); + AtomicLong metadataVersion = savedClusterState.v2(); + assertClusterStateNotSaved(savedClusterState.v1(), metadataVersion); + assertHasErrors(metadataVersion, "not_cluster_settings"); + + // write json with new error without version increment to simulate ES failing to process settings after a restart for a new reason + // (usually, this would be due to a code change) + writeJSONFile(masterNode, testOtherErrorJSON, logger, versionCounter.get()); + assertHasErrors(metadataVersion, "not_cluster_settings"); + internalCluster().restartNode(masterNode); + ensureGreen(); + + assertBusy(() -> assertHasErrors(metadataVersion, "bad_cluster_settings")); + } + public void testSettingsAppliedOnMasterReElection() throws Exception { internalCluster().setBootstrapMasterNodeIndex(0); logger.info("--> start master node"); @@ -356,7 +472,7 @@ public void testSettingsAppliedOnMasterReElection() throws Exception { assertTrue(masterFileSettingsService.watching()); - writeJSONFile(masterNode, testJSON, versionCounter, logger); + writeJSONFile(masterNode, testJSON, logger, versionCounter.incrementAndGet()); assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2(), "50mb"); internalCluster().stopCurrentMasterNode(); @@ -371,15 +487,32 @@ public void testSettingsAppliedOnMasterReElection() throws Exception { ensureStableCluster(3); savedClusterState = setupCleanupClusterStateListener(internalCluster().getMasterName()); - writeJSONFile(internalCluster().getMasterName(), testCleanupJSON, versionCounter, logger); + writeJSONFile(internalCluster().getMasterName(), testCleanupJSON, logger, versionCounter.incrementAndGet()); boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS); assertTrue(awaitSuccessful); savedClusterState = setupClusterStateListener(internalCluster().getMasterName()); - writeJSONFile(internalCluster().getMasterName(), testJSON43mb, versionCounter, logger); + writeJSONFile(internalCluster().getMasterName(), testJSON43mb, logger, versionCounter.incrementAndGet()); assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2(), "43mb"); } + private void assertHasErrors(AtomicLong waitForMetadataVersion, String expectedError) { + var errorMetadata = getErrorMetadata(waitForMetadataVersion); + assertThat(errorMetadata, is(notNullValue())); + assertThat(errorMetadata.errors(), containsInAnyOrder(containsString(expectedError))); + } + + private void assertNoErrors(AtomicLong waitForMetadataVersion) { + var errorMetadata = getErrorMetadata(waitForMetadataVersion); + assertThat(errorMetadata, is(nullValue())); + } + + private ReservedStateErrorMetadata getErrorMetadata(AtomicLong waitForMetadataVersion) { + final ClusterStateResponse clusterStateResponse = clusterAdmin().state( + new ClusterStateRequest(TEST_REQUEST_TIMEOUT).waitForMetadataVersion(waitForMetadataVersion.get()) + ).actionGet(); + return clusterStateResponse.getState().getMetadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE).errorMetadata(); + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/RepositoriesFileSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/RepositoriesFileSettingsIT.java index 7cec6e895a52e..7b284979611e2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/RepositoriesFileSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/RepositoriesFileSettingsIT.java @@ -1,13 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reservedstate.service; +import org.apache.lucene.tests.util.LuceneTestCase; import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesAction; import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest; import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest; @@ -21,7 +23,6 @@ import org.elasticsearch.cluster.metadata.ReservedStateMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.Strings; import org.elasticsearch.core.Tuple; import org.elasticsearch.repositories.RepositoryMissingException; import org.elasticsearch.test.ESIntegTestCase; @@ -29,9 +30,6 @@ import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -48,6 +46,7 @@ import static org.hamcrest.Matchers.notNullValue; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false) +@LuceneTestCase.SuppressFileSystems("*") public class RepositoriesFileSettingsIT extends ESIntegTestCase { private static AtomicLong versionCounter = new AtomicLong(1); @@ -101,15 +100,7 @@ private void assertMasterNode(Client client, String node) throws ExecutionExcept } private void writeJSONFile(String node, String json) throws Exception { - long version = versionCounter.incrementAndGet(); - - FileSettingsService fileSettingsService = internalCluster().getInstance(FileSettingsService.class, node); - - Files.createDirectories(fileSettingsService.watchedFileDir()); - Path tempFilePath = createTempFile(); - - Files.write(tempFilePath, Strings.format(json, version).getBytes(StandardCharsets.UTF_8)); - Files.move(tempFilePath, fileSettingsService.watchedFile(), StandardCopyOption.ATOMIC_MOVE); + FileSettingsServiceIT.writeJSONFile(node, json, logger, versionCounter.incrementAndGet()); } private Tuple setupClusterStateListener(String node) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/SnapshotsAndFileSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/SnapshotsAndFileSettingsIT.java index 087274a86221e..7d47ed391199c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/SnapshotsAndFileSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/SnapshotsAndFileSettingsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.reservedstate.service; @@ -18,9 +19,7 @@ import org.elasticsearch.cluster.metadata.ReservedStateHandlerMetadata; import org.elasticsearch.cluster.metadata.ReservedStateMetadata; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.Randomness; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.Strings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction; @@ -28,11 +27,7 @@ import org.elasticsearch.snapshots.SnapshotState; import org.junit.After; -import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -77,34 +72,8 @@ public void cleanUp() throws Exception { awaitNoMoreRunningOperations(); } - private long retryDelay(int retryCount) { - return 100 * (1 << retryCount) + Randomness.get().nextInt(10); - } - private void writeJSONFile(String node, String json) throws Exception { - long version = versionCounter.incrementAndGet(); - - FileSettingsService fileSettingsService = internalCluster().getInstance(FileSettingsService.class, node); - - Files.createDirectories(fileSettingsService.watchedFileDir()); - Path tempFilePath = createTempFile(); - - Files.write(tempFilePath, Strings.format(json, version).getBytes(StandardCharsets.UTF_8)); - int retryCount = 0; - do { - try { - // this can fail on Windows because of timing - Files.move(tempFilePath, fileSettingsService.watchedFile(), StandardCopyOption.ATOMIC_MOVE); - return; - } catch (IOException e) { - logger.info("--> retrying writing a settings file [" + retryCount + "]"); - if (retryCount == 4) { // retry 5 times - throw e; - } - Thread.sleep(retryDelay(retryCount)); - retryCount++; - } - } while (true); + FileSettingsServiceIT.writeJSONFile(node, json, logger, versionCounter.incrementAndGet()); } private Tuple setupClusterStateListener(String node) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/rest/ChunkedZipResponseIT.java b/server/src/internalClusterTest/java/org/elasticsearch/rest/ChunkedZipResponseIT.java index be6a7387d8a0b..3566d728c2106 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/rest/ChunkedZipResponseIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/rest/ChunkedZipResponseIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.rest; @@ -268,7 +269,6 @@ private static void handleZipRestRequest( ) ), between(1, 10), - () -> {}, Releasables.wrap(refs.acquire(), chunkedZipResponse)::close ); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/rest/RestControllerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/rest/RestControllerIT.java index 2e05dbfad6085..d3d46a87b9204 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/rest/RestControllerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/rest/RestControllerIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.rest; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/rest/StreamingXContentResponseIT.java b/server/src/internalClusterTest/java/org/elasticsearch/rest/StreamingXContentResponseIT.java index ae91caea888db..13f0abcd2f8c4 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/rest/StreamingXContentResponseIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/rest/StreamingXContentResponseIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.rest; @@ -191,7 +192,6 @@ private static void handleStreamingXContentRestRequest( }) ), between(1, 10), - () -> {}, () -> { try (streamingXContentResponse; finalRef) { streamingXContentResponse.writeFragment(p -> ChunkedToXContentHelper.endObject(), refs.acquire()); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateActionIT.java index 6f5ebcf17686d..fc24a1e86bd2c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateActionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.rest.action.admin.cluster; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsActionIT.java index 7e9fa065bb4dd..e03554fe46d7a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsActionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.rest.action.admin.cluster; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java b/server/src/internalClusterTest/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java index 2ab77444f86b9..cd5dfec996731 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.rest.discovery; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/routing/AliasResolveRoutingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/routing/AliasResolveRoutingIT.java index 1e18f156f1fcf..3e1fd65177c2b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/routing/AliasResolveRoutingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/routing/AliasResolveRoutingIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.routing; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/routing/AliasRoutingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/routing/AliasRoutingIT.java index 8fb56d17b93ff..45dce5789b9bc 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/routing/AliasRoutingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/routing/AliasRoutingIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.routing; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java index 20c197bf73893..7bccf3db1284e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.routing; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/routing/SimpleRoutingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/routing/SimpleRoutingIT.java index 2eb37291d41cf..246138fa5573d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/routing/SimpleRoutingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/routing/SimpleRoutingIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.routing; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/script/StoredScriptsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/script/StoredScriptsIT.java index fe798f9233cbb..76ea5b99a2a6b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/script/StoredScriptsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/script/StoredScriptsIT.java @@ -1,26 +1,23 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.script; -import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest; import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptAction; import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest; -import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest; import org.elasticsearch.action.admin.cluster.storedscripts.TransportDeleteStoredScriptAction; import org.elasticsearch.action.admin.cluster.storedscripts.TransportPutStoredScriptAction; import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Strings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.xcontent.XContentType; import java.util.Arrays; import java.util.Collection; @@ -28,6 +25,7 @@ import java.util.Map; import java.util.function.Function; +import static org.elasticsearch.action.admin.cluster.storedscripts.StoredScriptIntegTestUtils.newPutStoredScriptTestRequest; import static org.elasticsearch.action.admin.cluster.storedscripts.StoredScriptIntegTestUtils.putJsonStoredScript; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -70,21 +68,12 @@ public void testBasics() { assertEquals( "Validation Failed: 1: id cannot contain '#' for stored script;", - asInstanceOf( + safeAwaitAndUnwrapFailure( IllegalArgumentException.class, - ExceptionsHelper.unwrapCause( - safeAwaitFailure( - AcknowledgedResponse.class, - l -> client().execute( - TransportPutStoredScriptAction.TYPE, - new PutStoredScriptRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT).id("id#") - .content(new BytesArray(Strings.format(""" - {"script": {"lang": "%s", "source": "1"} } - """, LANG)), XContentType.JSON), - l - ) - ) - ) + AcknowledgedResponse.class, + l -> client().execute(TransportPutStoredScriptAction.TYPE, newPutStoredScriptTestRequest("id#", Strings.format(""" + {"script": {"lang": "%s", "source": "1"} } + """, LANG)), l) ).getMessage() ); } @@ -92,22 +81,12 @@ public void testBasics() { public void testMaxScriptSize() { assertEquals( "exceeded max allowed stored script size in bytes [64] with size [65] for script [foobar]", - asInstanceOf( + safeAwaitAndUnwrapFailure( IllegalArgumentException.class, - ExceptionsHelper.unwrapCause( - safeAwaitFailure( - AcknowledgedResponse.class, - l -> client().execute( - TransportPutStoredScriptAction.TYPE, - new PutStoredScriptRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT).id("foobar") - .content(new BytesArray(Strings.format(""" - {"script": { "lang": "%s", "source":"0123456789abcdef"} }\ - """, LANG)), XContentType.JSON), - l - ) - - ) - ) + AcknowledgedResponse.class, + l -> client().execute(TransportPutStoredScriptAction.TYPE, newPutStoredScriptTestRequest("foobar", Strings.format(""" + {"script": { "lang": "%s", "source":"0123456789abcdef"} }\ + """, LANG)), l) ).getMessage() ); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/CollapseSearchResultsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/CollapseSearchResultsIT.java index aa721122c2160..89474a0181597 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/CollapseSearchResultsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/CollapseSearchResultsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search; @@ -13,6 +14,7 @@ import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.search.collapse.CollapseBuilder; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xcontent.XContentType; import java.util.Map; import java.util.Set; @@ -84,4 +86,33 @@ public void testCollapseWithFields() { } ); } + + public void testCollapseWithStoredFields() { + final String indexName = "test_collapse"; + createIndex(indexName); + final String collapseField = "collapse_field"; + assertAcked(indicesAdmin().preparePutMapping(indexName).setSource(""" + { + "dynamic": "strict", + "properties": { + "collapse_field": { "type": "keyword", "store": true }, + "ts": { "type": "date", "store": true } + } + } + """, XContentType.JSON)); + index(indexName, "id_1_0", Map.of(collapseField, "value1", "ts", 0)); + index(indexName, "id_1_1", Map.of(collapseField, "value1", "ts", 1)); + index(indexName, "id_2_0", Map.of(collapseField, "value2", "ts", 2)); + refresh(indexName); + + assertNoFailuresAndResponse( + prepareSearch(indexName).setQuery(new MatchAllQueryBuilder()) + .setFetchSource(false) + .storedFields("*") + .setCollapse(new CollapseBuilder(collapseField)), + searchResponse -> { + assertEquals(collapseField, searchResponse.getHits().getCollapseField()); + } + ); + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/SearchCancellationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/SearchCancellationIT.java index 58c260fb70dfb..5f8bc57dcbe09 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/SearchCancellationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/SearchCancellationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/SearchServiceCleanupOnLostMasterIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/SearchServiceCleanupOnLostMasterIT.java index b71dd4a39b198..de2b23f7fa845 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/SearchServiceCleanupOnLostMasterIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/SearchServiceCleanupOnLostMasterIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/SearchTimeoutIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/SearchTimeoutIT.java index 702d4a99df2f1..ee1aac60da9c1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/SearchTimeoutIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/SearchTimeoutIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/SearchWithRejectionsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/SearchWithRejectionsIT.java index 5030438f829fc..8081c05945da6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/SearchWithRejectionsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/SearchWithRejectionsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/StressSearchServiceReaperIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/StressSearchServiceReaperIT.java index c7aa4b3179288..1b4ed1a19014a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/StressSearchServiceReaperIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/StressSearchServiceReaperIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/AggregationsIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/AggregationsIntegrationIT.java index c051bf6656bbc..c299108d4c12d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/AggregationsIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/AggregationsIntegrationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/CombiIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/CombiIT.java index 5b8c238d7b7db..d023c9de87ca5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/CombiIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/CombiIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/EquivalenceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/EquivalenceIT.java index bd9e154c394e7..5a21b600cacd4 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/EquivalenceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/EquivalenceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/FiltersAggsRewriteIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/FiltersAggsRewriteIT.java index 3568391279a7a..a820e6e8d1747 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/FiltersAggsRewriteIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/FiltersAggsRewriteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/MetadataIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/MetadataIT.java index 3634005d37ba4..4ea502a33b194 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/MetadataIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/MetadataIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/MissingValueIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/MissingValueIT.java index 8110bc124132a..aefbc15a54d99 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/MissingValueIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/MissingValueIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/BooleanTermsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/BooleanTermsIT.java index 4a6859620563c..a3af5a4e21039 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/BooleanTermsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/BooleanTermsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java index b83b74ca8b639..a8e2ca818d3f4 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket; @@ -87,11 +88,11 @@ private static String format(ZonedDateTime date, String pattern) { private IndexRequestBuilder indexDoc(String idx, ZonedDateTime date, int value) throws Exception { return prepareIndex(idx).setSource( jsonBuilder().startObject() - .timeField("date", date) + .timestampField("date", date) .field("value", value) .startArray("dates") - .timeValue(date) - .timeValue(date.plusMonths(1).plusDays(1)) + .timestampValue(date) + .timestampValue(date.plusMonths(1).plusDays(1)) .endArray() .endObject() ); @@ -102,10 +103,10 @@ private IndexRequestBuilder indexDoc(int month, int day, int value) throws Excep jsonBuilder().startObject() .field("value", value) .field("constant", 1) - .timeField("date", date(month, day)) + .timestampField("date", date(month, day)) .startArray("dates") - .timeValue(date(month, day)) - .timeValue(date(month + 1, day + 1)) + .timestampValue(date(month, day)) + .timestampValue(date(month + 1, day + 1)) .endArray() .endObject() ); @@ -161,53 +162,53 @@ private void getMultiSortDocs(List builders) throws IOExcep for (int i = 1; i <= 3; i++) { builders.add( prepareIndex("sort_idx").setSource( - jsonBuilder().startObject().timeField("date", date(1, 1)).field("l", 1).field("d", i).endObject() + jsonBuilder().startObject().timestampField("date", date(1, 1)).field("l", 1).field("d", i).endObject() ) ); builders.add( prepareIndex("sort_idx").setSource( - jsonBuilder().startObject().timeField("date", date(1, 2)).field("l", 2).field("d", i).endObject() + jsonBuilder().startObject().timestampField("date", date(1, 2)).field("l", 2).field("d", i).endObject() ) ); } builders.add( prepareIndex("sort_idx").setSource( - jsonBuilder().startObject().timeField("date", date(1, 3)).field("l", 3).field("d", 1).endObject() + jsonBuilder().startObject().timestampField("date", date(1, 3)).field("l", 3).field("d", 1).endObject() ) ); builders.add( prepareIndex("sort_idx").setSource( - jsonBuilder().startObject().timeField("date", date(1, 3).plusHours(1)).field("l", 3).field("d", 2).endObject() + jsonBuilder().startObject().timestampField("date", date(1, 3).plusHours(1)).field("l", 3).field("d", 2).endObject() ) ); builders.add( prepareIndex("sort_idx").setSource( - jsonBuilder().startObject().timeField("date", date(1, 4)).field("l", 3).field("d", 1).endObject() + jsonBuilder().startObject().timestampField("date", date(1, 4)).field("l", 3).field("d", 1).endObject() ) ); builders.add( prepareIndex("sort_idx").setSource( - jsonBuilder().startObject().timeField("date", date(1, 4).plusHours(2)).field("l", 3).field("d", 3).endObject() + jsonBuilder().startObject().timestampField("date", date(1, 4).plusHours(2)).field("l", 3).field("d", 3).endObject() ) ); builders.add( prepareIndex("sort_idx").setSource( - jsonBuilder().startObject().timeField("date", date(1, 5)).field("l", 5).field("d", 1).endObject() + jsonBuilder().startObject().timestampField("date", date(1, 5)).field("l", 5).field("d", 1).endObject() ) ); builders.add( prepareIndex("sort_idx").setSource( - jsonBuilder().startObject().timeField("date", date(1, 5).plusHours(12)).field("l", 5).field("d", 2).endObject() + jsonBuilder().startObject().timestampField("date", date(1, 5).plusHours(12)).field("l", 5).field("d", 2).endObject() ) ); builders.add( prepareIndex("sort_idx").setSource( - jsonBuilder().startObject().timeField("date", date(1, 6)).field("l", 5).field("d", 1).endObject() + jsonBuilder().startObject().timestampField("date", date(1, 6)).field("l", 5).field("d", 1).endObject() ) ); builders.add( prepareIndex("sort_idx").setSource( - jsonBuilder().startObject().timeField("date", date(1, 7)).field("l", 5).field("d", 1).endObject() + jsonBuilder().startObject().timestampField("date", date(1, 7)).field("l", 5).field("d", 1).endObject() ) ); } @@ -996,7 +997,7 @@ public void testSingleValueWithTimeZone() throws Exception { IndexRequestBuilder[] reqs = new IndexRequestBuilder[5]; ZonedDateTime date = date("2014-03-11T00:00:00+00:00"); for (int i = 0; i < reqs.length; i++) { - reqs[i] = prepareIndex("idx2").setId("" + i).setSource(jsonBuilder().startObject().timeField("date", date).endObject()); + reqs[i] = prepareIndex("idx2").setId("" + i).setSource(jsonBuilder().startObject().timestampField("date", date).endObject()); date = date.plusHours(1); } indexRandom(true, reqs); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java index 5abf52cf37f88..778be4ee0705f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramOffsetIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket; @@ -62,7 +63,7 @@ private void prepareIndex(ZonedDateTime date, int numHours, int stepSizeHours, i IndexRequestBuilder[] reqs = new IndexRequestBuilder[numHours]; for (int i = idxIdStart; i < idxIdStart + reqs.length; i++) { reqs[i - idxIdStart] = prepareIndex("idx2").setId("" + i) - .setSource(jsonBuilder().startObject().timeField("date", date).endObject()); + .setSource(jsonBuilder().startObject().timestampField("date", date).endObject()); date = date.plusHours(stepSizeHours); } indexRandom(true, reqs); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java index 9df1fae2431f0..afa3ad9d7e737 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket; @@ -57,10 +58,10 @@ private static IndexRequestBuilder indexDoc(int month, int day, int value) throw return prepareIndex("idx").setSource( jsonBuilder().startObject() .field("value", value) - .timeField("date", date(month, day)) + .timestampField("date", date(month, day)) .startArray("dates") - .timeValue(date(month, day)) - .timeValue(date(month + 1, day + 1)) + .timestampValue(date(month, day)) + .timestampValue(date(month + 1, day + 1)) .endArray() .endObject() ); @@ -619,8 +620,8 @@ public void testScriptCaching() throws Exception { ); indexRandom( true, - prepareIndex("cache_test_idx").setId("1").setSource(jsonBuilder().startObject().timeField("date", date(1, 1)).endObject()), - prepareIndex("cache_test_idx").setId("2").setSource(jsonBuilder().startObject().timeField("date", date(2, 1)).endObject()) + prepareIndex("cache_test_idx").setId("1").setSource(jsonBuilder().startObject().timestampField("date", date(1, 1)).endObject()), + prepareIndex("cache_test_idx").setId("2").setSource(jsonBuilder().startObject().timestampField("date", date(2, 1)).endObject()) ); // Make sure we are starting with a clear cache diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DiversifiedSamplerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DiversifiedSamplerIT.java index 5a58780a24817..1894336a9cf88 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DiversifiedSamplerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DiversifiedSamplerIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java index 237f296f25751..8416ab2c09b78 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java index b5dea9cbbba49..1b70b859426d5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java index b04cb5325a82d..b030370215cd3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java index 17b976bdd3748..843e50a5a7e21 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java index 57b11df3b7d31..1ad7d1a11bea7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GlobalIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GlobalIT.java index abe93597f02b9..0d028b1e95157 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GlobalIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/GlobalIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java index d117f593348d6..2edd567221bef 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java index 887afdb578fdb..55deb9336cf5d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/IpTermsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/IpTermsIT.java index 7f9cb01599a00..462f8cb80d5a0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/IpTermsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/IpTermsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java index cbb3850422800..2ae04cb7a4983 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/MinDocCountIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/MinDocCountIT.java index 4e93bf578cc87..eb353cde86d27 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/MinDocCountIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/MinDocCountIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/NaNSortingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/NaNSortingIT.java index e6d0b6a1f9f1c..5e642a71372b1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/NaNSortingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/NaNSortingIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java index 1cce1ab3b1c4c..72f1b0cc56b25 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java @@ -1,13 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket; import org.apache.lucene.search.join.ScoreMode; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.index.query.InnerHitBuilder; @@ -17,11 +19,15 @@ import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.bucket.filter.Filter; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; +import org.elasticsearch.search.aggregations.bucket.nested.InternalNested; import org.elasticsearch.search.aggregations.bucket.nested.Nested; +import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.terms.LongTerms; import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.Max; import org.elasticsearch.search.aggregations.metrics.Stats; import org.elasticsearch.search.aggregations.metrics.Sum; @@ -889,4 +895,87 @@ public void testSyntheticSource() throws Exception { assertEquals("a", nested.get("number")); }); } + + public void testScoring() throws Exception { + assertAcked( + prepareCreate("scoring").setMapping( + jsonBuilder().startObject() + .startObject("properties") + .startObject("tags") + .field("type", "nested") + .startObject("properties") + .startObject("key") + .field("type", "keyword") + .endObject() + .startObject("value") + .field("type", "keyword") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + ) + ); + ensureGreen("scoring"); + + prepareIndex("scoring").setId("1") + .setSource( + jsonBuilder().startObject() + .startArray("tags") + .startObject() + .field("key", "state") + .field("value", "texas") + .endObject() + .endArray() + .endObject() + ) + .get(); + refresh("scoring"); + prepareIndex("scoring").setId("2") + .setSource( + jsonBuilder().startObject() + .startArray("tags") + .startObject() + .field("key", "state") + .field("value", "utah") + .endObject() + .endArray() + .endObject() + ) + .get(); + refresh("scoring"); + prepareIndex("scoring").setId("3") + .setSource( + jsonBuilder().startObject() + .startArray("tags") + .startObject() + .field("key", "state") + .field("value", "texas") + .endObject() + .endArray() + .endObject() + ) + .get(); + refresh("scoring"); + + assertResponse( + client().prepareSearch("scoring") + .setSize(0) + .addAggregation( + new NestedAggregationBuilder("tags", "tags").subAggregation( + new TermsAggregationBuilder("keys").field("tags.key") + .executionHint("map") + .subAggregation(new TermsAggregationBuilder("values").field("tags.value")) + .subAggregation(new CardinalityAggregationBuilder("values_count").field("tags.value")) + ) + ), + searchResponse -> { + InternalNested nested = searchResponse.getAggregations().get("tags"); + assertThat(nested.getDocCount(), equalTo(3L)); + assertThat(nested.getAggregations().asList().size(), equalTo(1)); + } + ); + + assertAcked(indicesAdmin().delete(new DeleteIndexRequest("scoring")).get()); + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RandomSamplerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RandomSamplerIT.java index f8ace23057ea6..544f0a08eaa6c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RandomSamplerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RandomSamplerIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java index 0c39859856d56..8b63efd92a648 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java index 7585ed42da830..0cb85217d6636 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/SamplerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/SamplerIT.java index 00779ba9b256e..6742c0ad7854e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/SamplerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/SamplerIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/ShardReduceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/ShardReduceIT.java index 94db8926f59e7..f214dd0a6bda2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/ShardReduceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/ShardReduceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTermsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTermsIT.java index b8a1b3df8cf60..28b65e68b950b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTermsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTermsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java index 5fa010e4b091e..671f60e2b9d5e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket; @@ -494,7 +495,7 @@ public void testScriptScore() throws ExecutionException, InterruptedException, I for (SignificantTerms.Bucket bucket : sigTerms.getBuckets()) { assertThat( bucket.getSignificanceScore(), - is((double) bucket.getSubsetDf() + bucket.getSubsetSize() + bucket.getSupersetDf() + bucket.getSupersetSize()) + is((double) bucket.getSubsetDf() + sigTerms.getSubsetSize() + bucket.getSupersetDf() + sigTerms.getSupersetSize()) ); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java index 25d6dfb850bbc..a6c01852e2f16 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java index c8e23d65b4e37..2c4da7c9c97c2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsIT.java index 69b84c6b98286..841efb3109d57 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket.terms; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsIT.java index fa9a9ef2a7f41..99fc784ce70eb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.bucket.terms; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java index 5e086766dc711..fe8e850999faf 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.metrics; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java index fde18fb283a6e..5e2a44285e8fa 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.metrics; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java index f8b633dca1a10..73654d4f432d7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.metrics; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java index a7d32863718e3..03269debdceb2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.metrics; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java index a4da7c4e893be..762bc5bdfaf39 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.metrics; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java index 43e4aecb07f7f..12ed0a5c1a8e0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.metrics; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationIT.java index 4ded290f93961..1232a61fac2cb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.metrics; @@ -128,7 +129,7 @@ protected Collection> nodePlugins() { private static MedianAbsoluteDeviationAggregationBuilder randomBuilder() { final MedianAbsoluteDeviationAggregationBuilder builder = new MedianAbsoluteDeviationAggregationBuilder("mad"); if (randomBoolean()) { - builder.compression(randomDoubleBetween(25, 1000, false)); + builder.compression(randomDoubleBetween(30, 1000, false)); } return builder; } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java index 042c8c1fb0e35..52425ae1d9f17 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.metrics; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java index 78adca3377f0b..fbe70ec2a40d6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.metrics; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/SumIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/SumIT.java index fd173b8f48a12..2a8be6b4244dd 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/SumIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/SumIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.metrics; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java index 9c11b6cd14d54..2877f8882d6d6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.metrics; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java index c67a237b2fc17..bbcf7b191fe1b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.metrics; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java index 42f04ff54c82a..980252e6b3839 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.metrics; @@ -118,10 +119,10 @@ public static String randomExecutionHint() { @Override public void setupSuiteScopeCluster() throws Exception { - assertAcked(prepareCreate("idx").setMapping(TERMS_AGGS_FIELD, "type=keyword", "text", "type=text,store=true")); - assertAcked(prepareCreate("field-collapsing").setMapping("group", "type=keyword")); - createIndex("empty"); assertAcked( + prepareCreate("idx").setMapping(TERMS_AGGS_FIELD, "type=keyword", "text", "type=text,store=true"), + prepareCreate("field-collapsing").setMapping("group", "type=keyword"), + prepareCreate("empty"), prepareCreate("articles").setMapping( jsonBuilder().startObject() .startObject("_doc") diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java index 445ad8e0b9b11..3dee7a8d6e92f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.metrics; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketIT.java index a9e3c00e57e4d..91eed5ebc24b2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.pipeline; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsPipeLineAggregationTestCase.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsPipeLineAggregationTestCase.java index 3c9fbca476c0d..df5c75dd30818 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsPipeLineAggregationTestCase.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsPipeLineAggregationTestCase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.pipeline; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java index bc8142a629f7c..615f3854833cb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.pipeline; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketIT.java index 0da75854b9ab2..8b026c77c0165 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.pipeline; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketIT.java index 4e7e0fba0f0eb..ec60b9ca7fda0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.pipeline; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/MinBucketIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/MinBucketIT.java index 0dc3e8c87e819..2bec8797b31fb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/MinBucketIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/MinBucketIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.pipeline; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketIT.java index b4193b8f90e1f..fff62294e5685 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.pipeline; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketIT.java index cd87bd98a0926..5362e599d1aa7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.pipeline; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/SumBucketIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/SumBucketIT.java index 285322ef12355..4f9b3b7aa3c42 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/SumBucketIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/SumBucketIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.aggregations.pipeline; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/QueryRewriteContextIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/QueryRewriteContextIT.java index 753a0a62bce5e..965fd0d8011c0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/QueryRewriteContextIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/QueryRewriteContextIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.basic; @@ -153,8 +154,10 @@ public void testResolvedIndices_TransportSearchAction() { public void testResolvedIndices_TransportExplainAction() { final String[] indices = { "test1", "test2" }; createIndex(indices); - assertAcked(indicesAdmin().prepareAliases().addAlias("test1", "alias1")); - assertAcked(indicesAdmin().prepareAliases().addAlias(indices, "alias2")); + assertAcked( + indicesAdmin().prepareAliases().addAlias("test1", "alias1"), + indicesAdmin().prepareAliases().addAlias(indices, "alias2") + ); assertResolvedIndices(client().prepareExplain("test1", "1"), Set.of("test1"), Set.of("test1"), r -> {}); assertResolvedIndices(client().prepareExplain("alias1", "1"), Set.of("alias1"), Set.of("test1"), r -> {}); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchRedStateIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchRedStateIndexIT.java index 9e2139f832f15..ac549d0cd101d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchRedStateIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchRedStateIndexIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.basic; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java index df6994c57f425..3263be081a6f7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.basic; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java index a9b0f75fe45ba..4028a3633d955 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.basic; @@ -63,6 +64,8 @@ private void testSearchAndRelocateConcurrently(final int numberOfReplicas) throw } indexRandom(true, indexBuilders.toArray(new IndexRequestBuilder[indexBuilders.size()])); assertHitCount(prepareSearch(), (numDocs)); + // hold a copy of the node names before a new node is potentially added later + String[] nodeNamesBeforeClusterResize = internalCluster().getNodeNames(); final int numIters = scaledRandomIntBetween(5, 20); for (int i = 0; i < numIters; i++) { final AtomicBoolean stop = new AtomicBoolean(false); @@ -75,34 +78,37 @@ private void testSearchAndRelocateConcurrently(final int numberOfReplicas) throw public void run() { try { while (stop.get() == false) { - assertResponse(prepareSearch().setSize(numDocs), response -> { - if (response.getHits().getTotalHits().value != numDocs) { - // if we did not search all shards but had no serious failures that is potentially fine - // if only the hit-count is wrong. this can happen if the cluster-state is behind when the - // request comes in. It's a small window but a known limitation. - if (response.getTotalShards() != response.getSuccessfulShards() - && Stream.of(response.getShardFailures()) - .allMatch(ssf -> ssf.getCause() instanceof NoShardAvailableActionException)) { - nonCriticalExceptions.add( - "Count is " - + response.getHits().getTotalHits().value - + " but " - + numDocs - + " was expected. " - + formatShardStatus(response) - ); - } else { - assertHitCount(response, numDocs); + assertResponse( + client(randomFrom(nodeNamesBeforeClusterResize)).prepareSearch().setSize(numDocs), + response -> { + if (response.getHits().getTotalHits().value != numDocs) { + // if we did not search all shards but had no serious failures that is potentially fine + // if only the hit-count is wrong. this can happen if the cluster-state is behind when the + // request comes in. It's a small window but a known limitation. + if (response.getTotalShards() != response.getSuccessfulShards() + && Stream.of(response.getShardFailures()) + .allMatch(ssf -> ssf.getCause() instanceof NoShardAvailableActionException)) { + nonCriticalExceptions.add( + "Count is " + + response.getHits().getTotalHits().value + + " but " + + numDocs + + " was expected. " + + formatShardStatus(response) + ); + } else { + assertHitCount(response, numDocs); + } } - } - final SearchHits sh = response.getHits(); - assertThat( - "Expected hits to be the same size the actual hits array", - sh.getTotalHits().value, - equalTo((long) (sh.getHits().length)) - ); - }); + final SearchHits sh = response.getHits(); + assertThat( + "Expected hits to be the same size the actual hits array", + sh.getTotalHits().value, + equalTo((long) (sh.getHits().length)) + ); + } + ); // this is the more critical but that we hit the actual hit array has a different size than the // actual number of hits. } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomDisconnectsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomDisconnectsIT.java new file mode 100644 index 0000000000000..d2c7e10f8aa62 --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomDisconnectsIT.java @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.search.basic; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.discovery.AbstractDisruptionTestCase; +import org.elasticsearch.index.IndexModule; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.test.disruption.NetworkDisruption; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.IntStream; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; + +public class SearchWithRandomDisconnectsIT extends AbstractDisruptionTestCase { + + public void testSearchWithRandomDisconnects() throws InterruptedException, ExecutionException { + // make sure we have a couple data nodes + int minDataNodes = randomIntBetween(3, 7); + internalCluster().ensureAtLeastNumDataNodes(minDataNodes); + final int indexCount = randomIntBetween(minDataNodes, 10 * minDataNodes); + final String[] indexNames = IntStream.range(0, indexCount).mapToObj(i -> "test-" + i).toArray(String[]::new); + final Settings indexSettings = indexSettings(1, 0).put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false) + .put(IndexSettings.INDEX_CHECK_ON_STARTUP.getKey(), false) + .build(); + for (String indexName : indexNames) { + createIndex(indexName, indexSettings); + } + BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); + for (String indexName : indexNames) { + for (int i = 0; i < randomIntBetween(1, 10); i++) { + bulkRequestBuilder = bulkRequestBuilder.add(prepareIndex(indexName).setCreate(false).setSource("foo", "bar-" + i)); + } + } + assertFalse(bulkRequestBuilder.get().hasFailures()); + final AtomicBoolean done = new AtomicBoolean(); + final int concurrentSearches = randomIntBetween(2, 5); + final List> futures = new ArrayList<>(concurrentSearches); + for (int i = 0; i < concurrentSearches; i++) { + final PlainActionFuture finishFuture = new PlainActionFuture<>(); + futures.add(finishFuture); + prepareRandomSearch().execute(new ActionListener<>() { + @Override + public void onResponse(SearchResponse searchResponse) { + runMoreSearches(); + } + + @Override + public void onFailure(Exception e) { + runMoreSearches(); + } + + private void runMoreSearches() { + if (done.get() == false) { + prepareRandomSearch().execute(this); + } else { + finishFuture.onResponse(null); + } + } + }); + } + for (int i = 0, n = randomIntBetween(50, 100); i < n; i++) { + NetworkDisruption networkDisruption = new NetworkDisruption( + isolateNode(internalCluster().getRandomNodeName()), + NetworkDisruption.DISCONNECT + ); + setDisruptionScheme(networkDisruption); + networkDisruption.startDisrupting(); + networkDisruption.stopDisrupting(); + internalCluster().clearDisruptionScheme(); + ensureFullyConnectedCluster(); + } + done.set(true); + for (PlainActionFuture future : futures) { + future.get(); + } + ensureGreen(DISRUPTION_HEALING_OVERHEAD, indexNames); + assertAcked(indicesAdmin().prepareDelete(indexNames)); + } + + private static SearchRequestBuilder prepareRandomSearch() { + return prepareSearch("*").setQuery(new MatchAllQueryBuilder()) + .setSize(9999) + .setFetchSource(true) + .setAllowPartialSearchResults(randomBoolean()); + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java index 6985ebb17386c..dba42bf90a69f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.basic; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java index 3fe93f8d91be3..ed02594994b39 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.basic; @@ -41,7 +42,6 @@ protected Collection> nodePlugins() { return Arrays.asList(MockFSIndexStore.TestPlugin.class); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/106752") public void testRandomDirectoryIOExceptions() throws IOException, InterruptedException, ExecutionException { String mapping = Strings.toString( XContentFactory.jsonBuilder() diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/TransportSearchFailuresIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/TransportSearchFailuresIT.java index 951ea29a09e8e..4a93fbf2cd094 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/TransportSearchFailuresIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/TransportSearchFailuresIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.basic; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/TransportTwoNodesSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/TransportTwoNodesSearchIT.java index d4a4debbd61d6..1745ad82931ba 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/TransportTwoNodesSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/TransportTwoNodesSearchIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.basic; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSCanMatchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSCanMatchIT.java index 582df3a5bb396..3f354baace85a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSCanMatchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSCanMatchIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.ccs; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java index 23146d1907250..0706894fbef9f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.ccs; @@ -24,7 +25,6 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; -import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.plugins.Plugin; @@ -37,19 +37,12 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.test.AbstractMultiClustersTestCase; import org.elasticsearch.test.InternalTestCluster; +import org.elasticsearch.test.SkipUnavailableRule; +import org.elasticsearch.test.SkipUnavailableRule.NotSkipped; import org.elasticsearch.usage.UsageService; import org.junit.Assert; -import org.junit.BeforeClass; import org.junit.Rule; -import org.junit.rules.TestRule; -import org.junit.runner.Description; -import org.junit.runners.model.Statement; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; -import java.util.Arrays; + import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -57,11 +50,10 @@ import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.stream.Collectors; import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.ASYNC_FEATURE; import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.MRT_FEATURE; +import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.PIT_FEATURE; import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.WILDCARD_FEATURE; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; @@ -71,7 +63,6 @@ public class CCSUsageTelemetryIT extends AbstractMultiClustersTestCase { private static final Logger LOGGER = LogManager.getLogger(CCSUsageTelemetryIT.class); private static final String REMOTE1 = "cluster-a"; private static final String REMOTE2 = "cluster-b"; - private static final FeatureFlag CCS_TELEMETRY_FEATURE_FLAG = new FeatureFlag("ccs_telemetry"); @Override protected boolean reuseClusters() { @@ -86,11 +77,6 @@ protected Collection remoteClusterAlias() { @Rule public SkipUnavailableRule skipOverride = new SkipUnavailableRule(REMOTE1, REMOTE2); - @BeforeClass - protected static void skipIfTelemetryDisabled() { - assumeTrue("Skipping test as CCS_TELEMETRY_FEATURE_FLAG is disabled", CCS_TELEMETRY_FEATURE_FLAG.isEnabled()); - } - @Override protected Map skipUnavailableForRemoteClusters() { var map = skipOverride.getMap(); @@ -504,7 +490,7 @@ public void testRemoteOnlyTimesOut() throws Exception { assertThat(perCluster.get(REMOTE2), equalTo(null)); } - @SkipOverride(aliases = { REMOTE1 }) + @NotSkipped(aliases = { REMOTE1 }) public void testRemoteTimesOutFailure() throws Exception { Map testClusterInfo = setupClusters(); String remoteIndex = (String) testClusterInfo.get("remote.index"); @@ -534,7 +520,7 @@ public void testRemoteTimesOutFailure() throws Exception { /** * Search when all the remotes failed and not skipped */ - @SkipOverride(aliases = { REMOTE1, REMOTE2 }) + @NotSkipped(aliases = { REMOTE1, REMOTE2 }) public void testFailedAllRemotesSearch() throws Exception { Map testClusterInfo = setupClusters(); String localIndex = (String) testClusterInfo.get("local.index"); @@ -583,7 +569,7 @@ public void testRemoteHasNoIndex() throws Exception { /** * Test that we're still counting remote search even if remote cluster has no such index */ - @SkipOverride(aliases = { REMOTE1 }) + @NotSkipped(aliases = { REMOTE1 }) public void testRemoteHasNoIndexFailure() throws Exception { SearchRequest searchRequest = makeSearchRequest(REMOTE1 + ":no_such_index"); CCSTelemetrySnapshot telemetry = getTelemetryFromFailedSearch(searchRequest); @@ -621,6 +607,7 @@ public void testPITSearch() throws ExecutionException, InterruptedException { assertThat(telemetry.getTotalCount(), equalTo(2L)); assertThat(telemetry.getSuccessCount(), equalTo(2L)); + assertThat(telemetry.getFeatureCounts().get(PIT_FEATURE), equalTo(2L)); } public void testCompoundRetrieverSearch() throws ExecutionException, InterruptedException { @@ -659,7 +646,7 @@ private Map setupClusters() { int numShardsRemote = randomIntBetween(2, 10); for (String clusterAlias : remoteClusterAlias()) { final InternalTestCluster remoteCluster = cluster(clusterAlias); - remoteCluster.ensureAtLeastNumDataNodes(randomIntBetween(1, 3)); + remoteCluster.ensureAtLeastNumDataNodes(randomIntBetween(2, 3)); assertAcked( client(clusterAlias).admin() .indices() @@ -694,40 +681,4 @@ private int indexDocs(Client client, String index) { return numDocs; } - /** - * Annotation to mark specific cluster in a test as not to be skipped when unavailable - */ - @Retention(RetentionPolicy.RUNTIME) - @Target(ElementType.METHOD) - @interface SkipOverride { - String[] aliases(); - } - - /** - * Test rule to process skip annotations - */ - static class SkipUnavailableRule implements TestRule { - private final Map skipMap; - - SkipUnavailableRule(String... clusterAliases) { - this.skipMap = Arrays.stream(clusterAliases).collect(Collectors.toMap(Function.identity(), alias -> true)); - } - - public Map getMap() { - return skipMap; - } - - @Override - public Statement apply(Statement base, Description description) { - // Check for annotation named "SkipOverride" and set the overrides accordingly - var aliases = description.getAnnotation(SkipOverride.class); - if (aliases != null) { - for (String alias : aliases.aliases()) { - skipMap.put(alias, false); - } - } - return base; - } - - } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterIT.java index 9cc359f40d327..bed961658694d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.ccs; @@ -62,6 +63,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -181,14 +183,17 @@ public void testProxyConnectionDisconnect() throws Exception { } } }); - assertBusy(() -> assertTrue(future.isDone())); + try { + future.get(); + } catch (ExecutionException e) { + // ignored + } configureAndConnectsToRemoteClusters(); } finally { SearchListenerPlugin.allowQueryPhase(); } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/108061") public void testCancel() throws Exception { assertAcked(client(LOCAL_CLUSTER).admin().indices().prepareCreate("demo")); indexDocs(client(LOCAL_CLUSTER), "demo"); @@ -297,20 +302,21 @@ public void testCancel() throws Exception { } SearchListenerPlugin.allowQueryPhase(); - assertBusy(() -> assertTrue(queryFuture.isDone())); - assertBusy(() -> assertTrue(cancelFuture.isDone())); + try { + queryFuture.get(); + fail("query should have failed"); + } catch (ExecutionException e) { + assertNotNull(e.getCause()); + Throwable t = ExceptionsHelper.unwrap(e, TaskCancelledException.class); + assertNotNull(t); + } + cancelFuture.get(); assertBusy(() -> { final Iterable transportServices = cluster("cluster_a").getInstances(TransportService.class); for (TransportService transportService : transportServices) { assertThat(transportService.getTaskManager().getBannedTaskIds(), Matchers.empty()); } }); - - RuntimeException e = expectThrows(RuntimeException.class, () -> queryFuture.result()); - assertNotNull(e); - assertNotNull(e.getCause()); - Throwable t = ExceptionsHelper.unwrap(e, TaskCancelledException.class); - assertNotNull(t); } /** diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchIT.java index e772f94e868ae..2feded3013636 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.ccs; @@ -39,11 +40,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; +import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -211,7 +214,7 @@ public void testCCSClusterDetailsWhereAllShardsSkippedInCanMatch() throws Except // with DFS_QUERY_THEN_FETCH, the local shards are never skipped assertThat(localClusterSearchInfo.getSkippedShards(), equalTo(0)); } else { - assertThat(localClusterSearchInfo.getSkippedShards(), equalTo(localNumShards - 1)); + assertThat(localClusterSearchInfo.getSkippedShards(), equalTo(localNumShards)); } assertThat(localClusterSearchInfo.getFailedShards(), equalTo(0)); assertThat(localClusterSearchInfo.getFailures().size(), equalTo(0)); @@ -221,7 +224,7 @@ public void testCCSClusterDetailsWhereAllShardsSkippedInCanMatch() throws Except assertThat(remoteClusterSearchInfo.getTotalShards(), equalTo(remoteNumShards)); assertThat(remoteClusterSearchInfo.getSuccessfulShards(), equalTo(remoteNumShards)); if (clusters.isCcsMinimizeRoundtrips()) { - assertThat(remoteClusterSearchInfo.getSkippedShards(), equalTo(remoteNumShards - 1)); + assertThat(remoteClusterSearchInfo.getSkippedShards(), equalTo(remoteNumShards)); } else { assertThat(remoteClusterSearchInfo.getSkippedShards(), equalTo(remoteNumShards)); } @@ -376,11 +379,9 @@ public void testClusterDetailsAfterCCSWithFailuresOnRemoteClusterOnly() throws E r.incRef(); l.onResponse(r); })); - assertBusy(() -> assertTrue(queryFuture.isDone())); - // dfs=true overrides the minimize_roundtrips=true setting and does not minimize roundtrips if (skipUnavailable == false && minimizeRoundtrips && dfs == false) { - ExecutionException ee = expectThrows(ExecutionException.class, () -> queryFuture.get()); + ExecutionException ee = expectThrows(ExecutionException.class, queryFuture::get); assertNotNull(ee.getCause()); assertThat(ee.getCause(), instanceOf(RemoteTransportException.class)); Throwable rootCause = ExceptionsHelper.unwrap(ee.getCause(), IllegalStateException.class); @@ -619,10 +620,8 @@ public void testRemoteClusterOnlyCCSWithFailuresOnAllShards() throws Exception { r.incRef(); l.onResponse(r); })); - assertBusy(() -> assertTrue(queryFuture.isDone())); - if (skipUnavailable == false || minimizeRoundtrips == false) { - ExecutionException ee = expectThrows(ExecutionException.class, () -> queryFuture.get()); + ExecutionException ee = expectThrows(ExecutionException.class, queryFuture::get); assertNotNull(ee.getCause()); Throwable rootCause = ExceptionsHelper.unwrap(ee, IllegalStateException.class); assertThat(rootCause.getMessage(), containsString("index corrupted")); @@ -661,6 +660,161 @@ public void testRemoteClusterOnlyCCSWithFailuresOnAllShards() throws Exception { } } + public void testDateMathIndexes() throws ExecutionException, InterruptedException { + Map testClusterInfo = setupTwoClusters( + new String[] { "datemath-2001-01-01-14", "datemath-2001-01-01-15" }, + new String[] { "remotemath-2001-01-01-14", "remotemath-2001-01-01-15" } + ); + SearchRequest searchRequest = new SearchRequest( + REMOTE_CLUSTER + ":", + "" + ); + searchRequest.setCcsMinimizeRoundtrips(randomBoolean()); + searchRequest.allowPartialSearchResults(false); + searchRequest.source(new SearchSourceBuilder().query(new MatchAllQueryBuilder()).size(5000)); + assertResponse(client(LOCAL_CLUSTER).search(searchRequest), response -> { + assertNotNull(response); + Clusters clusters = response.getClusters(); + assertFalse("search cluster results should NOT be marked as partial", clusters.hasPartialResults()); + assertThat(clusters.getTotal(), equalTo(2)); + Cluster localClusterSearchInfo = clusters.getCluster(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); + assertNotNull(localClusterSearchInfo); + Cluster remoteClusterSearchInfo = clusters.getCluster(REMOTE_CLUSTER); + assertNotNull(remoteClusterSearchInfo); + assertThat(Objects.requireNonNull(response.getHits().getTotalHits()).value, greaterThan(2L)); + for (var hit : response.getHits()) { + assertThat(hit.getIndex(), anyOf(equalTo("datemath-2001-01-01-14"), equalTo("remotemath-2001-01-01-14"))); + } + }); + } + + /** + * Test for issue https://github.com/elastic/elasticsearch/issues/112243 + */ + public void testDateMathNegativeIndexesLocal() throws ExecutionException, InterruptedException { + Map testClusterInfo = setupTwoClusters( + new String[] { "datemath-2001-01-01-14", "datemath-2001-01-01-15" }, + new String[] { "datemath-2001-01-01-14", "datemath-2001-01-01-15" } + ); + SearchRequest searchRequest = new SearchRequest("da*", "-"); + searchRequest.allowPartialSearchResults(false); + searchRequest.source(new SearchSourceBuilder().query(new MatchAllQueryBuilder()).size(5000)); + assertResponse(client(LOCAL_CLUSTER).search(searchRequest), response -> { + assertNotNull(response); + Clusters clusters = response.getClusters(); + assertFalse("search cluster results should NOT be marked as partial", clusters.hasPartialResults()); + assertThat(clusters.getTotal(), equalTo(0)); + for (var hit : response.getHits()) { + assertThat(hit.getIndex(), equalTo("datemath-2001-01-01-15")); + } + }); + } + + /** + * Test for issue https://github.com/elastic/elasticsearch/issues/112243 + */ + public void testDateMathNegativeIndexesRemote() throws ExecutionException, InterruptedException { + Map testClusterInfo = setupTwoClusters( + new String[] { "datemath-2001-01-01-14", "datemath-2001-01-01-15" }, + new String[] { "datemath-2001-01-01-14", "datemath-2001-01-01-15" } + ); + SearchRequest searchRequest = new SearchRequest( + REMOTE_CLUSTER + ":*", + REMOTE_CLUSTER + ":-" + ); + searchRequest.setCcsMinimizeRoundtrips(randomBoolean()); + searchRequest.allowPartialSearchResults(false); + searchRequest.source(new SearchSourceBuilder().query(new MatchAllQueryBuilder()).size(5000)); + assertResponse(client(LOCAL_CLUSTER).search(searchRequest), response -> { + assertNotNull(response); + Clusters clusters = response.getClusters(); + assertFalse("search cluster results should NOT be marked as partial", clusters.hasPartialResults()); + assertThat(clusters.getTotal(), equalTo(1)); + Cluster localClusterSearchInfo = clusters.getCluster(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); + assertNull(localClusterSearchInfo); + Cluster remoteClusterSearchInfo = clusters.getCluster(REMOTE_CLUSTER); + assertNotNull(remoteClusterSearchInfo); + for (var hit : response.getHits()) { + assertThat(hit.getIndex(), equalTo("datemath-2001-01-01-15")); + } + }); + } + + public void testNegativeRemoteIndexNameThrows() { + SearchRequest searchRequest = new SearchRequest("*:*", "-" + REMOTE_CLUSTER + ":prod"); + searchRequest.setCcsMinimizeRoundtrips(true); + searchRequest.allowPartialSearchResults(false); + searchRequest.source(new SearchSourceBuilder().query(new MatchAllQueryBuilder()).size(5000)); + var queryFuture = client(LOCAL_CLUSTER).search(searchRequest); + // This should throw the wildcard error + ExecutionException ee = expectThrows(ExecutionException.class, queryFuture::get); + assertNotNull(ee.getCause()); + } + + public void testClusterDetailsWhenLocalClusterHasNoMatchingIndex() throws Exception { + Map testClusterInfo = setupTwoClusters(); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + int remoteNumShards = (Integer) testClusterInfo.get("remote.num_shards"); + + SearchRequest searchRequest = new SearchRequest("nomatch*", REMOTE_CLUSTER + ":" + remoteIndex); + if (randomBoolean()) { + searchRequest = searchRequest.scroll(TimeValue.timeValueMinutes(1)); + } + + searchRequest.allowPartialSearchResults(false); + if (randomBoolean()) { + searchRequest.setBatchedReduceSize(randomIntBetween(3, 20)); + } + + boolean minimizeRoundtrips = false; + searchRequest.setCcsMinimizeRoundtrips(minimizeRoundtrips); + + boolean dfs = randomBoolean(); + if (dfs) { + searchRequest.searchType(SearchType.DFS_QUERY_THEN_FETCH); + } + + if (randomBoolean()) { + searchRequest.setPreFilterShardSize(1); + } + + searchRequest.source(new SearchSourceBuilder().query(new MatchAllQueryBuilder()).size(10)); + assertResponse(client(LOCAL_CLUSTER).search(searchRequest), response -> { + assertNotNull(response); + + Clusters clusters = response.getClusters(); + assertFalse("search cluster results should BE successful", clusters.hasPartialResults()); + assertThat(clusters.getTotal(), equalTo(2)); + assertThat(clusters.getClusterStateCount(Cluster.Status.SUCCESSFUL), equalTo(2)); + assertThat(clusters.getClusterStateCount(Cluster.Status.SKIPPED), equalTo(0)); + assertThat(clusters.getClusterStateCount(Cluster.Status.RUNNING), equalTo(0)); + assertThat(clusters.getClusterStateCount(Cluster.Status.PARTIAL), equalTo(0)); + assertThat(clusters.getClusterStateCount(Cluster.Status.FAILED), equalTo(0)); + + Cluster localClusterSearchInfo = clusters.getCluster(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); + assertNotNull(localClusterSearchInfo); + assertThat(localClusterSearchInfo.getStatus(), equalTo(Cluster.Status.SUCCESSFUL)); + assertThat(localClusterSearchInfo.getIndexExpression(), equalTo("nomatch*")); + assertThat(localClusterSearchInfo.getTotalShards(), equalTo(0)); + assertThat(localClusterSearchInfo.getSuccessfulShards(), equalTo(0)); + assertThat(localClusterSearchInfo.getSkippedShards(), equalTo(0)); + assertThat(localClusterSearchInfo.getFailedShards(), equalTo(0)); + assertThat(localClusterSearchInfo.getFailures().size(), equalTo(0)); + assertThat(localClusterSearchInfo.getTook().millis(), equalTo(0L)); + + Cluster remoteClusterSearchInfo = clusters.getCluster(REMOTE_CLUSTER); + assertNotNull(remoteClusterSearchInfo); + assertThat(remoteClusterSearchInfo.getStatus(), equalTo(Cluster.Status.SUCCESSFUL)); + assertThat(remoteClusterSearchInfo.getIndexExpression(), equalTo(remoteIndex)); + assertThat(remoteClusterSearchInfo.getTotalShards(), equalTo(remoteNumShards)); + assertThat(remoteClusterSearchInfo.getSuccessfulShards(), equalTo(remoteNumShards)); + assertThat(remoteClusterSearchInfo.getSkippedShards(), equalTo(0)); + assertThat(remoteClusterSearchInfo.getFailedShards(), equalTo(0)); + assertThat(remoteClusterSearchInfo.getFailures().size(), equalTo(0)); + assertThat(remoteClusterSearchInfo.getTook().millis(), greaterThan(0L)); + }); + } + private static void assertOneFailedShard(Cluster cluster, int totalShards) { assertNotNull(cluster); assertThat(cluster.getStatus(), equalTo(Cluster.Status.PARTIAL)); @@ -674,40 +828,42 @@ private static void assertOneFailedShard(Cluster cluster, int totalShards) { assertTrue("should have 'index corrupted' in reason", remoteShardSearchFailure.reason().contains("index corrupted")); } - private Map setupTwoClusters() { - String localIndex = "demo"; + private Map setupTwoClusters(String[] localIndices, String[] remoteIndices) { int numShardsLocal = randomIntBetween(2, 10); Settings localSettings = indexSettings(numShardsLocal, randomIntBetween(0, 1)).build(); - assertAcked( - client(LOCAL_CLUSTER).admin() - .indices() - .prepareCreate(localIndex) - .setSettings(localSettings) - .setMapping("@timestamp", "type=date", "f", "type=text") - ); - indexDocs(client(LOCAL_CLUSTER), localIndex); + for (String localIndex : localIndices) { + assertAcked( + client(LOCAL_CLUSTER).admin() + .indices() + .prepareCreate(localIndex) + .setSettings(localSettings) + .setMapping("@timestamp", "type=date", "f", "type=text") + ); + indexDocs(client(LOCAL_CLUSTER), localIndex); + } - String remoteIndex = "prod"; int numShardsRemote = randomIntBetween(2, 10); final InternalTestCluster remoteCluster = cluster(REMOTE_CLUSTER); remoteCluster.ensureAtLeastNumDataNodes(randomIntBetween(1, 3)); - assertAcked( - client(REMOTE_CLUSTER).admin() - .indices() - .prepareCreate(remoteIndex) - .setSettings(indexSettings(numShardsRemote, randomIntBetween(0, 1))) - .setMapping("@timestamp", "type=date", "f", "type=text") - ); - assertFalse( - client(REMOTE_CLUSTER).admin() - .cluster() - .prepareHealth(TEST_REQUEST_TIMEOUT, remoteIndex) - .setWaitForYellowStatus() - .setTimeout(TimeValue.timeValueSeconds(10)) - .get() - .isTimedOut() - ); - indexDocs(client(REMOTE_CLUSTER), remoteIndex); + for (String remoteIndex : remoteIndices) { + assertAcked( + client(REMOTE_CLUSTER).admin() + .indices() + .prepareCreate(remoteIndex) + .setSettings(indexSettings(numShardsRemote, randomIntBetween(0, 1))) + .setMapping("@timestamp", "type=date", "f", "type=text") + ); + assertFalse( + client(REMOTE_CLUSTER).admin() + .cluster() + .prepareHealth(TEST_REQUEST_TIMEOUT, remoteIndex) + .setWaitForYellowStatus() + .setTimeout(TimeValue.timeValueSeconds(10)) + .get() + .isTimedOut() + ); + indexDocs(client(REMOTE_CLUSTER), remoteIndex); + } String skipUnavailableKey = Strings.format("cluster.remote.%s.skip_unavailable", REMOTE_CLUSTER); Setting skipUnavailableSetting = cluster(REMOTE_CLUSTER).clusterService().getClusterSettings().get(skipUnavailableKey); @@ -717,13 +873,18 @@ private Map setupTwoClusters() { Map clusterInfo = new HashMap<>(); clusterInfo.put("local.num_shards", numShardsLocal); - clusterInfo.put("local.index", localIndex); clusterInfo.put("remote.num_shards", numShardsRemote); - clusterInfo.put("remote.index", remoteIndex); clusterInfo.put("remote.skip_unavailable", skipUnavailable); return clusterInfo; } + private Map setupTwoClusters() { + var clusterInfo = setupTwoClusters(new String[] { "demo" }, new String[] { "prod" }); + clusterInfo.put("local.index", "demo"); + clusterInfo.put("remote.index", "prod"); + return clusterInfo; + } + private int indexDocs(Client client, String index) { int numDocs = between(500, 1200); for (int i = 0; i < numDocs; i++) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchLeakIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchLeakIT.java index 5c26899f2e3f0..8b493782d55b5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchLeakIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchLeakIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.ccs; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java index 15afd6897a40e..2cb2e186b257e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.fetch; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java index b536db040e39f..66d44a818b797 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.fetch.subphase; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesIT.java index c996725e6285e..c796522eda0e8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.fetch.subphase; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/CustomHighlighterSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/CustomHighlighterSearchIT.java index 8aaa1bce252d7..920cd821195a6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/CustomHighlighterSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/CustomHighlighterSearchIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.fetch.subphase.highlight; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java index d9d6979ffd710..0ce4f34463b03 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.fetch.subphase.highlight; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java index 08c4d2aab4bc9..56b34f9b1dfec 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.fieldcaps; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java index c89c7d8d749f6..fe49ce57d0400 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.fieldcaps; @@ -13,6 +14,8 @@ import org.apache.logging.log4j.Level; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteUtils; +import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.fieldcaps.FieldCapabilities; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesFailure; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest; @@ -20,6 +23,7 @@ import org.elasticsearch.action.fieldcaps.TransportFieldCapabilitiesAction; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.ActiveShardCount; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.Cancellable; import org.elasticsearch.client.Request; @@ -44,6 +48,7 @@ import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.IndexClosedException; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.FieldPredicate; import org.elasticsearch.plugins.MapperPlugin; @@ -318,9 +323,68 @@ public void testWithIndexAlias() { assertEquals(response1, response2); } + public void testNoIndices() { + boolean ignoreUnavailable = false; + IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false); + client().admin().indices().close(new CloseIndexRequest("old_index")).actionGet(); + FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields("*").setIndicesOptions(options).get(); + assertIndices(response, "new_index"); + } + + public void testNoIndicesIgnoreUnavailable() { + boolean ignoreUnavailable = true; + IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false); + client().admin().indices().close(new CloseIndexRequest("old_index")).actionGet(); + FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields("*").setIndicesOptions(options).get(); + assertIndices(response, "new_index"); + } + + public void testOneClosedIndex() { + boolean ignoreUnavailable = false; + IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false); + client().admin().indices().close(new CloseIndexRequest("old_index")).actionGet(); + IndexClosedException ex = expectThrows( + IndexClosedException.class, + client().prepareFieldCaps("old_index").setFields("*").setIndicesOptions(options) + ); + assertEquals("closed", ex.getMessage()); + } + + public void testOneClosedIndexIgnoreUnavailable() { + boolean ignoreUnavailable = true; + IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false); + client().admin().indices().close(new CloseIndexRequest("old_index")).actionGet(); + FieldCapabilitiesResponse response = client().prepareFieldCaps("old_index").setFields("*").setIndicesOptions(options).get(); + assertIndices(response); + } + + public void testTwoIndicesOneClosed() { + boolean ignoreUnavailable = false; + IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false); + client().admin().indices().close(new CloseIndexRequest("old_index")).actionGet(); + IndexClosedException ex = expectThrows( + IndexClosedException.class, + client().prepareFieldCaps("old_index", "new_index").setFields("*").setIndicesOptions(options) + ); + assertEquals("closed", ex.getMessage()); + } + + public void testTwoIndicesOneClosedIgnoreUnavailable() { + boolean ignoreUnavailable = true; + IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false); + client().admin().indices().close(new CloseIndexRequest("old_index")).actionGet(); + FieldCapabilitiesResponse response = client().prepareFieldCaps("old_index", "new_index") + .setFields("*") + .setIndicesOptions(options) + .get(); + assertIndices(response, "new_index"); + } + public void testWithIndexFilter() throws InterruptedException { - assertAcked(prepareCreate("index-1").setMapping("timestamp", "type=date", "field1", "type=keyword")); - assertAcked(prepareCreate("index-2").setMapping("timestamp", "type=date", "field1", "type=long")); + assertAcked( + prepareCreate("index-1").setMapping("timestamp", "type=date", "field1", "type=keyword"), + prepareCreate("index-2").setMapping("timestamp", "type=date", "field1", "type=long") + ); List reqs = new ArrayList<>(); reqs.add(prepareIndex("index-1").setSource("timestamp", "2015-07-08")); @@ -413,8 +477,7 @@ public void testFieldMetricsAndDimensions() { public void testFailures() throws InterruptedException { // in addition to the existing "old_index" and "new_index", create two where the test query throws an error on rewrite - assertAcked(prepareCreate("index1-error")); - assertAcked(prepareCreate("index2-error")); + assertAcked(prepareCreate("index1-error"), prepareCreate("index2-error")); ensureGreen("index1-error", "index2-error"); FieldCapabilitiesResponse response = client().prepareFieldCaps() .setFields("*") @@ -442,9 +505,7 @@ private void populateTimeRangeIndices() throws Exception { internalCluster().ensureAtLeastNumDataNodes(2); assertAcked( prepareCreate("log-index-1").setSettings(indexSettings(between(1, 5), 1)) - .setMapping("timestamp", "type=date", "field1", "type=keyword") - ); - assertAcked( + .setMapping("timestamp", "type=date", "field1", "type=keyword"), prepareCreate("log-index-2").setSettings(indexSettings(between(1, 5), 1)) .setMapping("timestamp", "type=date", "field1", "type=long") ); @@ -605,9 +666,11 @@ public void testManyIndicesWithSameMapping() { } """; String[] indices = IntStream.range(0, between(1, 9)).mapToObj(n -> "test_many_index_" + n).toArray(String[]::new); - for (String index : indices) { - assertAcked(indicesAdmin().prepareCreate(index).setMapping(mapping).get()); - } + assertAcked( + Arrays.stream(indices) + .map(index -> indicesAdmin().prepareCreate(index).setMapping(mapping)) + .toArray(CreateIndexRequestBuilder[]::new) + ); FieldCapabilitiesRequest request = new FieldCapabilitiesRequest(); request.indices("test_many_index_*"); request.fields("*"); @@ -692,7 +755,7 @@ public void testCancel() throws Exception { } }, 30, TimeUnit.SECONDS); cancellable.cancel(); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); logger.info("--> waiting for field-caps tasks to be cancelled"); assertBusy(() -> { List tasks = clusterAdmin().prepareListTasks() @@ -726,9 +789,11 @@ public void testIndexMode() throws Exception { Settings settings = Settings.builder().put("mode", "time_series").putList("routing_path", List.of("hostname")).build(); int numIndices = between(1, 5); for (int i = 0; i < numIndices; i++) { - assertAcked(indicesAdmin().prepareCreate("test_metrics_" + i).setSettings(settings).setMapping(metricsMapping).get()); + assertAcked( + indicesAdmin().prepareCreate("test_metrics_" + i).setSettings(settings).setMapping(metricsMapping), + indicesAdmin().prepareCreate("test_old_metrics_" + i).setMapping(metricsMapping) + ); indexModes.put("test_metrics_" + i, IndexMode.TIME_SERIES); - assertAcked(indicesAdmin().prepareCreate("test_old_metrics_" + i).setMapping(metricsMapping).get()); indexModes.put("test_old_metrics_" + i, IndexMode.STANDARD); } } @@ -747,9 +812,11 @@ public void testIndexMode() throws Exception { Settings settings = Settings.builder().put("mode", "logsdb").build(); int numIndices = between(1, 5); for (int i = 0; i < numIndices; i++) { - assertAcked(indicesAdmin().prepareCreate("test_logs_" + i).setSettings(settings).setMapping(logsMapping).get()); + assertAcked( + indicesAdmin().prepareCreate("test_logs_" + i).setSettings(settings).setMapping(logsMapping), + indicesAdmin().prepareCreate("test_old_logs_" + i).setMapping(logsMapping) + ); indexModes.put("test_logs_" + i, IndexMode.LOGSDB); - assertAcked(indicesAdmin().prepareCreate("test_old_logs_" + i).setMapping(logsMapping).get()); indexModes.put("test_old_logs_" + i, IndexMode.STANDARD); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapsHasValueTests.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapsHasValueTests.java index 337a663721474..c3488ed34db74 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapsHasValueTests.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapsHasValueTests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.fieldcaps; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fields/SearchFieldsIT.java index 9ad6363d0e57d..d1eb1ab533ab7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fields/SearchFieldsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fields/SearchFieldsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.fields; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java index dcbf4996358d7..36e75435bb5de 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.functionscore; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java index c59fc0f68c4d4..7fb06c0b83015 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.functionscore; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScoreFieldValueIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScoreFieldValueIT.java index 0a43255967dcd..9f79bf1459cf9 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScoreFieldValueIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScoreFieldValueIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.functionscore; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScoreIT.java index 422d6f06f2988..a0fe7e661020d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScoreIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.functionscore; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScorePluginIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScorePluginIT.java index d58e777b093ae..1bf72071b6bbf 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScorePluginIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScorePluginIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.functionscore; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java index 2b61e6ae5d1ad..c9ab3be0eba61 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java @@ -1,38 +1,54 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.functionscore; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Explanation; import org.apache.lucene.tests.util.English; +import org.elasticsearch.TransportVersion; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.search.function.CombineFunction; +import org.elasticsearch.common.lucene.search.function.LeafScoreFunction; +import org.elasticsearch.common.lucene.search.function.ScoreFunction; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings.Builder; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.Operator; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.collapse.CollapseBuilder; import org.elasticsearch.search.rescore.QueryRescoreMode; import org.elasticsearch.search.rescore.QueryRescorerBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Comparator; import java.util.List; @@ -847,6 +863,20 @@ public void testRescorePhaseWithInvalidSort() throws Exception { } } ); + + assertResponse( + prepareSearch().addSort(SortBuilders.scoreSort()) + .addSort(new FieldSortBuilder(FieldSortBuilder.SHARD_DOC_FIELD_NAME)) + .setTrackScores(true) + .addRescorer(new QueryRescorerBuilder(matchAllQuery()).setRescoreQueryWeight(100.0f), 50), + response -> { + assertThat(response.getHits().getTotalHits().value, equalTo(5L)); + assertThat(response.getHits().getHits().length, equalTo(5)); + for (SearchHit hit : response.getHits().getHits()) { + assertThat(hit.getScore(), equalTo(101f)); + } + } + ); } record GroupDoc(String id, String group, float firstPassScore, float secondPassScore, boolean shouldFilter) {} @@ -886,6 +916,10 @@ public void testRescoreAfterCollapse() throws Exception { .setQuery(fieldValueScoreQuery("firstPassScore")) .addRescorer(new QueryRescorerBuilder(fieldValueScoreQuery("secondPassScore"))) .setCollapse(new CollapseBuilder("group")); + if (randomBoolean()) { + request.addSort(SortBuilders.scoreSort()); + request.addSort(new FieldSortBuilder(FieldSortBuilder.SHARD_DOC_FIELD_NAME)); + } assertResponse(request, resp -> { assertThat(resp.getHits().getTotalHits().value, equalTo(5L)); assertThat(resp.getHits().getHits().length, equalTo(3)); @@ -965,6 +999,10 @@ public void testRescoreAfterCollapseRandom() throws Exception { .addRescorer(new QueryRescorerBuilder(fieldValueScoreQuery("secondPassScore")).setQueryWeight(0f).windowSize(numGroups)) .setCollapse(new CollapseBuilder("group")) .setSize(Math.min(numGroups, 10)); + if (randomBoolean()) { + request.addSort(SortBuilders.scoreSort()); + request.addSort(new FieldSortBuilder(FieldSortBuilder.SHARD_DOC_FIELD_NAME)); + } long expectedNumHits = numHits; assertResponse(request, resp -> { assertThat(resp.getHits().getTotalHits().value, equalTo(expectedNumHits)); @@ -978,9 +1016,119 @@ public void testRescoreAfterCollapseRandom() throws Exception { }); } + public void testRescoreWithTimeout() throws Exception { + // no dummy docs since merges can change scores while we run queries. + int numDocs = indexRandomNumbers("whitespace", -1, false); + + String intToEnglish = English.intToEnglish(between(0, numDocs - 1)); + String query = intToEnglish.split(" ")[0]; + assertResponse( + prepareSearch().setSearchType(SearchType.QUERY_THEN_FETCH) + .setQuery(QueryBuilders.matchQuery("field1", query).operator(Operator.OR)) + .setSize(10) + .addRescorer(new QueryRescorerBuilder(functionScoreQuery(new TestTimedScoreFunctionBuilder())).windowSize(100)) + .setTimeout(TimeValue.timeValueMillis(10)), + r -> assertTrue(r.isTimedOut()) + ); + } + + @Override + protected Collection> nodePlugins() { + return List.of(TestTimedQueryPlugin.class); + } + private QueryBuilder fieldValueScoreQuery(String scoreField) { return functionScoreQuery(termQuery("shouldFilter", false), ScoreFunctionBuilders.fieldValueFactorFunction(scoreField)).boostMode( CombineFunction.REPLACE ); } + + public static class TestTimedQueryPlugin extends Plugin implements SearchPlugin { + @Override + public List> getScoreFunctions() { + return List.of( + new ScoreFunctionSpec<>( + new ParseField("timed"), + TestTimedScoreFunctionBuilder::new, + p -> new TestTimedScoreFunctionBuilder() + ) + ); + } + } + + static class TestTimedScoreFunctionBuilder extends ScoreFunctionBuilder { + private final long time = 500; + + TestTimedScoreFunctionBuilder() {} + + TestTimedScoreFunctionBuilder(StreamInput in) throws IOException { + super(in); + } + + @Override + protected void doWriteTo(StreamOutput out) {} + + @Override + public String getName() { + return "timed"; + } + + @Override + protected void doXContent(XContentBuilder builder, Params params) {} + + @Override + protected boolean doEquals(TestTimedScoreFunctionBuilder functionBuilder) { + return false; + } + + @Override + protected int doHashCode() { + return 0; + } + + @Override + protected ScoreFunction doToFunction(SearchExecutionContext context) throws IOException { + return new ScoreFunction(REPLACE) { + @Override + public LeafScoreFunction getLeafScoreFunction(LeafReaderContext ctx) throws IOException { + return new LeafScoreFunction() { + @Override + public double score(int docId, float subQueryScore) { + try { + Thread.sleep(time); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return time; + } + + @Override + public Explanation explainScore(int docId, Explanation subQueryScore) { + return null; + } + }; + } + + @Override + public boolean needsScores() { + return true; + } + + @Override + protected boolean doEquals(ScoreFunction other) { + return false; + } + + @Override + protected int doHashCode() { + return 0; + } + }; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersion.current(); + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java index 1fe128da6889c..7fdb31a468998 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.functionscore; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryGeoPointIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryGeoPointIT.java index 7a3b1699c30e5..2489889be19e5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryGeoPointIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryGeoPointIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.geo; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoDistanceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoDistanceIT.java index 31524765d4e14..9b4e28055a988 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoDistanceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoDistanceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.geo; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoPointScriptDocValuesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoPointScriptDocValuesIT.java index c62f4932220fc..fb2b9a9f21418 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoPointScriptDocValuesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoPointScriptDocValuesIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.geo; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoPolygonIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoPolygonIT.java index e929487af9240..4b8f29f3cc9a5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoPolygonIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoPolygonIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.geo; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java index f632bbed134f0..535511935bdf7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.morelikethis; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/msearch/MultiSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/msearch/MultiSearchIT.java index 7072594eab8ec..5e4ae084eaa4a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/msearch/MultiSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/msearch/MultiSearchIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.msearch; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/nested/NestedWithMinScoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/nested/NestedWithMinScoreIT.java index 245fb1651f4d6..59ab6baa8050a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/nested/NestedWithMinScoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/nested/NestedWithMinScoreIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.nested; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/nested/SimpleNestedIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/nested/SimpleNestedIT.java index a9d33b268e73f..228b71d6c856d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/nested/SimpleNestedIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/nested/SimpleNestedIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.nested; @@ -20,7 +21,9 @@ import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.InnerHitBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.NestedSortBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortMode; @@ -1580,6 +1583,64 @@ public void testCheckFixedBitSetCache() throws Exception { assertThat(clusterStatsResponse.getIndicesStats().getSegments().getBitsetMemoryInBytes(), equalTo(0L)); } + public void testSkipNestedInnerHits() throws Exception { + assertAcked(prepareCreate("test").setMapping("nested1", "type=nested")); + ensureGreen(); + + prepareIndex("test").setId("1") + .setSource( + jsonBuilder().startObject() + .field("field1", "value1") + .startArray("nested1") + .startObject() + .field("n_field1", "foo") + .field("n_field2", "bar") + .endObject() + .endArray() + .endObject() + ) + .get(); + + waitForRelocation(ClusterHealthStatus.GREEN); + GetResponse getResponse = client().prepareGet("test", "1").get(); + assertThat(getResponse.isExists(), equalTo(true)); + assertThat(getResponse.getSourceAsBytesRef(), notNullValue()); + refresh(); + + assertNoFailuresAndResponse( + prepareSearch("test").setSource( + new SearchSourceBuilder().query( + QueryBuilders.nestedQuery("nested1", QueryBuilders.termQuery("nested1.n_field1", "foo"), ScoreMode.Avg) + .innerHit(new InnerHitBuilder()) + ) + ), + res -> { + assertNotNull(res.getHits()); + assertHitCount(res, 1); + assertThat(res.getHits().getHits().length, equalTo(1)); + // by default we should get inner hits + assertNotNull(res.getHits().getHits()[0].getInnerHits()); + assertNotNull(res.getHits().getHits()[0].getInnerHits().get("nested1")); + } + ); + + assertNoFailuresAndResponse( + prepareSearch("test").setSource( + new SearchSourceBuilder().query( + QueryBuilders.nestedQuery("nested1", QueryBuilders.termQuery("nested1.n_field1", "foo"), ScoreMode.Avg) + .innerHit(new InnerHitBuilder()) + ).skipInnerHits(true) + ), + res -> { + assertNotNull(res.getHits()); + assertHitCount(res, 1); + assertThat(res.getHits().getHits().length, equalTo(1)); + // if we explicitly say to ignore inner hits, then this should now be null + assertNull(res.getHits().getHits()[0].getInnerHits()); + } + ); + } + private void assertDocumentCount(String index, long numdocs) { IndicesStatsResponse stats = indicesAdmin().prepareStats(index).clear().setDocs(true).get(); assertNoFailures(stats); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/nested/VectorNestedIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/nested/VectorNestedIT.java index f830ca9ac0cb6..aaab14941d4bb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/nested/VectorNestedIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/nested/VectorNestedIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.nested; @@ -68,7 +69,7 @@ public void testSimpleNested() throws Exception { assertResponse( prepareSearch("test").setKnnSearch( - List.of(new KnnSearchBuilder("nested.vector", new float[] { 1, 1, 1 }, 1, 1, null).innerHit(new InnerHitBuilder())) + List.of(new KnnSearchBuilder("nested.vector", new float[] { 1, 1, 1 }, 1, 1, null, null).innerHit(new InnerHitBuilder())) ).setAllowPartialSearchResults(false), response -> assertThat(response.getHits().getHits().length, greaterThan(0)) ); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java index 81bb143c47729..3cb26eb97a71b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.profile.aggregation; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/profile/dfs/DfsProfilerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/profile/dfs/DfsProfilerIT.java index 7e504a100ba56..95d69a6ebaa86 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/profile/dfs/DfsProfilerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/profile/dfs/DfsProfilerIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.profile.dfs; @@ -18,6 +19,7 @@ import org.elasticsearch.search.profile.query.CollectorResult; import org.elasticsearch.search.profile.query.QueryProfileShardResult; import org.elasticsearch.search.vectors.KnnSearchBuilder; +import org.elasticsearch.search.vectors.RescoreVectorBuilder; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xcontent.XContentFactory; @@ -70,6 +72,7 @@ public void testProfileDfs() throws Exception { new float[] { randomFloat(), randomFloat(), randomFloat() }, randomIntBetween(5, 10), 50, + randomBoolean() ? null : new RescoreVectorBuilder(randomFloatBetween(1.0f, 10.0f, false)), randomBoolean() ? null : randomFloat() ); if (randomBoolean()) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java index e02bed8409bc4..bda0d6f7e88b0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.profile.query; @@ -67,7 +68,11 @@ public void testProfileQuery() throws Exception { prepareSearch().setQuery(q).setTrackTotalHits(true).setProfile(true).setSearchType(SearchType.QUERY_THEN_FETCH), response -> { assertNotNull("Profile response element should not be null", response.getProfileResults()); - assertThat("Profile response should not be an empty array", response.getProfileResults().size(), not(0)); + if (response.getSkippedShards() == response.getSuccessfulShards()) { + assertEquals(0, response.getProfileResults().size()); + } else { + assertThat("Profile response should not be an empty array", response.getProfileResults().size(), not(0)); + } for (Map.Entry shard : response.getProfileResults().entrySet()) { for (QueryProfileShardResult searchProfiles : shard.getValue().getQueryProfileResults()) { for (ProfileResult result : searchProfiles.getQueryResults()) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/ExistsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/ExistsIT.java index 81c612107e44a..f263ececfdc7d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/ExistsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/ExistsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.query; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/IntervalQueriesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/IntervalQueriesIT.java index 50a1924843e74..def137f9a9ec8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/IntervalQueriesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/IntervalQueriesIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.query; @@ -30,6 +31,7 @@ import static java.util.Collections.singletonMap; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; public class IntervalQueriesIT extends ESIntegTestCase { @@ -63,6 +65,58 @@ public void testEmptyIntervalsWithNestedMappings() throws InterruptedException { ); } + public void testPreserveInnerGap() { + assertAcked(prepareCreate("index").setMapping(""" + { + "_doc" : { + "properties" : { + "text" : { "type" : "text" } + } + } + } + """)); + + indexRandom(true, prepareIndex("index").setId("1").setSource("text", "w1 w2 w3 w4 w5")); + + // ordered + { + var res = prepareSearch("index").setQuery( + new IntervalQueryBuilder( + "text", + new IntervalsSourceProvider.Combine( + Arrays.asList( + new IntervalsSourceProvider.Match("w1 w4", -1, true, null, null, null), + new IntervalsSourceProvider.Match("w5", -1, true, null, null, null) + ), + true, + 1, + null + ) + ) + ); + assertSearchHits(res, "1"); + } + + // unordered + { + var res = prepareSearch("index").setQuery( + new IntervalQueryBuilder( + "text", + new IntervalsSourceProvider.Combine( + Arrays.asList( + new IntervalsSourceProvider.Match("w3", 0, false, null, null, null), + new IntervalsSourceProvider.Match("w4 w1", -1, false, null, null, null) + ), + false, + 0, + null + ) + ) + ); + assertSearchHits(res, "1"); + } + } + private static class EmptyAnalyzer extends Analyzer { @Override diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/MultiMatchQueryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/MultiMatchQueryIT.java index 2d6bb8176b091..c75d5c0ac7993 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/MultiMatchQueryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/MultiMatchQueryIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.query; @@ -56,6 +57,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailuresAndResponse; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponses; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHitsWithoutFailures; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSecondHit; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId; @@ -771,6 +773,7 @@ public void testCrossFieldMode() throws ExecutionException, InterruptedException ); // counter example assertHitCount( + 0L, prepareSearch("test").setQuery( randomizeType( multiMatchQuery("captain america marvel hero", "first_name", "last_name", "category").type( @@ -778,19 +781,13 @@ public void testCrossFieldMode() throws ExecutionException, InterruptedException ).operator(Operator.AND) ) ), - 0L - ); - - // counter example - assertHitCount( prepareSearch("test").setQuery( randomizeType( multiMatchQuery("captain america marvel hero", "first_name", "last_name", "category").type( randomBoolean() ? MultiMatchQueryBuilder.Type.CROSS_FIELDS : MultiMatchQueryBuilder.DEFAULT_TYPE ).operator(Operator.AND) ) - ), - 0L + ) ); // test if boosts work @@ -827,40 +824,21 @@ public void testCrossFieldMode() throws ExecutionException, InterruptedException } ); // Test group based on numeric fields - assertResponse( + assertResponses(response -> { + assertHitCount(response, 1L); + assertFirstHit(response, hasId("theone")); + }, prepareSearch("test").setQuery(randomizeType(multiMatchQuery("15", "skill").type(MultiMatchQueryBuilder.Type.CROSS_FIELDS))), - response -> { - assertHitCount(response, 1L); - assertFirstHit(response, hasId("theone")); - } - ); - assertResponse( prepareSearch("test").setQuery( randomizeType(multiMatchQuery("15", "skill", "first_name").type(MultiMatchQueryBuilder.Type.CROSS_FIELDS)) ), - response -> { - assertHitCount(response, 1L); - assertFirstHit(response, hasId("theone")); - } - ); - // Two numeric fields together caused trouble at one point! - assertResponse( + // Two numeric fields together caused trouble at one point! prepareSearch("test").setQuery( randomizeType(multiMatchQuery("15", "int-field", "skill").type(MultiMatchQueryBuilder.Type.CROSS_FIELDS)) ), - response -> { - assertHitCount(response, 1L); - assertFirstHit(response, hasId("theone")); - } - ); - assertResponse( prepareSearch("test").setQuery( randomizeType(multiMatchQuery("15", "int-field", "first_name", "skill").type(MultiMatchQueryBuilder.Type.CROSS_FIELDS)) - ), - response -> { - assertHitCount(response, 1L); - assertFirstHit(response, hasId("theone")); - } + ) ); assertResponse( prepareSearch("test").setQuery( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java index a7ce84f3cd02d..e25e330e072a6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.query; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/ScriptScoreQueryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/ScriptScoreQueryIT.java index 30eeb86ba44d3..713c169951770 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/ScriptScoreQueryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/ScriptScoreQueryIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.query; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/SearchQueryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/SearchQueryIT.java index 0a30de1bb3741..68abbbb1f9583 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/SearchQueryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/SearchQueryIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.query; @@ -107,6 +108,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailuresAndResponse; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponses; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHitsWithoutFailures; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSecondHit; @@ -215,21 +217,14 @@ public void testConstantScoreQuery() throws Exception { assertThat(searchHit, hasScore(1.0f)); } }); - assertResponse( + assertResponses(response -> { + assertHitCount(response, 2L); + assertFirstHit(response, hasScore(response.getHits().getAt(1).getScore())); + }, + prepareSearch("test").setQuery(constantScoreQuery(matchQuery("field1", "quick")).boost(1.0f + random().nextFloat())), prepareSearch("test").setQuery( boolQuery().must(matchAllQuery()).must(constantScoreQuery(matchQuery("field1", "quick")).boost(1.0f + random().nextFloat())) - ), - response -> { - assertHitCount(response, 2L); - assertFirstHit(response, hasScore(response.getHits().getAt(1).getScore())); - } - ); - assertResponse( - prepareSearch("test").setQuery(constantScoreQuery(matchQuery("field1", "quick")).boost(1.0f + random().nextFloat())), - response -> { - assertHitCount(response, 2L); - assertFirstHit(response, hasScore(response.getHits().getAt(1).getScore())); - } + ) ); assertResponse( prepareSearch("test").setQuery( @@ -799,20 +794,18 @@ public void testSpecialRangeSyntaxInQueryString() { prepareIndex("test").setId("2").setSource("str", "shay", "date", "2012-02-05", "num", 20).get(); refresh(); - assertResponse(prepareSearch().setQuery(queryStringQuery("num:>19")), response -> { + assertResponses(response -> { assertHitCount(response, 1L); assertFirstHit(response, hasId("2")); - }); - assertHitCount(prepareSearch().setQuery(queryStringQuery("num:>20")), 0L); + }, prepareSearch().setQuery(queryStringQuery("num:>19")), prepareSearch().setQuery(queryStringQuery("num:>=20"))); - assertResponse(prepareSearch().setQuery(queryStringQuery("num:>=20")), response -> { - assertHitCount(response, 1L); - assertFirstHit(response, hasId("2")); - }); - assertHitCount(prepareSearch().setQuery(queryStringQuery("num:>11")), 2L); - assertHitCount(prepareSearch().setQuery(queryStringQuery("num:<20")), 1L); - assertHitCount(prepareSearch().setQuery(queryStringQuery("num:<=20")), 2L); - assertHitCount(prepareSearch().setQuery(queryStringQuery("+num:>11 +num:<20")), 1L); + assertHitCount(prepareSearch().setQuery(queryStringQuery("num:>20")), 0L); + assertHitCount(2L, prepareSearch().setQuery(queryStringQuery("num:>11")), prepareSearch().setQuery(queryStringQuery("num:<=20"))); + assertHitCount( + 1L, + prepareSearch().setQuery(queryStringQuery("num:<20")), + prepareSearch().setQuery(queryStringQuery("+num:>11 +num:<20")) + ); } public void testEmptytermsQuery() throws Exception { @@ -825,8 +818,11 @@ public void testEmptytermsQuery() throws Exception { prepareIndex("test").setId("3").setSource("term", "3"), prepareIndex("test").setId("4").setSource("term", "4") ); - assertHitCount(prepareSearch("test").setQuery(constantScoreQuery(termsQuery("term", new String[0]))), 0L); - assertHitCount(prepareSearch("test").setQuery(idsQuery()), 0L); + assertHitCount( + 0L, + prepareSearch("test").setQuery(constantScoreQuery(termsQuery("term", new String[0]))), + prepareSearch("test").setQuery(idsQuery()) + ); } public void testTermsQuery() throws Exception { @@ -865,9 +861,12 @@ public void testTermsQuery() throws Exception { assertSearchHitsWithoutFailures(prepareSearch("test").setQuery(constantScoreQuery(termsQuery("dbl", new double[] { 2, 5 }))), "2"); assertSearchHitsWithoutFailures(prepareSearch("test").setQuery(constantScoreQuery(termsQuery("lng", new long[] { 2, 5 }))), "2"); // test valid type, but no matching terms - assertHitCount(prepareSearch("test").setQuery(constantScoreQuery(termsQuery("str", "5", "6"))), 0L); - assertHitCount(prepareSearch("test").setQuery(constantScoreQuery(termsQuery("dbl", new double[] { 5, 6 }))), 0L); - assertHitCount(prepareSearch("test").setQuery(constantScoreQuery(termsQuery("lng", new long[] { 5, 6 }))), 0L); + assertHitCount( + 0L, + prepareSearch("test").setQuery(constantScoreQuery(termsQuery("str", "5", "6"))), + prepareSearch("test").setQuery(constantScoreQuery(termsQuery("dbl", new double[] { 5, 6 }))), + prepareSearch("test").setQuery(constantScoreQuery(termsQuery("lng", new long[] { 5, 6 }))) + ); } public void testTermsLookupFilter() throws Exception { @@ -1063,106 +1062,35 @@ public void testNumericTermsAndRanges() throws Exception { .get(); refresh(); - logger.info("--> term query on 1"); - assertResponse(prepareSearch("test").setQuery(termQuery("num_byte", 1)), response -> { - assertHitCount(response, 1L); - assertFirstHit(response, hasId("1")); - }); - assertResponse(prepareSearch("test").setQuery(termQuery("num_short", 1)), response -> { - assertHitCount(response, 1L); - assertFirstHit(response, hasId("1")); - }); - assertResponse(prepareSearch("test").setQuery(termQuery("num_integer", 1)), response -> { - assertHitCount(response, 1L); - assertFirstHit(response, hasId("1")); - }); - assertResponse(prepareSearch("test").setQuery(termQuery("num_long", 1)), response -> { - assertHitCount(response, 1L); - assertFirstHit(response, hasId("1")); - }); - assertResponse(prepareSearch("test").setQuery(termQuery("num_float", 1)), response -> { - assertHitCount(response, 1L); - assertFirstHit(response, hasId("1")); - }); - assertResponse(prepareSearch("test").setQuery(termQuery("num_double", 1)), response -> { - assertHitCount(response, 1L); - assertFirstHit(response, hasId("1")); - }); - logger.info("--> terms query on 1"); - assertResponse(prepareSearch("test").setQuery(termsQuery("num_byte", new int[] { 1 })), response -> { - assertHitCount(response, 1L); - assertFirstHit(response, hasId("1")); - }); - assertResponse(prepareSearch("test").setQuery(termsQuery("num_short", new int[] { 1 })), response -> { - assertHitCount(response, 1L); - assertFirstHit(response, hasId("1")); - }); - assertResponse(prepareSearch("test").setQuery(termsQuery("num_integer", new int[] { 1 })), response -> { - assertHitCount(response, 1L); - assertFirstHit(response, hasId("1")); - }); - assertResponse(prepareSearch("test").setQuery(termsQuery("num_long", new int[] { 1 })), response -> { - assertHitCount(response, 1L); - assertFirstHit(response, hasId("1")); - }); - assertResponse(prepareSearch("test").setQuery(termsQuery("num_float", new double[] { 1 })), response -> { - assertHitCount(response, 1L); - assertFirstHit(response, hasId("1")); - }); - assertResponse(prepareSearch("test").setQuery(termsQuery("num_double", new double[] { 1 })), response -> { - assertHitCount(response, 1L); - assertFirstHit(response, hasId("1")); - }); - logger.info("--> term filter on 1"); - assertResponse(prepareSearch("test").setQuery(constantScoreQuery(termQuery("num_byte", 1))), response -> { - assertHitCount(response, 1L); - assertFirstHit(response, hasId("1")); - }); - assertResponse(prepareSearch("test").setQuery(constantScoreQuery(termQuery("num_short", 1))), response -> { - assertHitCount(response, 1L); - assertFirstHit(response, hasId("1")); - }); - assertResponse(prepareSearch("test").setQuery(constantScoreQuery(termQuery("num_integer", 1))), response -> { + assertResponses(response -> { assertHitCount(response, 1L); assertFirstHit(response, hasId("1")); - }); - assertResponse(prepareSearch("test").setQuery(constantScoreQuery(termQuery("num_long", 1))), response -> { - assertHitCount(response, 1L); - assertFirstHit(response, hasId("1")); - }); - assertResponse(prepareSearch("test").setQuery(constantScoreQuery(termQuery("num_float", 1))), response -> { - assertHitCount(response, 1L); - assertFirstHit(response, hasId("1")); - }); - assertResponse(prepareSearch("test").setQuery(constantScoreQuery(termQuery("num_double", 1))), response -> { - assertHitCount(response, 1L); - assertFirstHit(response, hasId("1")); - }); - logger.info("--> terms filter on 1"); - assertResponse(prepareSearch("test").setQuery(constantScoreQuery(termsQuery("num_byte", new int[] { 1 }))), response -> { - assertHitCount(response, 1L); - assertFirstHit(response, hasId("1")); - }); - assertResponse(prepareSearch("test").setQuery(constantScoreQuery(termsQuery("num_short", new int[] { 1 }))), response -> { - assertHitCount(response, 1L); - assertFirstHit(response, hasId("1")); - }); - assertResponse(prepareSearch("test").setQuery(constantScoreQuery(termsQuery("num_integer", new int[] { 1 }))), response -> { - assertHitCount(response, 1L); - assertFirstHit(response, hasId("1")); - }); - assertResponse(prepareSearch("test").setQuery(constantScoreQuery(termsQuery("num_long", new int[] { 1 }))), response -> { - assertHitCount(response, 1L); - assertFirstHit(response, hasId("1")); - }); - assertResponse(prepareSearch("test").setQuery(constantScoreQuery(termsQuery("num_float", new int[] { 1 }))), response -> { - assertHitCount(response, 1L); - assertFirstHit(response, hasId("1")); - }); - assertResponse(prepareSearch("test").setQuery(constantScoreQuery(termsQuery("num_double", new int[] { 1 }))), response -> { - assertHitCount(response, 1L); - assertFirstHit(response, hasId("1")); - }); + }, + prepareSearch("test").setQuery(termQuery("num_byte", 1)), + prepareSearch("test").setQuery(termQuery("num_short", 1)), + prepareSearch("test").setQuery(termQuery("num_integer", 1)), + prepareSearch("test").setQuery(termQuery("num_long", 1)), + prepareSearch("test").setQuery(termQuery("num_float", 1)), + prepareSearch("test").setQuery(termQuery("num_double", 1)), + prepareSearch("test").setQuery(termsQuery("num_byte", new int[] { 1 })), + prepareSearch("test").setQuery(termsQuery("num_short", new int[] { 1 })), + prepareSearch("test").setQuery(termsQuery("num_integer", new int[] { 1 })), + prepareSearch("test").setQuery(termsQuery("num_long", new int[] { 1 })), + prepareSearch("test").setQuery(termsQuery("num_float", new double[] { 1 })), + prepareSearch("test").setQuery(termsQuery("num_double", new double[] { 1 })), + prepareSearch("test").setQuery(constantScoreQuery(termQuery("num_byte", 1))), + prepareSearch("test").setQuery(constantScoreQuery(termQuery("num_short", 1))), + prepareSearch("test").setQuery(constantScoreQuery(termQuery("num_integer", 1))), + prepareSearch("test").setQuery(constantScoreQuery(termQuery("num_long", 1))), + prepareSearch("test").setQuery(constantScoreQuery(termQuery("num_float", 1))), + prepareSearch("test").setQuery(constantScoreQuery(termQuery("num_double", 1))), + prepareSearch("test").setQuery(constantScoreQuery(termsQuery("num_byte", new int[] { 1 }))), + prepareSearch("test").setQuery(constantScoreQuery(termsQuery("num_short", new int[] { 1 }))), + prepareSearch("test").setQuery(constantScoreQuery(termsQuery("num_integer", new int[] { 1 }))), + prepareSearch("test").setQuery(constantScoreQuery(termsQuery("num_long", new int[] { 1 }))), + prepareSearch("test").setQuery(constantScoreQuery(termsQuery("num_float", new int[] { 1 }))), + prepareSearch("test").setQuery(constantScoreQuery(termsQuery("num_double", new int[] { 1 }))) + ); } public void testNumericRangeFilter_2826() throws Exception { @@ -1300,16 +1228,19 @@ public void testSpanMultiTermQuery() throws IOException { prepareIndex("test").setId("4").setSource("description", "fop", "count", 4).get(); refresh(); - assertHitCount(prepareSearch("test").setQuery(spanOrQuery(spanMultiTermQueryBuilder(fuzzyQuery("description", "fop")))), 4); - assertHitCount(prepareSearch("test").setQuery(spanOrQuery(spanMultiTermQueryBuilder(prefixQuery("description", "fo")))), 4); - assertHitCount(prepareSearch("test").setQuery(spanOrQuery(spanMultiTermQueryBuilder(wildcardQuery("description", "oth*")))), 3); assertHitCount( + 4, + prepareSearch("test").setQuery(spanOrQuery(spanMultiTermQueryBuilder(fuzzyQuery("description", "fop")))), + prepareSearch("test").setQuery(spanOrQuery(spanMultiTermQueryBuilder(prefixQuery("description", "fo")))) + ); + assertHitCount( + 3, + prepareSearch("test").setQuery(spanOrQuery(spanMultiTermQueryBuilder(wildcardQuery("description", "oth*")))), prepareSearch("test").setQuery( spanOrQuery(spanMultiTermQueryBuilder(QueryBuilders.rangeQuery("description").from("ffa").to("foo"))) ), - 3 + prepareSearch("test").setQuery(spanOrQuery(spanMultiTermQueryBuilder(regexpQuery("description", "fo{2}")))) ); - assertHitCount(prepareSearch("test").setQuery(spanOrQuery(spanMultiTermQueryBuilder(regexpQuery("description", "fo{2}")))), 3); } public void testSpanNot() throws IOException, ExecutionException, InterruptedException { @@ -1320,6 +1251,7 @@ public void testSpanNot() throws IOException, ExecutionException, InterruptedExc refresh(); assertHitCount( + 1L, prepareSearch("test").setQuery( spanNotQuery( spanNearQuery(QueryBuilders.spanTermQuery("description", "quick"), 1).addClause( @@ -1328,9 +1260,6 @@ public void testSpanNot() throws IOException, ExecutionException, InterruptedExc spanTermQuery("description", "brown") ) ), - 1L - ); - assertHitCount( prepareSearch("test").setQuery( spanNotQuery( spanNearQuery(QueryBuilders.spanTermQuery("description", "quick"), 1).addClause( @@ -1339,9 +1268,6 @@ public void testSpanNot() throws IOException, ExecutionException, InterruptedExc spanTermQuery("description", "sleeping") ).dist(5) ), - 1L - ); - assertHitCount( prepareSearch("test").setQuery( spanNotQuery( spanNearQuery(QueryBuilders.spanTermQuery("description", "quick"), 1).addClause( @@ -1349,8 +1275,7 @@ public void testSpanNot() throws IOException, ExecutionException, InterruptedExc ), spanTermQuery("description", "jumped") ).pre(1).post(1) - ), - 1L + ) ); } @@ -1422,22 +1347,19 @@ public void testSimpleDFSQuery() throws IOException { public void testMultiFieldQueryString() { prepareIndex("test").setId("1").setSource("field1", "value1", "field2", "value2").setRefreshPolicy(IMMEDIATE).get(); - - logger.info("regular"); - assertHitCount(prepareSearch("test").setQuery(queryStringQuery("value1").field("field1").field("field2")), 1); - assertHitCount(prepareSearch("test").setQuery(queryStringQuery("field\\*:value1")), 1); - logger.info("prefix"); - assertHitCount(prepareSearch("test").setQuery(queryStringQuery("value*").field("field1").field("field2")), 1); - assertHitCount(prepareSearch("test").setQuery(queryStringQuery("field\\*:value*")), 1); - logger.info("wildcard"); - assertHitCount(prepareSearch("test").setQuery(queryStringQuery("v?lue*").field("field1").field("field2")), 1); - assertHitCount(prepareSearch("test").setQuery(queryStringQuery("field\\*:v?lue*")), 1); - logger.info("fuzzy"); - assertHitCount(prepareSearch("test").setQuery(queryStringQuery("value~").field("field1").field("field2")), 1); - assertHitCount(prepareSearch("test").setQuery(queryStringQuery("field\\*:value~")), 1); - logger.info("regexp"); - assertHitCount(prepareSearch("test").setQuery(queryStringQuery("/value[01]/").field("field1").field("field2")), 1); - assertHitCount(prepareSearch("test").setQuery(queryStringQuery("field\\*:/value[01]/")), 1); + assertHitCount( + 1, + prepareSearch("test").setQuery(queryStringQuery("value1").field("field1").field("field2")), + prepareSearch("test").setQuery(queryStringQuery("field\\*:value1")), + prepareSearch("test").setQuery(queryStringQuery("value*").field("field1").field("field2")), + prepareSearch("test").setQuery(queryStringQuery("field\\*:value*")), + prepareSearch("test").setQuery(queryStringQuery("v?lue*").field("field1").field("field2")), + prepareSearch("test").setQuery(queryStringQuery("field\\*:v?lue*")), + prepareSearch("test").setQuery(queryStringQuery("value~").field("field1").field("field2")), + prepareSearch("test").setQuery(queryStringQuery("field\\*:value~")), + prepareSearch("test").setQuery(queryStringQuery("/value[01]/").field("field1").field("field2")), + prepareSearch("test").setQuery(queryStringQuery("field\\*:/value[01]/")) + ); } // see #3797 @@ -1447,9 +1369,12 @@ public void testMultiMatchLenientIssue3797() { prepareIndex("test").setId("1").setSource("field1", 123, "field2", "value2").get(); refresh(); - assertHitCount(prepareSearch("test").setQuery(multiMatchQuery("value2", "field2").field("field1", 2).lenient(true)), 1L); - assertHitCount(prepareSearch("test").setQuery(multiMatchQuery("value2", "field2").field("field1", 2).lenient(true)), 1L); - assertHitCount(prepareSearch("test").setQuery(multiMatchQuery("value2").field("field2", 2).lenient(true)), 1L); + assertHitCount( + 1L, + prepareSearch("test").setQuery(multiMatchQuery("value2", "field2").field("field1", 2).lenient(true)), + prepareSearch("test").setQuery(multiMatchQuery("value2", "field2").field("field1", 2).lenient(true)), + prepareSearch("test").setQuery(multiMatchQuery("value2").field("field2", 2).lenient(true)) + ); } public void testMinScore() throws ExecutionException, InterruptedException { @@ -1482,24 +1407,15 @@ public void testQueryStringWithSlopAndFields() { assertHitCount(prepareSearch("test").setQuery(QueryBuilders.queryStringQuery("\"one two\"").defaultField("desc")), 2); assertHitCount( + 1, prepareSearch("test").setPostFilter(QueryBuilders.termQuery("type", "customer")) .setQuery(QueryBuilders.queryStringQuery("\"one two\"").field("desc")), - 1 - ); - assertHitCount( prepareSearch("test").setPostFilter(QueryBuilders.termQuery("type", "product")) .setQuery(QueryBuilders.queryStringQuery("\"one three\"~5").field("desc")), - 1 - ); - assertHitCount( prepareSearch("test").setPostFilter(QueryBuilders.termQuery("type", "customer")) .setQuery(QueryBuilders.queryStringQuery("\"one two\"").defaultField("desc")), - 1 - ); - assertHitCount( prepareSearch("test").setPostFilter(QueryBuilders.termQuery("type", "customer")) - .setQuery(QueryBuilders.queryStringQuery("\"one two\"").defaultField("desc")), - 1 + .setQuery(QueryBuilders.queryStringQuery("\"one two\"").defaultField("desc")) ); } @@ -1601,23 +1517,16 @@ public void testRangeQueryWithTimeZone() throws Exception { assertThat(response.getHits().getAt(0).getId(), is("2")); } ); - assertResponse( - prepareSearch("test").setQuery( - QueryBuilders.rangeQuery("date").from("2014-01-01T04:00:00").to("2014-01-01T04:59:00").timeZone("+03:00") - ), - response -> { - assertHitCount(response, 1L); - assertThat(response.getHits().getAt(0).getId(), is("3")); - } - ); - assertResponse( + assertResponses(response -> { + assertHitCount(response, 1L); + assertThat(response.getHits().getAt(0).getId(), is("3")); + }, prepareSearch("test").setQuery( QueryBuilders.rangeQuery("date").from("2014-01-01").to("2014-01-01T00:59:00").timeZone("-01:00") ), - response -> { - assertHitCount(response, 1L); - assertThat(response.getHits().getAt(0).getId(), is("3")); - } + prepareSearch("test").setQuery( + QueryBuilders.rangeQuery("date").from("2014-01-01T04:00:00").to("2014-01-01T04:59:00").timeZone("+03:00") + ) ); assertResponse(prepareSearch("test").setQuery(QueryBuilders.rangeQuery("date").from("now/d-1d").timeZone("+01:00")), response -> { assertHitCount(response, 1L); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/SimpleQueryStringIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/SimpleQueryStringIT.java index 7630ddb000140..35f11eb1429b4 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/SimpleQueryStringIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/SimpleQueryStringIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.query; @@ -581,6 +582,32 @@ public void testFieldAliasOnDisallowedFieldType() throws Exception { }); } + public void testSimpleQueryStringWithAnalysisStopWords() throws Exception { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject("body") + .field("type", "text") + .field("analyzer", "stop") + .endObject() + .endObject() + .endObject() + ); + + CreateIndexRequestBuilder mappingRequest = indicesAdmin().prepareCreate("test1").setMapping(mapping); + mappingRequest.get(); + indexRandom(true, prepareIndex("test1").setId("1").setSource("body", "Some Text")); + refresh(); + + assertHitCount( + prepareSearch().setQuery( + simpleQueryStringQuery("the* text*").analyzeWildcard(true).defaultOperator(Operator.AND).field("body") + ), + 1 + ); + } + private void assertHits(SearchHits hits, String... ids) { assertThat(hits.getTotalHits().value, equalTo((long) ids.length)); Set hitIds = new HashSet<>(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/rank/FieldBasedRerankerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/rank/FieldBasedRerankerIT.java index 1c0ef0f83be01..c8c1f50444c1d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/rank/FieldBasedRerankerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/rank/FieldBasedRerankerIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.rank; @@ -233,7 +234,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.RANK_FEATURE_PHASE_ADDED; + return TransportVersions.V_8_15_0; } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/rank/MockedRequestActionBasedRerankerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/rank/MockedRequestActionBasedRerankerIT.java index 636a7b033a4ee..dbdd6e8c50027 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/rank/MockedRequestActionBasedRerankerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/rank/MockedRequestActionBasedRerankerIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.rank; @@ -423,7 +424,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.RANK_FEATURE_PHASE_ADDED; + return TransportVersions.V_8_15_0; } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/MinimalCompoundRetrieverIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/MinimalCompoundRetrieverIT.java index 32dc34045cc8b..13a7d1fa59496 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/MinimalCompoundRetrieverIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/MinimalCompoundRetrieverIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.retriever; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java deleted file mode 100644 index fa4cafc66c822..0000000000000 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java +++ /dev/null @@ -1,755 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.search.retriever; - -import org.apache.lucene.search.TotalHits; -import org.apache.lucene.search.join.ScoreMode; -import org.apache.lucene.util.SetOnce; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.search.MultiSearchRequest; -import org.elasticsearch.action.search.MultiSearchResponse; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.TransportMultiSearchAction; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.Maps; -import org.elasticsearch.index.query.InnerHitBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.QueryRewriteContext; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.search.MockSearchService; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.elasticsearch.search.builder.PointInTimeBuilder; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.collapse.CollapseBuilder; -import org.elasticsearch.search.rank.RankDoc; -import org.elasticsearch.search.sort.FieldSortBuilder; -import org.elasticsearch.search.sort.NestedSortBuilder; -import org.elasticsearch.search.sort.ScoreSortBuilder; -import org.elasticsearch.search.sort.SortBuilder; -import org.elasticsearch.search.sort.SortOrder; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentType; -import org.junit.Before; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; -import static org.hamcrest.Matchers.equalTo; - -public class RankDocRetrieverBuilderIT extends ESIntegTestCase { - - @Override - protected Collection> nodePlugins() { - return List.of(MockSearchService.TestPlugin.class); - } - - public record RetrieverSource(RetrieverBuilder retriever, SearchSourceBuilder source) {} - - private static String INDEX = "test_index"; - private static final String ID_FIELD = "_id"; - private static final String DOC_FIELD = "doc"; - private static final String TEXT_FIELD = "text"; - private static final String VECTOR_FIELD = "vector"; - private static final String TOPIC_FIELD = "topic"; - private static final String LAST_30D_FIELD = "views.last30d"; - private static final String ALL_TIME_FIELD = "views.all"; - - @Before - public void setup() throws Exception { - String mapping = """ - { - "properties": { - "vector": { - "type": "dense_vector", - "dims": 3, - "element_type": "float", - "index": true, - "similarity": "l2_norm", - "index_options": { - "type": "hnsw" - } - }, - "text": { - "type": "text" - }, - "doc": { - "type": "keyword" - }, - "topic": { - "type": "keyword" - }, - "views": { - "type": "nested", - "properties": { - "last30d": { - "type": "integer" - }, - "all": { - "type": "integer" - } - } - } - } - } - """; - createIndex(INDEX, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).build()); - admin().indices().preparePutMapping(INDEX).setSource(mapping, XContentType.JSON).get(); - indexDoc( - INDEX, - "doc_1", - DOC_FIELD, - "doc_1", - TOPIC_FIELD, - "technology", - TEXT_FIELD, - "the quick brown fox jumps over the lazy dog", - LAST_30D_FIELD, - 100 - ); - indexDoc( - INDEX, - "doc_2", - DOC_FIELD, - "doc_2", - TOPIC_FIELD, - "astronomy", - TEXT_FIELD, - "you know, for Search!", - VECTOR_FIELD, - new float[] { 1.0f, 2.0f, 3.0f }, - LAST_30D_FIELD, - 3 - ); - indexDoc(INDEX, "doc_3", DOC_FIELD, "doc_3", TOPIC_FIELD, "technology", VECTOR_FIELD, new float[] { 6.0f, 6.0f, 6.0f }); - indexDoc( - INDEX, - "doc_4", - DOC_FIELD, - "doc_4", - TOPIC_FIELD, - "technology", - TEXT_FIELD, - "aardvark is a really awesome animal, but not very quick", - ALL_TIME_FIELD, - 100, - LAST_30D_FIELD, - 40 - ); - indexDoc(INDEX, "doc_5", DOC_FIELD, "doc_5", TOPIC_FIELD, "science", TEXT_FIELD, "irrelevant stuff"); - indexDoc( - INDEX, - "doc_6", - DOC_FIELD, - "doc_6", - TEXT_FIELD, - "quick quick quick quick search", - VECTOR_FIELD, - new float[] { 10.0f, 30.0f, 100.0f }, - LAST_30D_FIELD, - 15 - ); - indexDoc( - INDEX, - "doc_7", - DOC_FIELD, - "doc_7", - TOPIC_FIELD, - "biology", - TEXT_FIELD, - "dog", - VECTOR_FIELD, - new float[] { 3.0f, 3.0f, 3.0f }, - ALL_TIME_FIELD, - 1000 - ); - refresh(INDEX); - } - - public void testRankDocsRetrieverBasicWithPagination() { - final int rankWindowSize = 100; - SearchSourceBuilder source = new SearchSourceBuilder(); - StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); - // this one retrieves docs 1, 4, and 6 - standard0.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.queryStringQuery("quick").defaultField(TEXT_FIELD)) - .boost(10L); - StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); - // this one retrieves docs 2 and 6 due to prefilter - standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); - standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 7, 2, 3, and 6 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( - VECTOR_FIELD, - new float[] { 3.0f, 3.0f, 3.0f }, - null, - 10, - 100, - null - ); - // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and - // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results) - // so ideal rank would be: 6, 2, 1, 4, 7, 3 and with pagination, we'd just omit the first result - source.retriever( - new CompoundRetrieverWithRankDocs( - rankWindowSize, - Arrays.asList( - new RetrieverSource(standard0, null), - new RetrieverSource(standard1, null), - new RetrieverSource(knnRetrieverBuilder, null) - ) - ) - ); - // include some pagination as well - source.from(1); - SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); - ElasticsearchAssertions.assertResponse(req, resp -> { - assertNull(resp.pointInTimeId()); - assertNotNull(resp.getHits().getTotalHits()); - assertThat(resp.getHits().getTotalHits().value, equalTo(6L)); - assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); - assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_2")); - assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_1")); - assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_4")); - assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_7")); - assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_3")); - }); - } - - public void testRankDocsRetrieverWithAggs() { - // same as above, but we only want to bring back the top result from each subsearch - // so that would be 1, 2, and 7 - // and final rank would be (based on score): 2, 1, 7 - // aggs should still account for the same docs as the testRankDocsRetriever test, i.e. all but doc_5 - final int rankWindowSize = 1; - SearchSourceBuilder source = new SearchSourceBuilder(); - StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); - // this one retrieves docs 1, 4, and 6 - standard0.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.queryStringQuery("quick").defaultField(TEXT_FIELD)) - .boost(10L); - StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); - // this one retrieves docs 2 and 6 due to prefilter - standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); - standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 7, 2, 3, and 6 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( - VECTOR_FIELD, - new float[] { 3.0f, 3.0f, 3.0f }, - null, - 10, - 100, - null - ); - source.retriever( - new CompoundRetrieverWithRankDocs( - rankWindowSize, - Arrays.asList( - new RetrieverSource(standard0, null), - new RetrieverSource(standard1, null), - new RetrieverSource(knnRetrieverBuilder, null) - ) - ) - ); - source.aggregation(new TermsAggregationBuilder("topic").field(TOPIC_FIELD)); - SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); - ElasticsearchAssertions.assertResponse(req, resp -> { - assertNull(resp.pointInTimeId()); - assertNotNull(resp.getHits().getTotalHits()); - assertThat(resp.getHits().getTotalHits().value, equalTo(1L)); - assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); - assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_2")); - assertNotNull(resp.getAggregations()); - assertNotNull(resp.getAggregations().get("topic")); - Terms terms = resp.getAggregations().get("topic"); - // doc_3 is not part of the final aggs computation as it is only retrieved through the knn retriever - // and is outside of the rank window - assertThat(terms.getBucketByKey("technology").getDocCount(), equalTo(2L)); - assertThat(terms.getBucketByKey("astronomy").getDocCount(), equalTo(1L)); - assertThat(terms.getBucketByKey("biology").getDocCount(), equalTo(1L)); - }); - } - - public void testRankDocsRetrieverWithCollapse() { - final int rankWindowSize = 100; - SearchSourceBuilder source = new SearchSourceBuilder(); - StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); - // this one retrieves docs 1, 4, and 6 - standard0.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.queryStringQuery("quick").defaultField(TEXT_FIELD)) - .boost(10L); - StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); - // this one retrieves docs 2 and 6 due to prefilter - standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); - standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 7, 2, 3, and 6 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( - VECTOR_FIELD, - new float[] { 3.0f, 3.0f, 3.0f }, - null, - 10, - 100, - null - ); - // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and - // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results) - // so ideal rank would be: 6, 2, 1, 4, 7, 3 - // with collapsing on topic field we would have 6, 2, 1, 7 - source.retriever( - new CompoundRetrieverWithRankDocs( - rankWindowSize, - Arrays.asList( - new RetrieverSource(standard0, null), - new RetrieverSource(standard1, null), - new RetrieverSource(knnRetrieverBuilder, null) - ) - ) - ); - source.collapse( - new CollapseBuilder(TOPIC_FIELD).setInnerHits( - new InnerHitBuilder("a").addSort(new FieldSortBuilder(DOC_FIELD).order(SortOrder.DESC)).setSize(10) - ) - ); - source.fetchField(TOPIC_FIELD); - SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); - ElasticsearchAssertions.assertResponse(req, resp -> { - assertNull(resp.pointInTimeId()); - assertNotNull(resp.getHits().getTotalHits()); - assertThat(resp.getHits().getTotalHits().value, equalTo(6L)); - assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); - assertThat(resp.getHits().getHits().length, equalTo(4)); - assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_6")); - assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_2")); - assertThat(resp.getHits().getAt(1).field(TOPIC_FIELD).getValue().toString(), equalTo("astronomy")); - assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_1")); - assertThat(resp.getHits().getAt(2).field(TOPIC_FIELD).getValue().toString(), equalTo("technology")); - assertThat(resp.getHits().getAt(2).getInnerHits().get("a").getHits().length, equalTo(3)); - assertThat(resp.getHits().getAt(2).getInnerHits().get("a").getAt(0).getId(), equalTo("doc_4")); - assertThat(resp.getHits().getAt(2).getInnerHits().get("a").getAt(1).getId(), equalTo("doc_3")); - assertThat(resp.getHits().getAt(2).getInnerHits().get("a").getAt(2).getId(), equalTo("doc_1")); - assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_7")); - assertThat(resp.getHits().getAt(3).field(TOPIC_FIELD).getValue().toString(), equalTo("biology")); - }); - } - - public void testRankDocsRetrieverWithCollapseAndAggs() { - // same as above, but we only want to bring back the top result from each subsearch - // so that would be 1, 2, and 7 - // and final rank would be (based on score): 2, 1, 7 - // aggs should still account for the same docs as the testRankDocsRetriever test, i.e. all but doc_5 - final int rankWindowSize = 10; - SearchSourceBuilder source = new SearchSourceBuilder(); - StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); - // this one retrieves docs 1 and 6 as doc_4 is collapsed to doc_1 - standard0.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.queryStringQuery("quick").defaultField(TEXT_FIELD)) - .boost(10L); - standard0.collapseBuilder = new CollapseBuilder(TOPIC_FIELD).setInnerHits( - new InnerHitBuilder("a").addSort(new FieldSortBuilder(DOC_FIELD).order(SortOrder.DESC)).setSize(10) - ); - StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); - // this one retrieves docs 2 and 6 due to prefilter - standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); - standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 7, 2, 3, and 6 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( - VECTOR_FIELD, - new float[] { 3.0f, 3.0f, 3.0f }, - null, - 10, - 100, - null - ); - // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and - // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results) - // so ideal rank would be: 6, 2, 1, 4, 7, 3 - source.retriever( - new CompoundRetrieverWithRankDocs( - rankWindowSize, - Arrays.asList( - new RetrieverSource(standard0, null), - new RetrieverSource(standard1, null), - new RetrieverSource(knnRetrieverBuilder, null) - ) - ) - ); - source.aggregation(new TermsAggregationBuilder("topic").field(TOPIC_FIELD)); - SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); - ElasticsearchAssertions.assertResponse(req, resp -> { - assertNull(resp.pointInTimeId()); - assertNotNull(resp.getHits().getTotalHits()); - assertThat(resp.getHits().getTotalHits().value, equalTo(5L)); - assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); - assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_6")); - assertNotNull(resp.getAggregations()); - assertNotNull(resp.getAggregations().get("topic")); - Terms terms = resp.getAggregations().get("topic"); - // doc_3 is not part of the final aggs computation as it is only retrieved through the knn retriever - // and is outside of the rank window - assertThat(terms.getBucketByKey("technology").getDocCount(), equalTo(3L)); - assertThat(terms.getBucketByKey("astronomy").getDocCount(), equalTo(1L)); - assertThat(terms.getBucketByKey("biology").getDocCount(), equalTo(1L)); - }); - } - - public void testRankDocsRetrieverWithNestedQuery() { - final int rankWindowSize = 100; - SearchSourceBuilder source = new SearchSourceBuilder(); - StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); - // this one retrieves docs 1, 4, and 6 - standard0.queryBuilder = QueryBuilders.nestedQuery("views", QueryBuilders.rangeQuery(LAST_30D_FIELD).gt(10L), ScoreMode.Avg) - .innerHit(new InnerHitBuilder("a").addSort(new FieldSortBuilder(DOC_FIELD).order(SortOrder.DESC)).setSize(10)); - StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); - // this one retrieves docs 2 and 6 due to prefilter - standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); - standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 7, 2, 3, and 6 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( - VECTOR_FIELD, - new float[] { 3.0f, 3.0f, 3.0f }, - null, - 10, - 100, - null - ); - // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and - // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results) - // so ideal rank would be: 6, 2, 1, 4, 3, 7 - source.retriever( - new CompoundRetrieverWithRankDocs( - rankWindowSize, - Arrays.asList( - new RetrieverSource(standard0, null), - new RetrieverSource(standard1, null), - new RetrieverSource(knnRetrieverBuilder, null) - ) - ) - ); - source.fetchField(TOPIC_FIELD); - SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); - ElasticsearchAssertions.assertResponse(req, resp -> { - assertNull(resp.pointInTimeId()); - assertNotNull(resp.getHits().getTotalHits()); - assertThat(resp.getHits().getTotalHits().value, equalTo(6L)); - assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); - assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_6")); - assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_2")); - assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_1")); - assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_7")); - assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_4")); - assertThat(resp.getHits().getAt(5).getId(), equalTo("doc_3")); - }); - } - - public void testRankDocsRetrieverMultipleCompoundRetrievers() { - final int rankWindowSize = 100; - SearchSourceBuilder source = new SearchSourceBuilder(); - StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); - // this one retrieves docs 1, 4, and 6 - standard0.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.queryStringQuery("quick").defaultField(TEXT_FIELD)) - .boost(10L); - StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); - // this one retrieves docs 2 and 6 due to prefilter - standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); - standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 7, 2, 3, and 6 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( - VECTOR_FIELD, - new float[] { 3.0f, 3.0f, 3.0f }, - null, - 10, - 100, - null - ); - // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and - // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results) - // so ideal rank would be: 6, 2, 1, 4, 7, 3 - CompoundRetrieverWithRankDocs compoundRetriever1 = new CompoundRetrieverWithRankDocs( - rankWindowSize, - Arrays.asList( - new RetrieverSource(standard0, null), - new RetrieverSource(standard1, null), - new RetrieverSource(knnRetrieverBuilder, null) - ) - ); - // simple standard retriever that would have the doc_4 as its first (and only) result - StandardRetrieverBuilder standard2 = new StandardRetrieverBuilder(); - standard2.queryBuilder = QueryBuilders.queryStringQuery("aardvark").defaultField(TEXT_FIELD); - - // combining the two retrievers would bring doc_4 at the top as it would be the only one present in both doc sets - // the rest of the docs would be sorted based on their ranks as they have the same score (1/2) - source.retriever( - new CompoundRetrieverWithRankDocs( - rankWindowSize, - Arrays.asList(new RetrieverSource(compoundRetriever1, null), new RetrieverSource(standard2, null)) - ) - ); - - SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); - ElasticsearchAssertions.assertResponse(req, resp -> { - assertNull(resp.pointInTimeId()); - assertNotNull(resp.getHits().getTotalHits()); - assertThat(resp.getHits().getTotalHits().value, equalTo(6L)); - assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); - assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_4")); - assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_6")); - assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_2")); - assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_1")); - assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_7")); - assertThat(resp.getHits().getAt(5).getId(), equalTo("doc_3")); - }); - } - - public void testRankDocsRetrieverDifferentNestedSorting() { - final int rankWindowSize = 100; - SearchSourceBuilder source = new SearchSourceBuilder(); - StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); - // this one retrieves docs 1, 4, 6, 2 - standard0.queryBuilder = QueryBuilders.nestedQuery("views", QueryBuilders.rangeQuery(LAST_30D_FIELD).gt(0), ScoreMode.Avg); - standard0.sortBuilders = List.of( - new FieldSortBuilder(LAST_30D_FIELD).setNestedSort(new NestedSortBuilder("views")).order(SortOrder.DESC) - ); - StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); - // this one retrieves docs 4, 7 - standard1.queryBuilder = QueryBuilders.nestedQuery("views", QueryBuilders.rangeQuery(ALL_TIME_FIELD).gt(0), ScoreMode.Avg); - standard1.sortBuilders = List.of( - new FieldSortBuilder(ALL_TIME_FIELD).setNestedSort(new NestedSortBuilder("views")).order(SortOrder.ASC) - ); - - source.retriever( - new CompoundRetrieverWithRankDocs( - rankWindowSize, - Arrays.asList(new RetrieverSource(standard0, null), new RetrieverSource(standard1, null)) - ) - ); - - SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); - ElasticsearchAssertions.assertResponse(req, resp -> { - assertNull(resp.pointInTimeId()); - assertNotNull(resp.getHits().getTotalHits()); - assertThat(resp.getHits().getTotalHits().value, equalTo(5L)); - assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); - assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_4")); - assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_1")); - assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_7")); - assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_6")); - assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_2")); - }); - } - - class CompoundRetrieverWithRankDocs extends RetrieverBuilder { - - private final List sources; - private final int rankWindowSize; - - private CompoundRetrieverWithRankDocs(int rankWindowSize, List sources) { - this.rankWindowSize = rankWindowSize; - this.sources = Collections.unmodifiableList(sources); - } - - @Override - public boolean isCompound() { - return true; - } - - @Override - public QueryBuilder topDocsQuery() { - throw new UnsupportedOperationException("should not be called"); - } - - @Override - public RetrieverBuilder rewrite(QueryRewriteContext ctx) throws IOException { - if (ctx.getPointInTimeBuilder() == null) { - throw new IllegalStateException("PIT is required"); - } - - // Rewrite prefilters - boolean hasChanged = false; - var newPreFilters = rewritePreFilters(ctx); - hasChanged |= newPreFilters != preFilterQueryBuilders; - - // Rewrite retriever sources - List newRetrievers = new ArrayList<>(); - for (var entry : sources) { - RetrieverBuilder newRetriever = entry.retriever.rewrite(ctx); - if (newRetriever != entry.retriever) { - newRetrievers.add(new RetrieverSource(newRetriever, null)); - hasChanged |= newRetriever != entry.retriever; - } else if (newRetriever == entry.retriever) { - var sourceBuilder = entry.source != null - ? entry.source - : createSearchSourceBuilder(ctx.getPointInTimeBuilder(), newRetriever); - var rewrittenSource = sourceBuilder.rewrite(ctx); - newRetrievers.add(new RetrieverSource(newRetriever, rewrittenSource)); - hasChanged |= rewrittenSource != entry.source; - } - } - if (hasChanged) { - return new CompoundRetrieverWithRankDocs(rankWindowSize, newRetrievers); - } - - // execute searches - final SetOnce results = new SetOnce<>(); - final MultiSearchRequest multiSearchRequest = new MultiSearchRequest(); - for (var entry : sources) { - SearchRequest searchRequest = new SearchRequest().source(entry.source); - // The can match phase can reorder shards, so we disable it to ensure the stable ordering - searchRequest.setPreFilterShardSize(Integer.MAX_VALUE); - multiSearchRequest.add(searchRequest); - } - ctx.registerAsyncAction((client, listener) -> { - client.execute(TransportMultiSearchAction.TYPE, multiSearchRequest, new ActionListener<>() { - @Override - public void onResponse(MultiSearchResponse items) { - List topDocs = new ArrayList<>(); - for (int i = 0; i < items.getResponses().length; i++) { - var item = items.getResponses()[i]; - var rankDocs = getRankDocs(item.getResponse()); - sources.get(i).retriever().setRankDocs(rankDocs); - topDocs.add(rankDocs); - } - results.set(combineResults(topDocs)); - listener.onResponse(null); - } - - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }); - }); - - return new RankDocsRetrieverBuilder( - rankWindowSize, - newRetrievers.stream().map(s -> s.retriever).toList(), - results::get, - newPreFilters - ); - } - - @Override - public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) { - throw new UnsupportedOperationException("should not be called"); - } - - @Override - public String getName() { - return "compound_retriever"; - } - - @Override - protected void doToXContent(XContentBuilder builder, Params params) throws IOException { - - } - - @Override - protected boolean doEquals(Object o) { - return false; - } - - @Override - protected int doHashCode() { - return 0; - } - - private RankDoc[] getRankDocs(SearchResponse searchResponse) { - assert searchResponse != null; - int size = Math.min(rankWindowSize, searchResponse.getHits().getHits().length); - RankDoc[] docs = new RankDoc[size]; - for (int i = 0; i < size; i++) { - var hit = searchResponse.getHits().getAt(i); - long sortValue = (long) hit.getRawSortValues()[hit.getRawSortValues().length - 1]; - int doc = decodeDoc(sortValue); - int shardRequestIndex = decodeShardRequestIndex(sortValue); - docs[i] = new RankDoc(doc, hit.getScore(), shardRequestIndex); - docs[i].rank = i + 1; - } - return docs; - } - - public static int decodeDoc(long value) { - return (int) value; - } - - public static int decodeShardRequestIndex(long value) { - return (int) (value >> 32); - } - - record RankDocAndHitRatio(RankDoc rankDoc, float hitRatio) {} - - /** - * Combines the provided {@code rankResults} to return the final top documents. - */ - public RankDoc[] combineResults(List rankResults) { - int totalQueries = rankResults.size(); - final float step = 1.0f / totalQueries; - Map docsToRankResults = Maps.newMapWithExpectedSize(rankWindowSize); - for (var rankResult : rankResults) { - for (RankDoc scoreDoc : rankResult) { - docsToRankResults.compute(new RankDoc.RankKey(scoreDoc.doc, scoreDoc.shardIndex), (key, value) -> { - if (value == null) { - RankDoc res = new RankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex); - res.rank = scoreDoc.rank; - return new RankDocAndHitRatio(res, step); - } else { - RankDoc res = new RankDoc(scoreDoc.doc, Math.max(scoreDoc.score, value.rankDoc.score), scoreDoc.shardIndex); - res.rank = Math.min(scoreDoc.rank, value.rankDoc.rank); - return new RankDocAndHitRatio(res, value.hitRatio + step); - } - }); - } - } - // sort the results based on hit ratio, then doc, then rank, and final tiebreaker is based on smaller doc id - RankDocAndHitRatio[] sortedResults = docsToRankResults.values().toArray(RankDocAndHitRatio[]::new); - Arrays.sort(sortedResults, (RankDocAndHitRatio doc1, RankDocAndHitRatio doc2) -> { - if (doc1.hitRatio != doc2.hitRatio) { - return doc1.hitRatio < doc2.hitRatio ? 1 : -1; - } - if (false == (Float.isNaN(doc1.rankDoc.score) || Float.isNaN(doc2.rankDoc.score)) - && (doc1.rankDoc.score != doc2.rankDoc.score)) { - return doc1.rankDoc.score < doc2.rankDoc.score ? 1 : -1; - } - if (doc1.rankDoc.rank != doc2.rankDoc.rank) { - return doc1.rankDoc.rank < doc2.rankDoc.rank ? -1 : 1; - } - return doc1.rankDoc.doc < doc2.rankDoc.doc ? -1 : 1; - }); - // trim the results if needed, otherwise each shard will always return `rank_window_size` results. - // pagination and all else will happen on the coordinator when combining the shard responses - RankDoc[] topResults = new RankDoc[Math.min(rankWindowSize, sortedResults.length)]; - for (int rank = 0; rank < topResults.length; ++rank) { - topResults[rank] = sortedResults[rank].rankDoc; - topResults[rank].rank = rank + 1; - topResults[rank].score = sortedResults[rank].hitRatio; - } - return topResults; - } - } - - private SearchSourceBuilder createSearchSourceBuilder(PointInTimeBuilder pit, RetrieverBuilder retrieverBuilder) { - var sourceBuilder = new SearchSourceBuilder().pointInTimeBuilder(pit).trackTotalHits(false).size(100); - retrieverBuilder.extractToSearchSourceBuilder(sourceBuilder, false); - - // Record the shard id in the sort result - List> sortBuilders = sourceBuilder.sorts() != null ? new ArrayList<>(sourceBuilder.sorts()) : new ArrayList<>(); - if (sortBuilders.isEmpty()) { - sortBuilders.add(new ScoreSortBuilder()); - } - sortBuilders.add(new FieldSortBuilder(FieldSortBuilder.SHARD_DOC_FIELD_NAME)); - sourceBuilder.sort(sortBuilders); - return sourceBuilder; - } -} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverRewriteIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverRewriteIT.java index e6ecd9f1e3779..43197b77b2c1e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverRewriteIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverRewriteIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.retriever; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverTelemetryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverTelemetryIT.java new file mode 100644 index 0000000000000..40849bea5512e --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverTelemetryIT.java @@ -0,0 +1,155 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.search.retriever; + +import org.elasticsearch.action.admin.cluster.node.capabilities.NodesCapabilitiesRequest; +import org.elasticsearch.action.admin.cluster.node.capabilities.NodesCapabilitiesResponse; +import org.elasticsearch.action.admin.cluster.stats.SearchUsageStats; +import org.elasticsearch.client.Request; +import org.elasticsearch.common.Strings; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.vectors.KnnSearchBuilder; +import org.elasticsearch.search.vectors.KnnVectorQueryBuilder; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.junit.Before; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) +public class RetrieverTelemetryIT extends ESIntegTestCase { + + private static final String INDEX_NAME = "test_index"; + + @Override + protected boolean addMockHttpTransport() { + return false; // enable http + } + + @Before + public void setup() throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject("vector") + .field("type", "dense_vector") + .field("dims", 1) + .field("index", true) + .field("similarity", "l2_norm") + .startObject("index_options") + .field("type", "hnsw") + .endObject() + .endObject() + .startObject("text") + .field("type", "text") + .endObject() + .startObject("integer") + .field("type", "integer") + .endObject() + .startObject("topic") + .field("type", "keyword") + .endObject() + .endObject() + .endObject(); + + assertAcked(prepareCreate(INDEX_NAME).setMapping(builder)); + ensureGreen(INDEX_NAME); + } + + private void performSearch(SearchSourceBuilder source) throws IOException { + Request request = new Request("GET", INDEX_NAME + "/_search"); + request.setJsonEntity(Strings.toString(source)); + getRestClient().performRequest(request); + } + + public void testTelemetryForRetrievers() throws IOException { + + if (false == isRetrieverTelemetryEnabled()) { + return; + } + + // search#1 - this will record 1 entry for "retriever" in `sections`, and 1 for "knn" under `retrievers` + { + performSearch( + new SearchSourceBuilder().retriever(new KnnRetrieverBuilder("vector", new float[] { 1.0f }, null, 10, 15, null, null)) + ); + } + + // search#2 - this will record 1 entry for "retriever" in `sections`, 1 for "standard" under `retrievers`, and 1 for "range" under + // `queries` + { + performSearch(new SearchSourceBuilder().retriever(new StandardRetrieverBuilder(QueryBuilders.rangeQuery("integer").gte(2)))); + } + + // search#3 - this will record 1 entry for "retriever" in `sections`, and 1 for "standard" under `retrievers`, and 1 for "knn" under + // `queries` + { + performSearch( + new SearchSourceBuilder().retriever( + new StandardRetrieverBuilder(new KnnVectorQueryBuilder("vector", new float[] { 1.0f }, 10, 15, null, null)) + ) + ); + } + + // search#4 - this will record 1 entry for "retriever" in `sections`, and 1 for "standard" under `retrievers`, and 1 for "term" + // under `queries` + { + performSearch(new SearchSourceBuilder().retriever(new StandardRetrieverBuilder(QueryBuilders.termQuery("topic", "foo")))); + } + + // search#5 - t + // his will record 1 entry for "knn" in `sections` + { + performSearch( + new SearchSourceBuilder().knnSearch(List.of(new KnnSearchBuilder("vector", new float[] { 1.0f }, 10, 15, null, null))) + ); + } + + // search#6 - this will record 1 entry for "query" in `sections`, and 1 for "match_all" under `queries` + { + performSearch(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())); + } + + // cluster stats + { + SearchUsageStats stats = clusterAdmin().prepareClusterStats().get().getIndicesStats().getSearchUsageStats(); + assertEquals(6, stats.getTotalSearchCount()); + + assertThat(stats.getSectionsUsage().size(), equalTo(3)); + assertThat(stats.getSectionsUsage().get("retriever"), equalTo(4L)); + assertThat(stats.getSectionsUsage().get("query"), equalTo(1L)); + assertThat(stats.getSectionsUsage().get("knn"), equalTo(1L)); + + assertThat(stats.getRetrieversUsage().size(), equalTo(2)); + assertThat(stats.getRetrieversUsage().get("standard"), equalTo(3L)); + assertThat(stats.getRetrieversUsage().get("knn"), equalTo(1L)); + + assertThat(stats.getQueryUsage().size(), equalTo(4)); + assertThat(stats.getQueryUsage().get("range"), equalTo(1L)); + assertThat(stats.getQueryUsage().get("term"), equalTo(1L)); + assertThat(stats.getQueryUsage().get("match_all"), equalTo(1L)); + assertThat(stats.getQueryUsage().get("knn"), equalTo(1L)); + } + } + + private boolean isRetrieverTelemetryEnabled() throws IOException { + NodesCapabilitiesResponse res = clusterAdmin().nodesCapabilities( + new NodesCapabilitiesRequest().method(RestRequest.Method.GET).path("_cluster/stats").capabilities("retrievers-usage-stats") + ).actionGet(); + return res != null && res.isSupported().orElse(false); + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/routing/SearchPreferenceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/routing/SearchPreferenceIT.java index 17a0d6441ca47..31db65140e1ee 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/routing/SearchPreferenceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/routing/SearchPreferenceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.routing; @@ -32,6 +33,7 @@ import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponses; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -67,25 +69,20 @@ public void testStopOneNodePreferenceWithRedState() throws IOException { "_prefer_nodes:somenode,server2" }; for (String pref : preferences) { logger.info("--> Testing out preference={}", pref); - assertResponse(prepareSearch().setSize(0).setPreference(pref), response -> { + assertResponses(response -> { assertThat(RestStatus.OK, equalTo(response.status())); assertThat(pref, response.getFailedShards(), greaterThanOrEqualTo(0)); - }); - assertResponse(prepareSearch().setPreference(pref), response -> { - assertThat(RestStatus.OK, equalTo(response.status())); - assertThat(pref, response.getFailedShards(), greaterThanOrEqualTo(0)); - }); + }, prepareSearch().setSize(0).setPreference(pref), prepareSearch().setPreference(pref)); } // _only_local is a stricter preference, we need to send the request to a data node - assertResponse(dataNodeClient().prepareSearch().setSize(0).setPreference("_only_local"), response -> { + assertResponses(response -> { assertThat(RestStatus.OK, equalTo(response.status())); assertThat("_only_local", response.getFailedShards(), greaterThanOrEqualTo(0)); - }); - assertResponse(dataNodeClient().prepareSearch().setPreference("_only_local"), response -> { - assertThat(RestStatus.OK, equalTo(response.status())); - assertThat("_only_local", response.getFailedShards(), greaterThanOrEqualTo(0)); - }); + }, + dataNodeClient().prepareSearch().setSize(0).setPreference("_only_local"), + dataNodeClient().prepareSearch().setPreference("_only_local") + ); } public void testNoPreferenceRandom() { @@ -120,19 +117,11 @@ public void testSimplePreference() { prepareIndex("test").setSource("field1", "value1").get(); refresh(); - assertResponse( + assertResponses( + response -> assertThat(response.getHits().getTotalHits().value, equalTo(1L)), prepareSearch().setQuery(matchAllQuery()), - response -> assertThat(response.getHits().getTotalHits().value, equalTo(1L)) - ); - - assertResponse( prepareSearch().setQuery(matchAllQuery()).setPreference("_local"), - response -> assertThat(response.getHits().getTotalHits().value, equalTo(1L)) - ); - - assertResponse( - prepareSearch().setQuery(matchAllQuery()).setPreference("1234"), - response -> assertThat(response.getHits().getTotalHits().value, equalTo(1L)) + prepareSearch().setQuery(matchAllQuery()).setPreference("1234") ); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/routing/SearchReplicaSelectionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/routing/SearchReplicaSelectionIT.java index 439534c3e1743..33b554a508e2b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/routing/SearchReplicaSelectionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/routing/SearchReplicaSelectionIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.routing; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java index 9f6ad69a4eed6..2c96c27a0d12d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.scriptfilter; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/DuelScrollIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/DuelScrollIT.java index 89d4c0cc74852..d3da4639a3927 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/DuelScrollIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/DuelScrollIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.scroll; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/SearchScrollIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/SearchScrollIT.java index 24a3d3ac422f3..7c3dde22ce9d0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/SearchScrollIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/SearchScrollIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.scroll; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/SearchScrollWithFailingNodesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/SearchScrollWithFailingNodesIT.java index 42be70e5ff8b2..918c5019207ee 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/SearchScrollWithFailingNodesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/scroll/SearchScrollWithFailingNodesIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.scroll; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchAfterIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchAfterIT.java index b9c3c27abf2d8..7c459f91a1ac0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchAfterIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchAfterIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.searchafter; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java index dd8cf5e527055..a62a042a3cab5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.simple; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/slice/SearchSliceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/slice/SearchSliceIT.java index 59373380d539c..979cb9e8a8c4c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/slice/SearchSliceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/slice/SearchSliceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.slice; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java index 7915637ca7ac5..1cca9e814617c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.sort; @@ -2100,10 +2101,12 @@ public void testLongSortOptimizationCorrectResults() { } public void testSortMixedFieldTypes() { - assertAcked(prepareCreate("index_long").setMapping("foo", "type=long").get()); - assertAcked(prepareCreate("index_integer").setMapping("foo", "type=integer").get()); - assertAcked(prepareCreate("index_double").setMapping("foo", "type=double").get()); - assertAcked(prepareCreate("index_keyword").setMapping("foo", "type=keyword").get()); + assertAcked( + prepareCreate("index_long").setMapping("foo", "type=long"), + prepareCreate("index_integer").setMapping("foo", "type=integer"), + prepareCreate("index_double").setMapping("foo", "type=double"), + prepareCreate("index_keyword").setMapping("foo", "type=keyword") + ); prepareIndex("index_long").setId("1").setSource("foo", "123").get(); prepareIndex("index_integer").setId("1").setSource("foo", "123").get(); @@ -2135,9 +2138,11 @@ public void testSortMixedFieldTypes() { } public void testSortMixedFieldTypesWithNoDocsForOneType() { - assertAcked(prepareCreate("index_long").setMapping("foo", "type=long").get()); - assertAcked(prepareCreate("index_other").setMapping("bar", "type=keyword").get()); - assertAcked(prepareCreate("index_double").setMapping("foo", "type=double").get()); + assertAcked( + prepareCreate("index_long").setMapping("foo", "type=long"), + prepareCreate("index_other").setMapping("bar", "type=keyword"), + prepareCreate("index_double").setMapping("foo", "type=double") + ); prepareIndex("index_long").setId("1").setSource("foo", "123").get(); prepareIndex("index_long").setId("2").setSource("foo", "124").get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/GeoDistanceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/GeoDistanceIT.java index 2d0fbb42a42e2..e80678c4f5fc6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/GeoDistanceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/GeoDistanceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.sort; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java index 67426caf2faab..1383f33a41d84 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.sort; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/SimpleSortIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/SimpleSortIT.java index c7b934df0394f..ae0d2cbeb841f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/SimpleSortIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/SimpleSortIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.sort; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/source/MetadataFetchingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/source/MetadataFetchingIT.java index 9e0dd984c9a2a..6351d8d906389 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/source/MetadataFetchingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/source/MetadataFetchingIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.source; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/source/SourceFetchingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/source/SourceFetchingIT.java index 81facfa8116c0..616fc2e1f3483 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/source/SourceFetchingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/source/SourceFetchingIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.source; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/stats/FieldUsageStatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/stats/FieldUsageStatsIT.java index 160cba19700ac..3d5120226ebed 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/stats/FieldUsageStatsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/stats/FieldUsageStatsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.stats; @@ -157,11 +158,15 @@ public void testFieldUsageStats() throws ExecutionException, InterruptedExceptio assertTrue(stats.hasField("date_field")); assertEquals(Set.of(UsageContext.POINTS), stats.get("date_field").keySet()); - // can_match does not enter search stats - // there is a special case though where we have no hit but we need to get at least one search response in order - // to produce a valid search result with all the aggs etc., so we hit one of the two shards + + long expectedShards = 2L * numShards; + if (numShards == 1) { + // with 1 shard and setPreFilterShardSize(1) we don't perform can_match phase but instead directly query the shard + expectedShards += 1; + } + assertEquals( - (2 * numShards) + 1, + expectedShards, indicesAdmin().prepareStats("test") .clear() .setSearch(true) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/stats/SearchStatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/stats/SearchStatsIT.java index e5ca2c6968bb9..2530dd35946fe 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/stats/SearchStatsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/stats/SearchStatsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.stats; @@ -12,11 +13,14 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.index.search.stats.SearchStats.Stats; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.MockScriptPlugin; @@ -25,10 +29,14 @@ import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.lookup.Source; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.FailingFieldPlugin; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.json.JsonXContent; import java.util.Collection; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Function; @@ -52,7 +60,7 @@ public class SearchStatsIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - return Collections.singleton(CustomScriptPlugin.class); + return List.of(CustomScriptPlugin.class, FailingFieldPlugin.class); } public static class CustomScriptPlugin extends MockScriptPlugin { @@ -243,4 +251,64 @@ protected int numAssignedShards(String... indices) { GroupShardsIterator allAssignedShardsGrouped = state.routingTable().allAssignedShardsGrouped(indices, true); return allAssignedShardsGrouped.size(); } + + public void testFailureStats() throws Exception { + String indexName = "test"; + XContentBuilder mapping = JsonXContent.contentBuilder().startObject(); + mapping.startObject("runtime"); + { + mapping.startObject("fail_me"); + { + mapping.field("type", "long"); + mapping.startObject("script").field("source", "").field("lang", "failing_field").endObject(); + } + mapping.endObject(); + } + mapping.endObject(); + mapping.endObject(); + int numOfShards = between(1, 5); + client().admin() + .indices() + .prepareCreate(indexName) + .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards)) + .setMapping(mapping) + .get(); + int numDocs = between(20, 100); + for (int i = 1; i < numDocs; i++) { + index(indexName, Integer.toString(i), Map.of("position", i)); + } + refresh(indexName); + int numQueries = between(1, 10); + long failedQueries = 0; + for (int q = 0; q < numQueries; q++) { + expectThrows(Exception.class, () -> { + client().prepareSearch(indexName) + .setQuery(new RangeQueryBuilder("fail_me").gt(10)) + .setAllowPartialSearchResults(true) + .get(); + }); + failedQueries += numOfShards; + var stats = client().admin().indices().prepareStats(indexName).all().get().getTotal().search.getTotal(); + assertThat(stats.getQueryCount(), equalTo(0L)); + assertThat(stats.getQueryFailure(), equalTo(failedQueries)); + assertThat(stats.getFetchCount(), equalTo(0L)); + assertThat(stats.getFetchFailure(), equalTo(0L)); + } + int numFetches = between(1, 10); + for (int q = 0; q < numFetches; q++) { + expectThrows(Exception.class, () -> { + client().prepareSearch(indexName) + .setQuery(new RangeQueryBuilder("position").gt(0)) + .setFetchSource(false) + .addFetchField("fail_me") + .setSize(1000) + .get(); + }); + var stats = client().admin().indices().prepareStats(indexName).all().get().getTotal().search.getTotal(); + assertThat(stats.getQueryCount(), equalTo((q + 1L) * numOfShards)); + assertThat(stats.getQueryFailure(), equalTo(failedQueries)); + assertThat(stats.getFetchCount(), equalTo(0L)); + assertThat(stats.getFetchFailure(), equalTo((q + 1L) * numOfShards)); + } + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java index 20c5c11f36756..8b21bb54361b6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.suggest; @@ -60,6 +61,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponses; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasScore; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; @@ -158,21 +160,13 @@ public void testTextAndGlobalText() throws Exception { } indexRandom(true, indexRequestBuilders); CompletionSuggestionBuilder noText = SuggestBuilders.completionSuggestion(FIELD); - assertResponse( - prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", noText).setGlobalText("sugg")), - response -> assertSuggestions(response, "foo", "suggestion10", "suggestion9", "suggestion8", "suggestion7", "suggestion6") - ); - CompletionSuggestionBuilder withText = SuggestBuilders.completionSuggestion(FIELD).text("sugg"); - assertResponse( + assertResponses( + response -> assertSuggestions(response, "foo", "suggestion10", "suggestion9", "suggestion8", "suggestion7", "suggestion6"), + prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", noText).setGlobalText("sugg")), prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", withText)), - response -> assertSuggestions(response, "foo", "suggestion10", "suggestion9", "suggestion8", "suggestion7", "suggestion6") - ); - - // test that suggestion text takes precedence over global text - assertResponse( - prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", withText).setGlobalText("bogus")), - response -> assertSuggestions(response, "foo", "suggestion10", "suggestion9", "suggestion8", "suggestion7", "suggestion6") + // test that suggestion text takes precedence over global text + prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", withText).setGlobalText("bogus")) ); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java index 22d48c2f282d5..8eca50353b945 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.suggest; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/suggest/SuggestSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/suggest/SuggestSearchIT.java index a7cf141eb7669..9bb6b35b007ee 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/suggest/SuggestSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/suggest/SuggestSearchIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.search.suggest; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/similarity/SimilarityIT.java b/server/src/internalClusterTest/java/org/elasticsearch/similarity/SimilarityIT.java index 55dca7810f845..2952150c2cb22 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/similarity/SimilarityIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/similarity/SimilarityIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.similarity; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/AbortedRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/AbortedRestoreIT.java index d3dba66055e01..6f1940732db98 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/AbortedRestoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/AbortedRestoreIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.snapshots; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/AbortedSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/AbortedSnapshotIT.java index 86a4d728df787..6ad2c4d94a826 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/AbortedSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/AbortedSnapshotIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.snapshots; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java index ed0e226fc377b..616a4b5a772da 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.snapshots; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CloneSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CloneSnapshotIT.java index d7c7acf9737a1..caf7f852cc15e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CloneSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CloneSnapshotIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.snapshots; @@ -564,9 +565,7 @@ public void testSnapshotQueuedAfterCloneFromBrokenSourceSnapshot() throws Except assertSuccessful(sn1); assertSuccessful(sn2); assertSuccessful(sn3); - assertAcked(clone1.get()); - assertAcked(clone2.get()); - assertAcked(clone3.get()); + assertAcked(clone1, clone2, clone3); } public void testStartSnapshotWithSuccessfulShardClonePendingFinalization() throws Exception { @@ -623,8 +622,7 @@ public void testStartCloneWithSuccessfulShardClonePendingFinalization() throws E unblockNode(repoName, masterName); awaitNoMoreRunningOperations(masterName); awaitMasterFinishRepoOperations(); - assertAcked(blockedClone.get()); - assertAcked(otherClone.get()); + assertAcked(blockedClone, otherClone); assertEquals(getSnapshot(repoName, cloneName).state(), SnapshotState.SUCCESS); assertEquals(getSnapshot(repoName, otherCloneName).state(), SnapshotState.SUCCESS); } @@ -731,8 +729,7 @@ public void testManyConcurrentClonesStartOutOfOrder() throws Exception { awaitClusterState(state -> SnapshotsInProgress.get(state).forRepo(repoName).stream().anyMatch(entry -> entry.state().completed())); repo.unblock(); - assertAcked(clone1.get()); - assertAcked(clone2.get()); + assertAcked(clone1, clone2); } public void testRemoveFailedCloneFromCSWithoutIO() throws Exception { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java index 5bdf156e39999..e16aff56527c3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.snapshots; @@ -17,7 +18,6 @@ import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStatus; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexClusterStateUpdateRequest; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.GroupedActionListener; import org.elasticsearch.action.support.PlainActionFuture; @@ -32,9 +32,7 @@ import org.elasticsearch.common.util.concurrent.ListenableFuture; import org.elasticsearch.common.util.concurrent.UncategorizedExecutionException; import org.elasticsearch.core.PathUtils; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.discovery.AbstractDisruptionTestCase; -import org.elasticsearch.index.Index; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.RepositoryConflictException; @@ -58,6 +56,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -479,8 +478,7 @@ public void testCascadedAborts() throws Exception { assertThat(thirdSnapshotResponse.get().getSnapshotInfo().state(), is(SnapshotState.FAILED)); logger.info("--> verify both deletes have completed"); - assertAcked(deleteSnapshotsResponse.get()); - assertAcked(allDeletedResponse.get()); + assertAcked(deleteSnapshotsResponse, allDeletedResponse); logger.info("--> verify that all snapshots are gone"); assertThat(clusterAdmin().prepareGetSnapshots(TEST_REQUEST_TIMEOUT, repoName).get().getSnapshots(), empty()); @@ -716,8 +714,7 @@ public void testQueuedOperationsOnMasterDisconnect() throws Exception { networkDisruption.stopDisrupting(); logger.info("--> make sure all failing requests get a response"); - assertAcked(firstDeleteFuture.get()); - assertAcked(secondDeleteFuture.get()); + assertAcked(firstDeleteFuture, secondDeleteFuture); expectThrows(SnapshotException.class, createSnapshot); awaitNoMoreRunningOperations(); } @@ -1015,8 +1012,7 @@ public void testBackToBackQueuedDeletes() throws Exception { awaitNDeletionsInProgress(2); unblockNode(repoName, masterName); - assertAcked(deleteSnapshotOne.get()); - assertAcked(deleteSnapshotTwo.get()); + assertAcked(deleteSnapshotOne, deleteSnapshotTwo); final RepositoryData repositoryData = getRepositoryData(repoName); assertThat(repositoryData.getSnapshotIds(), empty()); @@ -1362,9 +1358,12 @@ public void testConcurrentOperationsLimit() throws Exception { if (deleteAndAbortAll) { awaitNumberOfSnapshotsInProgress(0); for (ActionFuture snapshotFuture : snapshotFutures) { - // just check that the futures resolve, whether or not things worked out with the snapshot actually finalizing or failing - // due to the abort does not matter - assertBusy(() -> assertTrue(snapshotFuture.isDone())); + try { + snapshotFuture.get(); + } catch (ExecutionException e) { + // just check that the futures resolve, whether or not things worked out with the snapshot actually finalizing or + // failing due to the abort does not matter + } } assertThat(getRepositoryData(repoName).getSnapshotIds(), empty()); } else { @@ -1891,8 +1890,7 @@ public void testIndexDeletedWhileSnapshotAndCloneQueuedAfterClone() throws Excep assertSuccessful(snapshot3); unblockNode(repository, master); - assertAcked(cloneSnapshot.get()); - assertAcked(cloneSnapshot2.get()); + assertAcked(cloneSnapshot, cloneSnapshot2); assertAcked(startDeleteSnapshot(repository, cloneTarget).get()); assertThat( @@ -2032,8 +2030,7 @@ public void testCloneQueuedAfterMissingShard() throws Exception { awaitNumberOfSnapshotsInProgress(2); unblockNode(repository, master); - assertAcked(deleteFuture.get()); - assertAcked(cloneFuture.get()); + assertAcked(deleteFuture, cloneFuture); awaitNoMoreRunningOperations(); assertThat(snapshot1.get().getSnapshotInfo().state(), is(SnapshotState.PARTIAL)); } @@ -2110,8 +2107,7 @@ public void testSnapshotAndCloneQueuedAfterMissingShard() throws Exception { awaitNumberOfSnapshotsInProgress(3); unblockNode(repository, master); - assertAcked(deleteFuture.get()); - assertAcked(cloneFuture.get()); + assertAcked(deleteFuture, cloneFuture); awaitNoMoreRunningOperations(); assertThat(snapshot1.get().getSnapshotInfo().state(), is(SnapshotState.PARTIAL)); assertThat(snapshot2.get().getSnapshotInfo().state(), is(SnapshotState.PARTIAL)); @@ -2213,12 +2209,17 @@ public void testDeleteIndexWithOutOfOrderFinalization() { .anyMatch(e -> e.snapshot().getSnapshotId().getName().equals("snapshot-with-index-1") && e.state().completed()) ) // execute the index deletion _directly on the master_ so it happens before the snapshot finalization executes - .andThen(l -> masterDeleteIndexService.deleteIndices(new DeleteIndexClusterStateUpdateRequest(l.map(r -> { - assertTrue(r.isAcknowledged()); - return null; - })).indices(new Index[] { internalCluster().clusterService().state().metadata().index(indexToDelete).getIndex() }) - .ackTimeout(TimeValue.timeValueSeconds(10)) - .masterNodeTimeout(TimeValue.timeValueSeconds(10)))) + .andThen( + l -> masterDeleteIndexService.deleteIndices( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + Set.of(internalCluster().clusterService().state().metadata().index(indexToDelete).getIndex()), + l.map(r -> { + assertTrue(r.isAcknowledged()); + return null; + }) + ) + ) // ultimately create the index again so that taking a full snapshot will pick up any missing shard gen blob, and deleting that // full snapshot will clean up all dangling shard-level blobs .andThen((l, ignored) -> prepareCreate(indexToDelete, indexSettingsNoReplicas(1)).execute(l.map(r -> { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java index 5a82b4b1ab99e..eea60ce13af2f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.snapshots; @@ -381,9 +382,10 @@ public void testMountCorruptedRepositoryData() throws Exception { Files.write(repo.resolve(getRepositoryDataBlobName(repositoryData.getGenId())), randomByteArrayOfLength(randomIntBetween(1, 100))); logger.info("--> verify loading repository data throws RepositoryException"); - asInstanceOf( + safeAwaitFailure( RepositoryException.class, - safeAwaitFailure(RepositoryData.class, l -> repository.getRepositoryData(EsExecutors.DIRECT_EXECUTOR_SERVICE, l)) + RepositoryData.class, + l -> repository.getRepositoryData(EsExecutors.DIRECT_EXECUTOR_SERVICE, l) ); final String otherRepoName = "other-repo"; @@ -396,9 +398,10 @@ public void testMountCorruptedRepositoryData() throws Exception { final Repository otherRepo = getRepositoryOnMaster(otherRepoName); logger.info("--> verify loading repository data from newly mounted repository throws RepositoryException"); - asInstanceOf( + safeAwaitFailure( RepositoryException.class, - safeAwaitFailure(RepositoryData.class, l -> repository.getRepositoryData(EsExecutors.DIRECT_EXECUTOR_SERVICE, l)) + RepositoryData.class, + l -> repository.getRepositoryData(EsExecutors.DIRECT_EXECUTOR_SERVICE, l) ); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CustomMetadataContextIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CustomMetadataContextIT.java index 08f9d74ab477e..cec43d2dc834c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CustomMetadataContextIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CustomMetadataContextIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.snapshots; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 3788f2dd2cb61..0d359300bbdc1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.snapshots; @@ -663,8 +664,7 @@ public void testRestoreShrinkIndex() throws Exception { createSnapshot(repo, snapshot, Collections.singletonList(shrunkIdx)); logger.info("--> delete index and stop the data node"); - assertAcked(indicesAdmin().prepareDelete(sourceIdx).get()); - assertAcked(indicesAdmin().prepareDelete(shrunkIdx).get()); + assertAcked(indicesAdmin().prepareDelete(sourceIdx), indicesAdmin().prepareDelete(shrunkIdx)); internalCluster().stopRandomDataNode(); clusterAdmin().prepareHealth(TEST_REQUEST_TIMEOUT).setTimeout(TimeValue.timeValueSeconds(30)).setWaitForNodes("1"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/FeatureStateResetApiIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/FeatureStateResetApiIT.java index 64c168ae73905..c3eda84ee9e5e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/FeatureStateResetApiIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/FeatureStateResetApiIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.snapshots; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java index 6f02ac5c983a4..c0e63520fee9e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.snapshots; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java index 8ab1ddd85fc55..d07e5a94bc401 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.snapshots; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MultiClusterRepoAccessIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MultiClusterRepoAccessIT.java index 2daa36ee00a01..77c7b4b762e6a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MultiClusterRepoAccessIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MultiClusterRepoAccessIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.snapshots; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java index 6870a1d6b2649..c318ebf78dd96 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.snapshots; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorServiceIT.java index 8a0b74242ba9d..4b042d0301d64 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorServiceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.snapshots; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoryThrottlingStatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoryThrottlingStatsIT.java index 7664bbd6c91ee..b0563bc86b122 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoryThrottlingStatsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoryThrottlingStatsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.snapshots; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java index 725fcbc1a5849..5b54a0dbbb799 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.snapshots; @@ -22,6 +23,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.indices.InvalidIndexNameException; @@ -760,6 +762,86 @@ public void testChangeSettingsOnRestore() throws Exception { assertHitCount(client.prepareSearch("test-idx").setSize(0).setQuery(matchQuery("field1", "bar")), numdocs); } + public void testRestoreChangeIndexMode() { + Client client = client(); + createRepository("test-repo", "fs"); + String indexName = "test-idx"; + assertAcked(client.admin().indices().prepareCreate(indexName).setSettings(Settings.builder().put(indexSettings()))); + createSnapshot("test-repo", "test-snap", Collections.singletonList(indexName)); + cluster().wipeIndices(indexName); + for (IndexMode mode : IndexMode.values()) { + var error = expectThrows(SnapshotRestoreException.class, () -> { + client.admin() + .cluster() + .prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, "test-repo", "test-snap") + .setIndexSettings(Settings.builder().put("index.mode", mode.name())) + .setWaitForCompletion(true) + .get(); + }); + assertThat(error.getMessage(), containsString("cannot modify setting [index.mode] on restore")); + } + } + + public void testRestoreChangeSyntheticSource() { + Client client = client(); + createRepository("test-repo", "fs"); + String indexName = "test-idx"; + assertAcked(client.admin().indices().prepareCreate(indexName).setSettings(Settings.builder().put(indexSettings()))); + createSnapshot("test-repo", "test-snap", Collections.singletonList(indexName)); + cluster().wipeIndices(indexName); + var error = expectThrows(SnapshotRestoreException.class, () -> { + client.admin() + .cluster() + .prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, "test-repo", "test-snap") + .setIndexSettings(Settings.builder().put("index.mapping.source.mode", "synthetic")) + .setWaitForCompletion(true) + .get(); + }); + assertThat(error.getMessage(), containsString("cannot modify setting [index.mapping.source.mode] on restore")); + } + + public void testRestoreChangeRecoveryUseSyntheticSource() { + Client client = client(); + createRepository("test-repo", "fs"); + String indexName = "test-idx"; + assertAcked(client.admin().indices().prepareCreate(indexName).setSettings(Settings.builder().put(indexSettings()))); + createSnapshot("test-repo", "test-snap", Collections.singletonList(indexName)); + cluster().wipeIndices(indexName); + var error = expectThrows(SnapshotRestoreException.class, () -> { + client.admin() + .cluster() + .prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, "test-repo", "test-snap") + .setIndexSettings(Settings.builder().put("index.recovery.use_synthetic_source", true)) + .setWaitForCompletion(true) + .get(); + }); + assertThat(error.getMessage(), containsString("cannot modify setting [index.recovery.use_synthetic_source] on restore")); + } + + public void testRestoreChangeIndexSorts() { + Client client = client(); + createRepository("test-repo", "fs"); + String indexName = "test-idx"; + assertAcked( + client.admin() + .indices() + .prepareCreate(indexName) + .setMapping("host.name", "type=keyword", "@timestamp", "type=date") + .setSettings(Settings.builder().put(indexSettings()).putList("index.sort.field", List.of("@timestamp", "host.name"))) + ); + createSnapshot("test-repo", "test-snap", Collections.singletonList(indexName)); + cluster().wipeIndices(indexName); + var error = expectThrows(SnapshotRestoreException.class, () -> { + client.admin() + .cluster() + .prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, "test-repo", "test-snap") + .setIndexSettings(Settings.builder().putList("index.sort.field", List.of("host.name"))) + .setWaitForCompletion(true) + .get(); + }); + assertThat(error.getMessage(), containsString("cannot modify setting [index.sort.field] on restore")); + } + public void testRecreateBlocksOnRestore() throws Exception { Client client = client(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index 08daeaaec016b..2b00efa49ca55 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.snapshots; @@ -62,6 +63,7 @@ import org.elasticsearch.repositories.RepositoryException; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.snapshots.mockstore.MockRepository; +import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.threadpool.ThreadPool; import java.nio.channels.SeekableByteChannel; @@ -75,8 +77,10 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.function.Predicate; @@ -86,6 +90,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY; import static org.elasticsearch.index.shard.IndexShardTests.getEngineFromShard; +import static org.elasticsearch.node.NodeRoleSettings.NODE_ROLES_SETTING; import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.METADATA_NAME_FORMAT; import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.READONLY_SETTING_KEY; import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.SNAPSHOT_NAME_FORMAT; @@ -1404,6 +1409,88 @@ public void testCloseOrDeleteIndexDuringSnapshot() throws Exception { assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo((SnapshotState.SUCCESS))); } + public void testCloseOrReallocateDuringPartialSnapshot() { + final var repoName = randomIdentifier(); + createRepository(repoName, "mock"); + + final var blockingNode = internalCluster().startNode( + Settings.builder().put(NODE_ROLES_SETTING.getKey(), "data").put("thread_pool." + ThreadPool.Names.SNAPSHOT + ".max", 1) + ); + + // blocking the snapshot thread pool to ensure that we only retain the shard lock while actively running snapshot tasks + final var barrier = new CyclicBarrier(2); + final var keepGoing = new AtomicBoolean(true); + final var blockingNodeExecutor = internalCluster().getInstance(ThreadPool.class, blockingNode).executor(ThreadPool.Names.SNAPSHOT); + blockingNodeExecutor.execute(new Runnable() { + @Override + public void run() { + safeAwait(barrier); + safeAwait(barrier); + if (keepGoing.get()) { + blockingNodeExecutor.execute(this); + } + } + }); + + final var indexName = randomIdentifier(); + createIndex(indexName, indexSettings(1, 0).put(IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "._name", blockingNode).build()); + indexRandomDocs(indexName, between(1, 100)); + flushAndRefresh(indexName); + + safeAwait(barrier); + + final var snapshotName = randomIdentifier(); + final var partialSnapshot = randomBoolean(); + ActionFuture snapshotFuture = clusterAdmin().prepareCreateSnapshot( + TEST_REQUEST_TIMEOUT, + repoName, + snapshotName + ).setIndices(indexName).setWaitForCompletion(true).setPartial(partialSnapshot).execute(); + + // we have currently blocked the start-snapshot task from running, and it will be followed by at least three blob uploads + // (segments_N, .cfe, .cfs), executed one-at-a-time because of throttling to the max threadpool size, so it's safe to let up to + // three tasks through without the snapshot being able to complete + final var snapshotTasks = between(0, 3); + logger.info("--> running (at most) {} tasks", snapshotTasks); + for (int i = 0; i < snapshotTasks; i++) { + safeAwait(barrier); + safeAwait(barrier); + } + assertFalse(snapshotFuture.isDone()); + + try { + if (partialSnapshot && randomBoolean()) { + logger.info("--> closing index [{}]", indexName); + safeGet(indicesAdmin().prepareClose(indexName).execute()); + ensureGreen(indexName); + } else { + logger.info("--> failing index [{}] to trigger recovery", indexName); + IndexShard indexShard = null; + for (IndexService indexService : internalCluster().getInstance(IndicesService.class, blockingNode)) { + if (indexService.index().getName().equals(indexName)) { + indexShard = indexService.getShard(0); + break; + } + } + assertNotNull(indexShard); + final var primaryTerm = indexShard.getOperationPrimaryTerm(); + indexShard.failShard("simulated", new ElasticsearchException("simulated")); + safeAwait( + ClusterServiceUtils.addTemporaryStateListener( + internalCluster().getInstance(ClusterService.class), + cs -> cs.metadata().index(indexName).primaryTerm(0) > primaryTerm + ) + ); + ensureGreen(indexName); + } + } finally { + keepGoing.set(false); + safeAwait(barrier); + } + + safeGet(snapshotFuture); + } + public void testCloseIndexDuringRestore() throws Exception { Client client = client(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotBrokenSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotBrokenSettingsIT.java index 92e0b437cbebb..18e047f3f6687 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotBrokenSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotBrokenSettingsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.snapshots; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotCustomPluginStateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotCustomPluginStateIT.java index 8f2702099c102..110c33afdd9c6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotCustomPluginStateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotCustomPluginStateIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.snapshots; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShardsServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShardsServiceIT.java index 773be25a4ca9d..feb73d4b6f4c1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShardsServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShardsServiceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.snapshots; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java index 3d16293c1462b..53263468bf0de 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.snapshots; @@ -501,11 +502,16 @@ private static void putShutdownMetadata( clusterService.submitUnbatchedStateUpdateTask("mark node for removal", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) { - final var nodeId = currentState.nodes().resolveNode(nodeName).getId(); + final var node = currentState.nodes().resolveNode(nodeName); return currentState.copyAndUpdateMetadata( mdb -> mdb.putCustom( NodesShutdownMetadata.TYPE, - new NodesShutdownMetadata(Map.of(nodeId, shutdownMetadataBuilder.setNodeId(nodeId).build())) + new NodesShutdownMetadata( + Map.of( + node.getId(), + shutdownMetadataBuilder.setNodeId(node.getId()).setNodeEphemeralId(node.getEphemeralId()).build() + ) + ) ) ); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStatusApisIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStatusApisIT.java index d4e27d5630c5b..6922b21be37f7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStatusApisIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStatusApisIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.snapshots; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java index e8d61e4677c98..ac322868989bb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.snapshots; @@ -1234,15 +1235,16 @@ public ClusterState execute(ClusterState currentState) { Strings.toString(currentState), currentState.metadata().nodeShutdowns().getAll().isEmpty() ); - final var nodeId = currentState.nodes().resolveNode(node.nodeName).getId(); + final var discoveryNode = currentState.nodes().resolveNode(node.nodeName); return currentState.copyAndUpdateMetadata( mdb -> mdb.putCustom( NodesShutdownMetadata.TYPE, new NodesShutdownMetadata( Map.of( - nodeId, + discoveryNode.getId(), SingleNodeShutdownMetadata.builder() - .setNodeId(nodeId) + .setNodeId(discoveryNode.getId()) + .setNodeEphemeralId(discoveryNode.getEphemeralId()) .setType(SingleNodeShutdownMetadata.Type.REMOVE) .setStartedAtMillis(clusterService.threadPool().absoluteTimeInMillis()) .setReason("test") diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotThrottlingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotThrottlingIT.java index c1b3203b15666..d664e31b91383 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotThrottlingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotThrottlingIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.snapshots; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceDoubleFinalizationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceDoubleFinalizationIT.java index 6bdb70bfba3e2..616ad294cf865 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceDoubleFinalizationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceDoubleFinalizationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.snapshots; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceIT.java index ca9fdc1284e83..b9e47740e2945 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.snapshots; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java index 32f76bdcc61e8..c6e02300ccef6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.snapshots; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/synonyms/SynonymsManagementAPIServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/synonyms/SynonymsManagementAPIServiceIT.java index f6392954912ac..a7b98b4639130 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/synonyms/SynonymsManagementAPIServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/synonyms/SynonymsManagementAPIServiceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.synonyms; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java b/server/src/internalClusterTest/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java index d98b1e7d4e526..be875421e036f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.threadpool; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java b/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java index 2ef951b28d3aa..8991f94081505 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.timeseries.support; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/transport/RemoteClusterPortSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/transport/RemoteClusterPortSettingsIT.java index 8a54d41ccb65d..d94dd8ca7429d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/transport/RemoteClusterPortSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/transport/RemoteClusterPortSettingsIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/transport/TransportServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/transport/TransportServiceIT.java index e6fc8e90f6676..e592bf509da6c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/transport/TransportServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/transport/TransportServiceIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.transport; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/update/UpdateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/update/UpdateIT.java index f6ff9c4f21f1b..c21ad82a6095f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/update/UpdateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/update/UpdateIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.update; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/update/UpdateNoopIT.java b/server/src/internalClusterTest/java/org/elasticsearch/update/UpdateNoopIT.java index eb3f248d38044..0532a4e09d9a0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/update/UpdateNoopIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/update/UpdateNoopIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.update; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/validate/SimpleValidateQueryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/validate/SimpleValidateQueryIT.java index 1e8cc954cd850..388421b6dd53f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/validate/SimpleValidateQueryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/validate/SimpleValidateQueryIT.java @@ -1,23 +1,26 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.validate; import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; -import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.MoreLikeThisQueryBuilder.Item; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.TermsQueryBuilder; +import org.elasticsearch.indices.IndexClosedException; import org.elasticsearch.indices.TermsLookup; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; @@ -206,12 +209,8 @@ public void testExplainDateRangeInQueryString() { } public void testValidateEmptyCluster() { - try { - indicesAdmin().prepareValidateQuery().get(); - fail("Expected IndexNotFoundException"); - } catch (IndexNotFoundException e) { - assertThat(e.getMessage(), is("no such index [_all] and no indices exist")); - } + ValidateQueryResponse response = indicesAdmin().prepareValidateQuery().get(); + assertThat(response.getTotalShards(), is(0)); } public void testExplainNoQuery() { @@ -378,4 +377,52 @@ public void testExplainTermsQueryWithLookup() { ValidateQueryResponse response = indicesAdmin().prepareValidateQuery("twitter").setQuery(termsLookupQuery).setExplain(true).get(); assertThat(response.isValid(), is(true)); } + + public void testOneClosedIndex() { + createIndex("test"); + + boolean ignoreUnavailable = false; + IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false); + client().admin().indices().close(new CloseIndexRequest("test")).actionGet(); + IndexClosedException ex = expectThrows( + IndexClosedException.class, + indicesAdmin().prepareValidateQuery("test").setIndicesOptions(options) + ); + assertEquals("closed", ex.getMessage()); + } + + public void testOneClosedIndexIgnoreUnavailable() { + createIndex("test"); + + boolean ignoreUnavailable = true; + IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false); + client().admin().indices().close(new CloseIndexRequest("test")).actionGet(); + ValidateQueryResponse response = indicesAdmin().prepareValidateQuery("test").setIndicesOptions(options).get(); + assertThat(response.getTotalShards(), is(0)); + } + + public void testTwoIndicesOneClosed() { + createIndex("test1"); + createIndex("test2"); + + boolean ignoreUnavailable = false; + IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false); + client().admin().indices().close(new CloseIndexRequest("test1")).actionGet(); + IndexClosedException ex = expectThrows( + IndexClosedException.class, + indicesAdmin().prepareValidateQuery("test1", "test2").setIndicesOptions(options) + ); + assertEquals("closed", ex.getMessage()); + } + + public void testTwoIndicesOneClosedIgnoreUnavailable() { + createIndex("test1"); + createIndex("test2"); + + boolean ignoreUnavailable = true; + IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false); + client().admin().indices().close(new CloseIndexRequest("test1")).actionGet(); + ValidateQueryResponse response = indicesAdmin().prepareValidateQuery("test1", "test2").setIndicesOptions(options).get(); + assertThat(response.getTotalShards(), is(1)); + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentDocumentOperationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentDocumentOperationIT.java index 69c10edf89809..2baf68212a1dd 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentDocumentOperationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentDocumentOperationIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.versioning; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java b/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java index 2b804293cd506..8db81b985b439 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.versioning; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/versioning/SimpleVersioningIT.java b/server/src/internalClusterTest/java/org/elasticsearch/versioning/SimpleVersioningIT.java index 339187f56d8c3..0af49f5aac93f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/versioning/SimpleVersioningIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/versioning/SimpleVersioningIT.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.versioning; diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 086bfece87172..5f23316b31988 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -1,13 +1,12 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ -import org.elasticsearch.index.codec.Elasticsearch814Codec; -import org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat; import org.elasticsearch.plugins.internal.RestExtension; /** The Elasticsearch Server Module. */ @@ -24,7 +23,6 @@ requires org.elasticsearch.nativeaccess; requires org.elasticsearch.geo; requires org.elasticsearch.lz4; - requires org.elasticsearch.pluginclassloader; requires org.elasticsearch.securesm; requires org.elasticsearch.xcontent; requires org.elasticsearch.logging; @@ -33,6 +31,7 @@ requires org.elasticsearch.grok; requires org.elasticsearch.tdigest; requires org.elasticsearch.simdvec; + requires org.elasticsearch.entitlement; requires hppc; requires HdrHistogram; @@ -159,7 +158,6 @@ exports org.elasticsearch.client.internal.support; exports org.elasticsearch.client.internal.transport; exports org.elasticsearch.cluster; - exports org.elasticsearch.cluster.ack; exports org.elasticsearch.cluster.action.index; exports org.elasticsearch.cluster.action.shard; exports org.elasticsearch.cluster.block; @@ -419,7 +417,6 @@ uses org.elasticsearch.internal.BuildExtension; uses org.elasticsearch.features.FeatureSpecification; uses org.elasticsearch.plugins.internal.LoggingDataProvider; - uses org.elasticsearch.cluster.metadata.DataStreamFactoryRetention; provides org.elasticsearch.features.FeatureSpecification with @@ -437,7 +434,8 @@ org.elasticsearch.search.SearchFeatures, org.elasticsearch.script.ScriptFeatures, org.elasticsearch.search.retriever.RetrieversFeatures, - org.elasticsearch.reservedstate.service.FileSettingsFeatures; + org.elasticsearch.reservedstate.service.FileSettingsFeatures, + org.elasticsearch.action.admin.cluster.stats.ClusterStatsFeatures; uses org.elasticsearch.plugins.internal.SettingsExtension; uses RestExtension; @@ -448,16 +446,23 @@ org.elasticsearch.index.codec.bloomfilter.ES85BloomFilterPostingsFormat, org.elasticsearch.index.codec.bloomfilter.ES87BloomFilterPostingsFormat, org.elasticsearch.index.codec.postings.ES812PostingsFormat; - provides org.apache.lucene.codecs.DocValuesFormat with ES87TSDBDocValuesFormat; + provides org.apache.lucene.codecs.DocValuesFormat with org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat; provides org.apache.lucene.codecs.KnnVectorsFormat with org.elasticsearch.index.codec.vectors.ES813FlatVectorFormat, org.elasticsearch.index.codec.vectors.ES813Int8FlatVectorFormat, org.elasticsearch.index.codec.vectors.ES814HnswScalarQuantizedVectorsFormat, org.elasticsearch.index.codec.vectors.ES815HnswBitVectorsFormat, - org.elasticsearch.index.codec.vectors.ES815BitFlatVectorFormat; + org.elasticsearch.index.codec.vectors.ES815BitFlatVectorFormat, + org.elasticsearch.index.codec.vectors.es816.ES816BinaryQuantizedVectorsFormat, + org.elasticsearch.index.codec.vectors.es816.ES816HnswBinaryQuantizedVectorsFormat, + org.elasticsearch.index.codec.vectors.es818.ES818BinaryQuantizedVectorsFormat, + org.elasticsearch.index.codec.vectors.es818.ES818HnswBinaryQuantizedVectorsFormat; - provides org.apache.lucene.codecs.Codec with Elasticsearch814Codec; + provides org.apache.lucene.codecs.Codec + with + org.elasticsearch.index.codec.Elasticsearch814Codec, + org.elasticsearch.index.codec.Elasticsearch816Codec; provides org.apache.logging.log4j.core.util.ContextDataProvider with org.elasticsearch.common.logging.DynamicContextDataProvider; @@ -467,5 +472,7 @@ org.elasticsearch.serverless.shardhealth, org.elasticsearch.serverless.apifiltering; exports org.elasticsearch.lucene.spatial; - + exports org.elasticsearch.inference.configuration; + exports org.elasticsearch.monitor.metrics; + exports org.elasticsearch.plugins.internal.rewriter to org.elasticsearch.inference; } diff --git a/server/src/main/java/org/elasticsearch/Build.java b/server/src/main/java/org/elasticsearch/Build.java index 1ab2128a72f53..feccfa489d76f 100644 --- a/server/src/main/java/org/elasticsearch/Build.java +++ b/server/src/main/java/org/elasticsearch/Build.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch; diff --git a/server/src/main/java/org/elasticsearch/ElasticsearchAuthenticationProcessingError.java b/server/src/main/java/org/elasticsearch/ElasticsearchAuthenticationProcessingError.java index c12ef1c213bf2..de0943123ed65 100644 --- a/server/src/main/java/org/elasticsearch/ElasticsearchAuthenticationProcessingError.java +++ b/server/src/main/java/org/elasticsearch/ElasticsearchAuthenticationProcessingError.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch; diff --git a/server/src/main/java/org/elasticsearch/ElasticsearchCorruptionException.java b/server/src/main/java/org/elasticsearch/ElasticsearchCorruptionException.java index d53b6149ded94..a8ecca892ab3f 100644 --- a/server/src/main/java/org/elasticsearch/ElasticsearchCorruptionException.java +++ b/server/src/main/java/org/elasticsearch/ElasticsearchCorruptionException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch; diff --git a/server/src/main/java/org/elasticsearch/ElasticsearchException.java b/server/src/main/java/org/elasticsearch/ElasticsearchException.java index c7f3e25b6a96e..981aafa5424f0 100644 --- a/server/src/main/java/org/elasticsearch/ElasticsearchException.java +++ b/server/src/main/java/org/elasticsearch/ElasticsearchException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch; @@ -13,6 +14,7 @@ import org.apache.lucene.index.IndexFormatTooOldException; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.LockObtainFailedException; +import org.elasticsearch.action.bulk.IndexDocFailureStoreStatus; import org.elasticsearch.action.support.replication.ReplicationOperation; import org.elasticsearch.cluster.action.shard.ShardStateAction; import org.elasticsearch.common.io.stream.NotSerializableExceptionWrapper; @@ -1921,13 +1923,19 @@ private enum ElasticsearchExceptionHandle { ResourceAlreadyUploadedException.class, ResourceAlreadyUploadedException::new, 181, - TransportVersions.ADD_RESOURCE_ALREADY_UPLOADED_EXCEPTION + TransportVersions.V_8_15_0 ), INGEST_PIPELINE_EXCEPTION( org.elasticsearch.ingest.IngestPipelineException.class, org.elasticsearch.ingest.IngestPipelineException::new, 182, - TransportVersions.INGEST_PIPELINE_EXCEPTION_ADDED + TransportVersions.V_8_16_0 + ), + INDEX_RESPONSE_WRAPPER_EXCEPTION( + IndexDocFailureStoreStatus.ExceptionWithFailureStoreStatus.class, + IndexDocFailureStoreStatus.ExceptionWithFailureStoreStatus::new, + 183, + TransportVersions.V_8_16_0 ); final Class exceptionClass; diff --git a/server/src/main/java/org/elasticsearch/ElasticsearchGenerationException.java b/server/src/main/java/org/elasticsearch/ElasticsearchGenerationException.java index 3aaf696198142..1b81fe1d74264 100644 --- a/server/src/main/java/org/elasticsearch/ElasticsearchGenerationException.java +++ b/server/src/main/java/org/elasticsearch/ElasticsearchGenerationException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch; diff --git a/server/src/main/java/org/elasticsearch/ElasticsearchParseException.java b/server/src/main/java/org/elasticsearch/ElasticsearchParseException.java index 9b2e401bece93..9d77725549c90 100644 --- a/server/src/main/java/org/elasticsearch/ElasticsearchParseException.java +++ b/server/src/main/java/org/elasticsearch/ElasticsearchParseException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch; diff --git a/server/src/main/java/org/elasticsearch/ElasticsearchRoleRestrictionException.java b/server/src/main/java/org/elasticsearch/ElasticsearchRoleRestrictionException.java index a399617d0dccd..4515caf5be694 100644 --- a/server/src/main/java/org/elasticsearch/ElasticsearchRoleRestrictionException.java +++ b/server/src/main/java/org/elasticsearch/ElasticsearchRoleRestrictionException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch; diff --git a/server/src/main/java/org/elasticsearch/ElasticsearchSecurityException.java b/server/src/main/java/org/elasticsearch/ElasticsearchSecurityException.java index dacc1cee6f779..bdefee544a29e 100644 --- a/server/src/main/java/org/elasticsearch/ElasticsearchSecurityException.java +++ b/server/src/main/java/org/elasticsearch/ElasticsearchSecurityException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch; diff --git a/server/src/main/java/org/elasticsearch/ElasticsearchStatusException.java b/server/src/main/java/org/elasticsearch/ElasticsearchStatusException.java index b09b0279296c1..bbc9fa89ce110 100644 --- a/server/src/main/java/org/elasticsearch/ElasticsearchStatusException.java +++ b/server/src/main/java/org/elasticsearch/ElasticsearchStatusException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch; diff --git a/server/src/main/java/org/elasticsearch/ElasticsearchTimeoutException.java b/server/src/main/java/org/elasticsearch/ElasticsearchTimeoutException.java index ddb6525e70e59..eddce19c77888 100644 --- a/server/src/main/java/org/elasticsearch/ElasticsearchTimeoutException.java +++ b/server/src/main/java/org/elasticsearch/ElasticsearchTimeoutException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch; diff --git a/server/src/main/java/org/elasticsearch/ElasticsearchWrapperException.java b/server/src/main/java/org/elasticsearch/ElasticsearchWrapperException.java index ddfe5a9d99bfa..178d71626b651 100644 --- a/server/src/main/java/org/elasticsearch/ElasticsearchWrapperException.java +++ b/server/src/main/java/org/elasticsearch/ElasticsearchWrapperException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch; diff --git a/server/src/main/java/org/elasticsearch/ExceptionsHelper.java b/server/src/main/java/org/elasticsearch/ExceptionsHelper.java index 3e109fb1600b9..ec04b63a575db 100644 --- a/server/src/main/java/org/elasticsearch/ExceptionsHelper.java +++ b/server/src/main/java/org/elasticsearch/ExceptionsHelper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch; @@ -18,6 +19,9 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.index.Index; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.transport.ConnectTransportException; +import org.elasticsearch.transport.NoSeedNodeLeftException; +import org.elasticsearch.transport.NoSuchRemoteClusterException; import org.elasticsearch.xcontent.XContentParseException; import java.io.IOException; @@ -471,7 +475,7 @@ public static ShardOperationFailedException[] groupBy(ShardOperationFailedExcept } /** - * Utility method useful for determine whether to log an Exception or perhaps + * Utility method useful for determining whether to log an Exception or perhaps * avoid logging a stacktrace if the caller/logger is not interested in these * types of node/shard issues. * @@ -489,6 +493,27 @@ public static boolean isNodeOrShardUnavailableTypeException(Throwable t) { || t instanceof org.elasticsearch.cluster.block.ClusterBlockException); } + /** + * Checks the exception against a known list of exceptions that indicate a remote cluster + * cannot be connected to. + * + * @param e Exception to inspect + * @return true if the Exception is known to indicate that a remote cluster + * is unavailable (cannot be connected to by the transport layer) + */ + public static boolean isRemoteUnavailableException(Exception e) { + Throwable unwrap = unwrap(e, ConnectTransportException.class, NoSuchRemoteClusterException.class, NoSeedNodeLeftException.class); + if (unwrap != null) { + return true; + } + Throwable ill = unwrap(e, IllegalStateException.class, IllegalArgumentException.class); + if (ill != null && (ill.getMessage().contains("Unable to open any connections") || ill.getMessage().contains("unknown host"))) { + return true; + } + // doesn't look like any of the known remote exceptions + return false; + } + private static class GroupBy { final String reason; final String index; diff --git a/server/src/main/java/org/elasticsearch/ReleaseVersions.java b/server/src/main/java/org/elasticsearch/ReleaseVersions.java index bb90bc79a528a..181021a7d344d 100644 --- a/server/src/main/java/org/elasticsearch/ReleaseVersions.java +++ b/server/src/main/java/org/elasticsearch/ReleaseVersions.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch; diff --git a/server/src/main/java/org/elasticsearch/ResourceAlreadyExistsException.java b/server/src/main/java/org/elasticsearch/ResourceAlreadyExistsException.java index e28a350088b5e..137dd79d8d22a 100644 --- a/server/src/main/java/org/elasticsearch/ResourceAlreadyExistsException.java +++ b/server/src/main/java/org/elasticsearch/ResourceAlreadyExistsException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch; diff --git a/server/src/main/java/org/elasticsearch/ResourceAlreadyUploadedException.java b/server/src/main/java/org/elasticsearch/ResourceAlreadyUploadedException.java index b18ee264b703b..11d4829038417 100644 --- a/server/src/main/java/org/elasticsearch/ResourceAlreadyUploadedException.java +++ b/server/src/main/java/org/elasticsearch/ResourceAlreadyUploadedException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch; diff --git a/server/src/main/java/org/elasticsearch/ResourceNotFoundException.java b/server/src/main/java/org/elasticsearch/ResourceNotFoundException.java index 880ebfdbc6c2f..c461d84aa6740 100644 --- a/server/src/main/java/org/elasticsearch/ResourceNotFoundException.java +++ b/server/src/main/java/org/elasticsearch/ResourceNotFoundException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch; diff --git a/server/src/main/java/org/elasticsearch/SecuredConfigFileAccessPermission.java b/server/src/main/java/org/elasticsearch/SecuredConfigFileAccessPermission.java index d6372b5ef9885..43bb55ffe6f2f 100644 --- a/server/src/main/java/org/elasticsearch/SecuredConfigFileAccessPermission.java +++ b/server/src/main/java/org/elasticsearch/SecuredConfigFileAccessPermission.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch; diff --git a/server/src/main/java/org/elasticsearch/SecuredConfigFileSettingAccessPermission.java b/server/src/main/java/org/elasticsearch/SecuredConfigFileSettingAccessPermission.java index fdea47d449a15..c6835be9de94e 100644 --- a/server/src/main/java/org/elasticsearch/SecuredConfigFileSettingAccessPermission.java +++ b/server/src/main/java/org/elasticsearch/SecuredConfigFileSettingAccessPermission.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch; diff --git a/server/src/main/java/org/elasticsearch/SpecialPermission.java b/server/src/main/java/org/elasticsearch/SpecialPermission.java index b29bc43fa7b05..33bfc6719e6e9 100644 --- a/server/src/main/java/org/elasticsearch/SpecialPermission.java +++ b/server/src/main/java/org/elasticsearch/SpecialPermission.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch; diff --git a/server/src/main/java/org/elasticsearch/TransportVersion.java b/server/src/main/java/org/elasticsearch/TransportVersion.java index 48970b97e480e..37f6e6434735c 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersion.java +++ b/server/src/main/java/org/elasticsearch/TransportVersion.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch; diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 5189094294239..ef410e6181c3b 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch; @@ -101,113 +102,60 @@ static TransportVersion def(int id) { public static final TransportVersion V_8_13_0 = def(8_595_00_0); public static final TransportVersion V_8_13_4 = def(8_595_00_1); public static final TransportVersion V_8_14_0 = def(8_636_00_1); - // 8.15.0+ - public static final TransportVersion WATERMARK_THRESHOLDS_STATS = def(8_637_00_0); - public static final TransportVersion ENRICH_CACHE_ADDITIONAL_STATS = def(8_638_00_0); - public static final TransportVersion ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED = def(8_639_00_0); - public static final TransportVersion ML_TRAINED_MODEL_CACHE_METADATA_ADDED = def(8_640_00_0); - public static final TransportVersion TOP_LEVEL_KNN_SUPPORT_QUERY_NAME = def(8_641_00_0); - public static final TransportVersion INDEX_SEGMENTS_VECTOR_FORMATS = def(8_642_00_0); - public static final TransportVersion ADD_RESOURCE_ALREADY_UPLOADED_EXCEPTION = def(8_643_00_0); - public static final TransportVersion ESQL_MV_ORDERING_SORTED_ASCENDING = def(8_644_00_0); - public static final TransportVersion ESQL_PAGE_MAPPING_TO_ITERATOR = def(8_645_00_0); - public static final TransportVersion BINARY_PIT_ID = def(8_646_00_0); - public static final TransportVersion SECURITY_ROLE_MAPPINGS_IN_CLUSTER_STATE = def(8_647_00_0); - public static final TransportVersion ESQL_REQUEST_TABLES = def(8_648_00_0); - public static final TransportVersion ROLE_REMOTE_CLUSTER_PRIVS = def(8_649_00_0); - public static final TransportVersion NO_GLOBAL_RETENTION_FOR_SYSTEM_DATA_STREAMS = def(8_650_00_0); - public static final TransportVersion SHUTDOWN_REQUEST_TIMEOUTS_FIX = def(8_651_00_0); - public static final TransportVersion INDEXING_PRESSURE_REQUEST_REJECTIONS_COUNT = def(8_652_00_0); - public static final TransportVersion ROLLUP_USAGE = def(8_653_00_0); - public static final TransportVersion SECURITY_ROLE_DESCRIPTION = def(8_654_00_0); - public static final TransportVersion ML_INFERENCE_AZURE_OPENAI_COMPLETIONS = def(8_655_00_0); - public static final TransportVersion JOIN_STATUS_AGE_SERIALIZATION = def(8_656_00_0); - public static final TransportVersion ML_RERANK_DOC_OPTIONAL = def(8_657_00_0); - public static final TransportVersion FAILURE_STORE_FIELD_PARITY = def(8_658_00_0); - public static final TransportVersion ML_INFERENCE_AZURE_AI_STUDIO = def(8_659_00_0); - public static final TransportVersion ML_INFERENCE_COHERE_COMPLETION_ADDED = def(8_660_00_0); - public static final TransportVersion ESQL_REMOVE_ES_SOURCE_OPTIONS = def(8_661_00_0); - public static final TransportVersion NODE_STATS_INGEST_BYTES = def(8_662_00_0); - public static final TransportVersion SEMANTIC_QUERY = def(8_663_00_0); - public static final TransportVersion GET_AUTOSCALING_CAPACITY_UNUSED_TIMEOUT = def(8_664_00_0); - public static final TransportVersion SIMULATE_VALIDATES_MAPPINGS = def(8_665_00_0); - public static final TransportVersion RULE_QUERY_RENAME = def(8_666_00_0); - public static final TransportVersion SPARSE_VECTOR_QUERY_ADDED = def(8_667_00_0); - public static final TransportVersion ESQL_ADD_INDEX_MODE_TO_SOURCE = def(8_668_00_0); - public static final TransportVersion GET_SHUTDOWN_STATUS_TIMEOUT = def(8_669_00_0); - public static final TransportVersion FAILURE_STORE_TELEMETRY = def(8_670_00_0); - public static final TransportVersion ADD_METADATA_FLATTENED_TO_ROLES = def(8_671_00_0); - public static final TransportVersion ML_INFERENCE_GOOGLE_AI_STUDIO_COMPLETION_ADDED = def(8_672_00_0); - public static final TransportVersion WATCHER_REQUEST_TIMEOUTS = def(8_673_00_0); - public static final TransportVersion ML_INFERENCE_ENHANCE_DELETE_ENDPOINT = def(8_674_00_0); - public static final TransportVersion ML_INFERENCE_GOOGLE_AI_STUDIO_EMBEDDINGS_ADDED = def(8_675_00_0); - public static final TransportVersion ADD_MISTRAL_EMBEDDINGS_INFERENCE = def(8_676_00_0); - public static final TransportVersion ML_CHUNK_INFERENCE_OPTION = def(8_677_00_0); - public static final TransportVersion RANK_FEATURE_PHASE_ADDED = def(8_678_00_0); - public static final TransportVersion RANK_DOC_IN_SHARD_FETCH_REQUEST = def(8_679_00_0); - public static final TransportVersion SECURITY_SETTINGS_REQUEST_TIMEOUTS = def(8_680_00_0); - public static final TransportVersion QUERY_RULE_CRUD_API_PUT = def(8_681_00_0); - public static final TransportVersion DROP_UNUSED_NODES_REQUESTS = def(8_682_00_0); - public static final TransportVersion QUERY_RULE_CRUD_API_GET_DELETE = def(8_683_00_0); - public static final TransportVersion MORE_LIGHTER_NODES_REQUESTS = def(8_684_00_0); - public static final TransportVersion DROP_UNUSED_NODES_IDS = def(8_685_00_0); - public static final TransportVersion DELETE_SNAPSHOTS_ASYNC_ADDED = def(8_686_00_0); - public static final TransportVersion VERSION_SUPPORTING_SPARSE_VECTOR_STATS = def(8_687_00_0); - public static final TransportVersion ML_AD_OUTPUT_MEMORY_ALLOCATOR_FIELD = def(8_688_00_0); - public static final TransportVersion FAILURE_STORE_LAZY_CREATION = def(8_689_00_0); - public static final TransportVersion SNAPSHOT_REQUEST_TIMEOUTS = def(8_690_00_0); - public static final TransportVersion INDEX_METADATA_MAPPINGS_UPDATED_VERSION = def(8_691_00_0); - public static final TransportVersion ML_INFERENCE_ELAND_SETTINGS_ADDED = def(8_692_00_0); - public static final TransportVersion ML_ANTHROPIC_INTEGRATION_ADDED = def(8_693_00_0); - public static final TransportVersion ML_INFERENCE_GOOGLE_VERTEX_AI_EMBEDDINGS_ADDED = def(8_694_00_0); - public static final TransportVersion EVENT_INGESTED_RANGE_IN_CLUSTER_STATE = def(8_695_00_0); - public static final TransportVersion ESQL_ADD_AGGREGATE_TYPE = def(8_696_00_0); - public static final TransportVersion SECURITY_MIGRATIONS_MIGRATION_NEEDED_ADDED = def(8_697_00_0); - public static final TransportVersion K_FOR_KNN_QUERY_ADDED = def(8_698_00_0); - public static final TransportVersion TEXT_SIMILARITY_RERANKER_RETRIEVER = def(8_699_00_0); - public static final TransportVersion ML_INFERENCE_GOOGLE_VERTEX_AI_RERANKING_ADDED = def(8_700_00_0); - public static final TransportVersion VERSIONED_MASTER_NODE_REQUESTS = def(8_701_00_0); - public static final TransportVersion ML_INFERENCE_AMAZON_BEDROCK_ADDED = def(8_702_00_0); - public static final TransportVersion ENTERPRISE_GEOIP_DOWNLOADER_BACKPORT_8_15 = def(8_702_00_1); - public static final TransportVersion FIX_VECTOR_SIMILARITY_INNER_HITS_BACKPORT_8_15 = def(8_702_00_2); - public static final TransportVersion ML_INFERENCE_DONT_DELETE_WHEN_SEMANTIC_TEXT_EXISTS = def(8_703_00_0); - public static final TransportVersion INFERENCE_ADAPTIVE_ALLOCATIONS = def(8_704_00_0); - public static final TransportVersion INDEX_REQUEST_UPDATE_BY_SCRIPT_ORIGIN = def(8_705_00_0); - public static final TransportVersion ML_INFERENCE_COHERE_UNUSED_RERANK_SETTINGS_REMOVED = def(8_706_00_0); - public static final TransportVersion ENRICH_CACHE_STATS_SIZE_ADDED = def(8_707_00_0); - public static final TransportVersion ENTERPRISE_GEOIP_DOWNLOADER = def(8_708_00_0); - public static final TransportVersion NODES_STATS_ENUM_SET = def(8_709_00_0); - public static final TransportVersion MASTER_NODE_METRICS = def(8_710_00_0); - public static final TransportVersion SEGMENT_LEVEL_FIELDS_STATS = def(8_711_00_0); - public static final TransportVersion ML_ADD_DETECTION_RULE_PARAMS = def(8_712_00_0); - public static final TransportVersion FIX_VECTOR_SIMILARITY_INNER_HITS = def(8_713_00_0); - public static final TransportVersion INDEX_REQUEST_UPDATE_BY_DOC_ORIGIN = def(8_714_00_0); - public static final TransportVersion ESQL_ATTRIBUTE_CACHED_SERIALIZATION = def(8_715_00_0); - public static final TransportVersion REGISTER_SLM_STATS = def(8_716_00_0); - public static final TransportVersion ESQL_NESTED_UNSUPPORTED = def(8_717_00_0); - public static final TransportVersion ESQL_SINGLE_VALUE_QUERY_SOURCE = def(8_718_00_0); - public static final TransportVersion ESQL_ORIGINAL_INDICES = def(8_719_00_0); - public static final TransportVersion ML_INFERENCE_EIS_INTEGRATION_ADDED = def(8_720_00_0); - public static final TransportVersion INGEST_PIPELINE_EXCEPTION_ADDED = def(8_721_00_0); - public static final TransportVersion ZDT_NANOS_SUPPORT_BROKEN = def(8_722_00_0); - public static final TransportVersion REMOVE_GLOBAL_RETENTION_FROM_TEMPLATES = def(8_723_00_0); - public static final TransportVersion RANDOM_RERANKER_RETRIEVER = def(8_724_00_0); - public static final TransportVersion ESQL_PROFILE_SLEEPS = def(8_725_00_0); - public static final TransportVersion ZDT_NANOS_SUPPORT = def(8_726_00_0); - public static final TransportVersion LTR_SERVERLESS_RELEASE = def(8_727_00_0); - public static final TransportVersion ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT = def(8_728_00_0); - public static final TransportVersion RANK_DOCS_RETRIEVER = def(8_729_00_0); - public static final TransportVersion ESQL_ES_FIELD_CACHED_SERIALIZATION = def(8_730_00_0); - public static final TransportVersion ADD_MANAGE_ROLES_PRIVILEGE = def(8_731_00_0); - public static final TransportVersion REPOSITORIES_TELEMETRY = def(8_732_00_0); - public static final TransportVersion ML_INFERENCE_ALIBABACLOUD_SEARCH_ADDED = def(8_733_00_0); - public static final TransportVersion FIELD_CAPS_RESPONSE_INDEX_MODE = def(8_734_00_0); - public static final TransportVersion GET_DATA_STREAMS_VERBOSE = def(8_735_00_0); - public static final TransportVersion ESQL_ADD_INDEX_MODE_CONCRETE_INDICES = def(8_736_00_0); - public static final TransportVersion UNASSIGNED_PRIMARY_COUNT_ON_CLUSTER_HEALTH = def(8_737_00_0); - public static final TransportVersion ESQL_AGGREGATE_EXEC_TRACKS_INTERMEDIATE_ATTRS = def(8_738_00_0); - public static final TransportVersion CCS_TELEMETRY_STATS = def(8_739_00_0); - public static final TransportVersion GLOBAL_RETENTION_TELEMETRY = def(8_740_00_0); + public static final TransportVersion V_8_15_0 = def(8_702_00_2); + public static final TransportVersion V_8_15_2 = def(8_702_00_3); + public static final TransportVersion V_8_16_0 = def(8_772_00_1); + public static final TransportVersion ADD_COMPATIBILITY_VERSIONS_TO_NODE_INFO_BACKPORT_8_16 = def(8_772_00_2); + public static final TransportVersion SKIP_INNER_HITS_SEARCH_SOURCE_BACKPORT_8_16 = def(8_772_00_3); + public static final TransportVersion QUERY_RULES_LIST_INCLUDES_TYPES_BACKPORT_8_16 = def(8_772_00_4); + public static final TransportVersion REMOVE_MIN_COMPATIBLE_SHARD_NODE = def(8_773_00_0); + public static final TransportVersion REVERT_REMOVE_MIN_COMPATIBLE_SHARD_NODE = def(8_774_00_0); + public static final TransportVersion ESQL_FIELD_ATTRIBUTE_PARENT_SIMPLIFIED = def(8_775_00_0); + public static final TransportVersion INFERENCE_DONT_PERSIST_ON_READ = def(8_776_00_0); + public static final TransportVersion SIMULATE_MAPPING_ADDITION = def(8_777_00_0); + public static final TransportVersion INTRODUCE_ALL_APPLICABLE_SELECTOR = def(8_778_00_0); + public static final TransportVersion INDEX_MODE_LOOKUP = def(8_779_00_0); + public static final TransportVersion INDEX_REQUEST_REMOVE_METERING = def(8_780_00_0); + public static final TransportVersion CPU_STAT_STRING_PARSING = def(8_781_00_0); + public static final TransportVersion QUERY_RULES_RETRIEVER = def(8_782_00_0); + public static final TransportVersion ESQL_CCS_EXEC_INFO_WITH_FAILURES = def(8_783_00_0); + public static final TransportVersion LOGSDB_TELEMETRY = def(8_784_00_0); + public static final TransportVersion LOGSDB_TELEMETRY_STATS = def(8_785_00_0); + public static final TransportVersion KQL_QUERY_ADDED = def(8_786_00_0); + public static final TransportVersion ROLE_MONITOR_STATS = def(8_787_00_0); + public static final TransportVersion DATA_STREAM_INDEX_VERSION_DEPRECATION_CHECK = def(8_788_00_0); + public static final TransportVersion ADD_COMPATIBILITY_VERSIONS_TO_NODE_INFO = def(8_789_00_0); + public static final TransportVersion VERTEX_AI_INPUT_TYPE_ADDED = def(8_790_00_0); + public static final TransportVersion SKIP_INNER_HITS_SEARCH_SOURCE = def(8_791_00_0); + public static final TransportVersion QUERY_RULES_LIST_INCLUDES_TYPES = def(8_792_00_0); + public static final TransportVersion INDEX_STATS_ADDITIONAL_FIELDS = def(8_793_00_0); + public static final TransportVersion INDEX_STATS_ADDITIONAL_FIELDS_REVERT = def(8_794_00_0); + public static final TransportVersion FAST_REFRESH_RCO_2 = def(8_795_00_0); + public static final TransportVersion ESQL_ENRICH_RUNTIME_WARNINGS = def(8_796_00_0); + public static final TransportVersion INGEST_PIPELINE_CONFIGURATION_AS_MAP = def(8_797_00_0); + public static final TransportVersion LOGSDB_TELEMETRY_CUSTOM_CUTOFF_DATE_FIX_8_17 = def(8_797_00_1); + public static final TransportVersion SOURCE_MODE_TELEMETRY_FIX_8_17 = def(8_797_00_2); + public static final TransportVersion INDEXING_PRESSURE_THROTTLING_STATS = def(8_798_00_0); + public static final TransportVersion REINDEX_DATA_STREAMS = def(8_799_00_0); + public static final TransportVersion ESQL_REMOVE_NODE_LEVEL_PLAN = def(8_800_00_0); + public static final TransportVersion LOGSDB_TELEMETRY_CUSTOM_CUTOFF_DATE = def(8_801_00_0); + public static final TransportVersion SOURCE_MODE_TELEMETRY = def(8_802_00_0); + public static final TransportVersion NEW_REFRESH_CLUSTER_BLOCK = def(8_803_00_0); + public static final TransportVersion RETRIES_AND_OPERATIONS_IN_BLOBSTORE_STATS = def(8_804_00_0); + public static final TransportVersion ADD_DATA_STREAM_OPTIONS_TO_TEMPLATES = def(8_805_00_0); + public static final TransportVersion KNN_QUERY_RESCORE_OVERSAMPLE = def(8_806_00_0); + public static final TransportVersion SEMANTIC_QUERY_LENIENT = def(8_807_00_0); + public static final TransportVersion ESQL_QUERY_BUILDER_IN_SEARCH_FUNCTIONS = def(8_808_00_0); + public static final TransportVersion EQL_ALLOW_PARTIAL_SEARCH_RESULTS = def(8_809_00_0); + public static final TransportVersion NODE_VERSION_INFORMATION_WITH_MIN_READ_ONLY_INDEX_VERSION = def(8_810_00_0); + public static final TransportVersion ERROR_TRACE_IN_TRANSPORT_HEADER = def(8_811_00_0); + public static final TransportVersion FAILURE_STORE_ENABLED_BY_CLUSTER_SETTING = def(8_812_00_0); + public static final TransportVersion SIMULATE_IGNORED_FIELDS = def(8_813_00_0); + public static final TransportVersion TRANSFORMS_UPGRADE_MODE = def(8_814_00_0); + public static final TransportVersion NODE_SHUTDOWN_EPHEMERAL_ID_ADDED = def(8_815_00_0); + public static final TransportVersion ESQL_CCS_TELEMETRY_STATS = def(8_816_00_0); + public static final TransportVersion TEXT_EMBEDDING_QUERY_VECTOR_BUILDER_INFER_MODEL_ID = def(8_817_00_0); + public static final TransportVersion ESQL_ENABLE_NODE_LEVEL_REDUCTION = def(8_818_00_0); /* * STOP! READ THIS FIRST! No, really, @@ -272,7 +220,7 @@ static TransportVersion def(int id) { * Reference to the minimum transport version that can be used with CCS. * This should be the transport version used by the previous minor release. */ - public static final TransportVersion MINIMUM_CCS_VERSION = FIX_VECTOR_SIMILARITY_INNER_HITS_BACKPORT_8_15; + public static final TransportVersion MINIMUM_CCS_VERSION = QUERY_RULES_LIST_INCLUDES_TYPES_BACKPORT_8_16; static final NavigableMap VERSION_IDS = getAllVersionIds(TransportVersions.class); diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index 0164c6b80fa6b..1a20f714d3a33 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch; @@ -125,6 +126,8 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_7_17_23 = new Version(7_17_23_99); public static final Version V_7_17_24 = new Version(7_17_24_99); public static final Version V_7_17_25 = new Version(7_17_25_99); + public static final Version V_7_17_26 = new Version(7_17_26_99); + public static final Version V_7_17_27 = new Version(7_17_27_99); public static final Version V_8_0_0 = new Version(8_00_00_99); public static final Version V_8_0_1 = new Version(8_00_01_99); @@ -184,8 +187,17 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_8_15_0 = new Version(8_15_00_99); public static final Version V_8_15_1 = new Version(8_15_01_99); public static final Version V_8_15_2 = new Version(8_15_02_99); + public static final Version V_8_15_3 = new Version(8_15_03_99); + public static final Version V_8_15_4 = new Version(8_15_04_99); + public static final Version V_8_15_5 = new Version(8_15_05_99); public static final Version V_8_16_0 = new Version(8_16_00_99); - public static final Version CURRENT = V_8_16_0; + public static final Version V_8_16_1 = new Version(8_16_01_99); + public static final Version V_8_16_2 = new Version(8_16_02_99); + public static final Version V_8_16_3 = new Version(8_16_03_99); + public static final Version V_8_17_0 = new Version(8_17_00_99); + public static final Version V_8_17_1 = new Version(8_17_01_99); + public static final Version V_8_18_0 = new Version(8_18_00_99); + public static final Version CURRENT = V_8_18_0; private static final NavigableMap VERSION_IDS; private static final Map VERSION_STRINGS; diff --git a/server/src/main/java/org/elasticsearch/action/ActionFuture.java b/server/src/main/java/org/elasticsearch/action/ActionFuture.java index 061875e42fec8..806c97351f6b9 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionFuture.java +++ b/server/src/main/java/org/elasticsearch/action/ActionFuture.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/ActionListener.java b/server/src/main/java/org/elasticsearch/action/ActionListener.java index 5841648700756..a158669d936fe 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionListener.java +++ b/server/src/main/java/org/elasticsearch/action/ActionListener.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; @@ -474,4 +475,12 @@ static void runWithResource( ActionListener.run(ActionListener.runBefore(listener, resource::close), l -> action.accept(l, resource)); } + /** + * Increments ref count and returns a listener that will decrement ref count on listener completion. + */ + static ActionListener withRef(ActionListener listener, RefCounted ref) { + ref.mustIncRef(); + return releaseAfter(listener, ref::decRef); + } + } diff --git a/server/src/main/java/org/elasticsearch/action/ActionListenerImplementations.java b/server/src/main/java/org/elasticsearch/action/ActionListenerImplementations.java index 93f6ac6e49d41..00cbe475f162e 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionListenerImplementations.java +++ b/server/src/main/java/org/elasticsearch/action/ActionListenerImplementations.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/ActionListenerResponseHandler.java b/server/src/main/java/org/elasticsearch/action/ActionListenerResponseHandler.java index f511d5a333062..68ecc3b213550 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionListenerResponseHandler.java +++ b/server/src/main/java/org/elasticsearch/action/ActionListenerResponseHandler.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java index 37a33eab4e4e8..eb0a005dc83ee 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; @@ -159,6 +160,7 @@ import org.elasticsearch.action.admin.indices.template.put.TransportPutIndexTemplateAction; import org.elasticsearch.action.admin.indices.validate.query.TransportValidateQueryAction; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryAction; +import org.elasticsearch.action.bulk.IncrementalBulkService; import org.elasticsearch.action.bulk.SimulateBulkAction; import org.elasticsearch.action.bulk.TransportBulkAction; import org.elasticsearch.action.bulk.TransportShardBulkAction; @@ -241,6 +243,7 @@ import org.elasticsearch.injection.guice.AbstractModule; import org.elasticsearch.injection.guice.TypeLiteral; import org.elasticsearch.injection.guice.multibindings.MapBinder; +import org.elasticsearch.monitor.metrics.IndexModeStatsActionType; import org.elasticsearch.persistent.CompletionPersistentTaskAction; import org.elasticsearch.persistent.RemovePersistentTaskAction; import org.elasticsearch.persistent.StartPersistentTaskAction; @@ -447,6 +450,7 @@ public class ActionModule extends AbstractModule { private final List actionPlugins; private final Map> actions; private final ActionFilters actionFilters; + private final IncrementalBulkService bulkService; private final AutoCreateIndex autoCreateIndex; private final DestructiveOperations destructiveOperations; private final RestController restController; @@ -475,7 +479,8 @@ public ActionModule( ClusterService clusterService, RerouteService rerouteService, List> reservedStateHandlers, - RestExtension restExtension + RestExtension restExtension, + IncrementalBulkService bulkService ) { this.settings = settings; this.indexNameExpressionResolver = indexNameExpressionResolver; @@ -487,6 +492,7 @@ public ActionModule( this.threadPool = threadPool; actions = setupActions(actionPlugins); actionFilters = setupActionFilters(actionPlugins); + this.bulkService = bulkService; autoCreateIndex = new AutoCreateIndex(settings, clusterSettings, indexNameExpressionResolver, systemIndices); destructiveOperations = new DestructiveOperations(settings, clusterSettings); Set headers = Stream.concat( @@ -625,6 +631,7 @@ public void reg actions.register(TransportNodesFeaturesAction.TYPE, TransportNodesFeaturesAction.class); actions.register(RemoteClusterNodesAction.TYPE, RemoteClusterNodesAction.TransportAction.class); actions.register(TransportNodesStatsAction.TYPE, TransportNodesStatsAction.class); + actions.register(IndexModeStatsActionType.TYPE, IndexModeStatsActionType.TransportAction.class); actions.register(TransportNodesUsageAction.TYPE, TransportNodesUsageAction.class); actions.register(TransportNodesHotThreadsAction.TYPE, TransportNodesHotThreadsAction.class); actions.register(TransportListTasksAction.TYPE, TransportListTasksAction.class); @@ -927,7 +934,7 @@ public void initRestHandlers(Supplier nodesInCluster, Predicate< registerHandler.accept(new RestCountAction()); registerHandler.accept(new RestTermVectorsAction()); registerHandler.accept(new RestMultiTermVectorsAction()); - registerHandler.accept(new RestBulkAction(settings)); + registerHandler.accept(new RestBulkAction(settings, bulkService)); registerHandler.accept(new RestUpdateAction()); registerHandler.accept(new RestSearchAction(restController.getSearchUsageHolder(), clusterSupportsFeature)); diff --git a/server/src/main/java/org/elasticsearch/action/ActionRequest.java b/server/src/main/java/org/elasticsearch/action/ActionRequest.java index 02b3001343e7f..cc2963f49b07e 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionRequest.java +++ b/server/src/main/java/org/elasticsearch/action/ActionRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/ActionRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/ActionRequestBuilder.java index 32c8bcf155522..162c74de8e35f 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/ActionRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/ActionRequestLazyBuilder.java b/server/src/main/java/org/elasticsearch/action/ActionRequestLazyBuilder.java index 7779b71c46717..be16dd35c406d 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionRequestLazyBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/ActionRequestLazyBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/ActionRequestValidationException.java b/server/src/main/java/org/elasticsearch/action/ActionRequestValidationException.java index 646b1536889d5..3ccf1447c19ac 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionRequestValidationException.java +++ b/server/src/main/java/org/elasticsearch/action/ActionRequestValidationException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/ActionResponse.java b/server/src/main/java/org/elasticsearch/action/ActionResponse.java index 504834d7161cb..2c1002505d47b 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionResponse.java +++ b/server/src/main/java/org/elasticsearch/action/ActionResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/ActionRunnable.java b/server/src/main/java/org/elasticsearch/action/ActionRunnable.java index b1c4482e703eb..492102c153e45 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionRunnable.java +++ b/server/src/main/java/org/elasticsearch/action/ActionRunnable.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/ActionType.java b/server/src/main/java/org/elasticsearch/action/ActionType.java index 51927baf7b4db..be359088f7216 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionType.java +++ b/server/src/main/java/org/elasticsearch/action/ActionType.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/AliasesRequest.java b/server/src/main/java/org/elasticsearch/action/AliasesRequest.java index 5c53980452e66..8083c924e935b 100644 --- a/server/src/main/java/org/elasticsearch/action/AliasesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/AliasesRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/ClusterStatsLevel.java b/server/src/main/java/org/elasticsearch/action/ClusterStatsLevel.java index 744504c084cea..c6d298340fe95 100644 --- a/server/src/main/java/org/elasticsearch/action/ClusterStatsLevel.java +++ b/server/src/main/java/org/elasticsearch/action/ClusterStatsLevel.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/CompositeIndicesRequest.java b/server/src/main/java/org/elasticsearch/action/CompositeIndicesRequest.java index c8616d18c0240..102cd48177b19 100644 --- a/server/src/main/java/org/elasticsearch/action/CompositeIndicesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/CompositeIndicesRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/DelegatingActionListener.java b/server/src/main/java/org/elasticsearch/action/DelegatingActionListener.java index 6cc3c4cb92fae..688bfd2617514 100644 --- a/server/src/main/java/org/elasticsearch/action/DelegatingActionListener.java +++ b/server/src/main/java/org/elasticsearch/action/DelegatingActionListener.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/DocWriteRequest.java b/server/src/main/java/org/elasticsearch/action/DocWriteRequest.java index bfe1ff04b7b77..292f962869558 100644 --- a/server/src/main/java/org/elasticsearch/action/DocWriteRequest.java +++ b/server/src/main/java/org/elasticsearch/action/DocWriteRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; @@ -159,9 +160,14 @@ public interface DocWriteRequest extends IndicesRequest, Accountable { boolean isRequireDataStream(); /** - * Finalize the request before executing or routing it. + * Finalize the request before routing it. */ - void process(IndexRouting indexRouting); + default void preRoutingProcess(IndexRouting indexRouting) {} + + /** + * Finalize the request after routing it. + */ + default void postRoutingProcess(IndexRouting indexRouting) {} /** * Pick the appropriate shard id to receive this request. diff --git a/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java b/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java index 685fc032431c3..095ccd71fa266 100644 --- a/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java +++ b/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java @@ -1,13 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; import org.elasticsearch.TransportVersions; +import org.elasticsearch.action.bulk.IndexDocFailureStoreStatus; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.action.support.WriteResponse; @@ -248,6 +250,10 @@ public String getLocation(@Nullable String routing) { return location.toString(); } + public IndexDocFailureStoreStatus getFailureStoreStatus() { + return IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN; + } + public void writeThin(StreamOutput out) throws IOException { super.writeTo(out); writeWithoutShardId(out); diff --git a/server/src/main/java/org/elasticsearch/action/FailedNodeException.java b/server/src/main/java/org/elasticsearch/action/FailedNodeException.java index dfcac2b9c6f38..45795098c86e6 100644 --- a/server/src/main/java/org/elasticsearch/action/FailedNodeException.java +++ b/server/src/main/java/org/elasticsearch/action/FailedNodeException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/IndicesRequest.java b/server/src/main/java/org/elasticsearch/action/IndicesRequest.java index 20913148fd9b3..176ecbc7a2395 100644 --- a/server/src/main/java/org/elasticsearch/action/IndicesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/IndicesRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/LatchedActionListener.java b/server/src/main/java/org/elasticsearch/action/LatchedActionListener.java index edec9bb7b6a1b..80eb88a91053c 100644 --- a/server/src/main/java/org/elasticsearch/action/LatchedActionListener.java +++ b/server/src/main/java/org/elasticsearch/action/LatchedActionListener.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/NoShardAvailableActionException.java b/server/src/main/java/org/elasticsearch/action/NoShardAvailableActionException.java index 564055aa36750..1049ab30b802d 100644 --- a/server/src/main/java/org/elasticsearch/action/NoShardAvailableActionException.java +++ b/server/src/main/java/org/elasticsearch/action/NoShardAvailableActionException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/NoSuchNodeException.java b/server/src/main/java/org/elasticsearch/action/NoSuchNodeException.java index 72f6774aaee69..a3f21c86c27cb 100644 --- a/server/src/main/java/org/elasticsearch/action/NoSuchNodeException.java +++ b/server/src/main/java/org/elasticsearch/action/NoSuchNodeException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/NodeStatsLevel.java b/server/src/main/java/org/elasticsearch/action/NodeStatsLevel.java index 38cfff79ee514..3992dda9c0c19 100644 --- a/server/src/main/java/org/elasticsearch/action/NodeStatsLevel.java +++ b/server/src/main/java/org/elasticsearch/action/NodeStatsLevel.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/OriginalIndices.java b/server/src/main/java/org/elasticsearch/action/OriginalIndices.java index f3df0f9ed579a..e6d11bb6ffca7 100644 --- a/server/src/main/java/org/elasticsearch/action/OriginalIndices.java +++ b/server/src/main/java/org/elasticsearch/action/OriginalIndices.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/PrimaryMissingActionException.java b/server/src/main/java/org/elasticsearch/action/PrimaryMissingActionException.java index 59912e6044d90..61125643d9087 100644 --- a/server/src/main/java/org/elasticsearch/action/PrimaryMissingActionException.java +++ b/server/src/main/java/org/elasticsearch/action/PrimaryMissingActionException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/RealtimeRequest.java b/server/src/main/java/org/elasticsearch/action/RealtimeRequest.java index 2391158ab879d..d4f8bc0f9eb20 100644 --- a/server/src/main/java/org/elasticsearch/action/RealtimeRequest.java +++ b/server/src/main/java/org/elasticsearch/action/RealtimeRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/RemoteClusterActionType.java b/server/src/main/java/org/elasticsearch/action/RemoteClusterActionType.java index 6e993fdbad8e1..daf3a424a00cd 100644 --- a/server/src/main/java/org/elasticsearch/action/RemoteClusterActionType.java +++ b/server/src/main/java/org/elasticsearch/action/RemoteClusterActionType.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/RequestBuilder.java b/server/src/main/java/org/elasticsearch/action/RequestBuilder.java index 5db55d6569876..4be8735c9605e 100644 --- a/server/src/main/java/org/elasticsearch/action/RequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/RequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/RequestValidators.java b/server/src/main/java/org/elasticsearch/action/RequestValidators.java index c9fa3b936ecf5..5f661003137e0 100644 --- a/server/src/main/java/org/elasticsearch/action/RequestValidators.java +++ b/server/src/main/java/org/elasticsearch/action/RequestValidators.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/ResolvedIndices.java b/server/src/main/java/org/elasticsearch/action/ResolvedIndices.java index 822a75c4dec42..16f37c9573a8e 100644 --- a/server/src/main/java/org/elasticsearch/action/ResolvedIndices.java +++ b/server/src/main/java/org/elasticsearch/action/ResolvedIndices.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; @@ -149,10 +150,26 @@ public static ResolvedIndices resolveWithIndicesRequest( RemoteClusterService remoteClusterService, long startTimeInMillis ) { - final Map remoteClusterIndices = remoteClusterService.groupIndices( + return resolveWithIndexNamesAndOptions( + request.indices(), request.indicesOptions(), - request.indices() + clusterState, + indexNameExpressionResolver, + remoteClusterService, + startTimeInMillis ); + } + + public static ResolvedIndices resolveWithIndexNamesAndOptions( + String[] indexNames, + IndicesOptions indicesOptions, + ClusterState clusterState, + IndexNameExpressionResolver indexNameExpressionResolver, + RemoteClusterService remoteClusterService, + long startTimeInMillis + ) { + final Map remoteClusterIndices = remoteClusterService.groupIndices(indicesOptions, indexNames); + final OriginalIndices localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); Index[] concreteLocalIndices = localIndices == null diff --git a/server/src/main/java/org/elasticsearch/action/ResultDeduplicator.java b/server/src/main/java/org/elasticsearch/action/ResultDeduplicator.java index 2a9887d8a30d7..483c0a028cd23 100644 --- a/server/src/main/java/org/elasticsearch/action/ResultDeduplicator.java +++ b/server/src/main/java/org/elasticsearch/action/ResultDeduplicator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/RoutingMissingException.java b/server/src/main/java/org/elasticsearch/action/RoutingMissingException.java index 0999e7154b058..5804598febd10 100644 --- a/server/src/main/java/org/elasticsearch/action/RoutingMissingException.java +++ b/server/src/main/java/org/elasticsearch/action/RoutingMissingException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/ShardOperationFailedException.java b/server/src/main/java/org/elasticsearch/action/ShardOperationFailedException.java index fbe3510f26aa0..fa8be8c38934d 100644 --- a/server/src/main/java/org/elasticsearch/action/ShardOperationFailedException.java +++ b/server/src/main/java/org/elasticsearch/action/ShardOperationFailedException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/SingleResultDeduplicator.java b/server/src/main/java/org/elasticsearch/action/SingleResultDeduplicator.java index aa07a11ba124e..d6280ef5bef79 100644 --- a/server/src/main/java/org/elasticsearch/action/SingleResultDeduplicator.java +++ b/server/src/main/java/org/elasticsearch/action/SingleResultDeduplicator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/TaskOperationFailure.java b/server/src/main/java/org/elasticsearch/action/TaskOperationFailure.java index 505e769144add..fe0b02336966f 100644 --- a/server/src/main/java/org/elasticsearch/action/TaskOperationFailure.java +++ b/server/src/main/java/org/elasticsearch/action/TaskOperationFailure.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/TimestampParsingException.java b/server/src/main/java/org/elasticsearch/action/TimestampParsingException.java index 9d239b5c6127b..2c4fe1d448db0 100644 --- a/server/src/main/java/org/elasticsearch/action/TimestampParsingException.java +++ b/server/src/main/java/org/elasticsearch/action/TimestampParsingException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/UnavailableShardsException.java b/server/src/main/java/org/elasticsearch/action/UnavailableShardsException.java index 647e98e3599f5..ac9ca2d83be25 100644 --- a/server/src/main/java/org/elasticsearch/action/UnavailableShardsException.java +++ b/server/src/main/java/org/elasticsearch/action/UnavailableShardsException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/ValidateActions.java b/server/src/main/java/org/elasticsearch/action/ValidateActions.java index 9f64df492a741..b57b44bc19a23 100644 --- a/server/src/main/java/org/elasticsearch/action/ValidateActions.java +++ b/server/src/main/java/org/elasticsearch/action/ValidateActions.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/AllocationStatsFeatures.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/AllocationStatsFeatures.java index c74a23c6b17ea..d754d652c8d88 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/AllocationStatsFeatures.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/AllocationStatsFeatures.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.allocation; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java index 11731c0ccade5..445bac4cd659e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.allocation; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainResponse.java index 92413fc104be4..f2e2b9bc02708 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.allocation; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java index abb4f478cff54..23f88540c2666 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.allocation; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceRequest.java index d01c05b63ab67..fba65ee0db9b5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.allocation; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceResponse.java index 54187ababccdf..2f22b63f78bc6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.allocation; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java index c8d6a25c3ba6e..186c9f4b8654e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.allocation; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceAction.java index f296dbb9bb5cb..6b490f52df464 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.allocation; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java index 2f094b0fc6006..d929fb457d5d1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.allocation; @@ -117,7 +118,7 @@ public Request(TimeValue masterNodeTimeout, TaskId parentTaskId, EnumSet public Request(StreamInput in) throws IOException { super(in); - this.metrics = in.getTransportVersion().onOrAfter(TransportVersions.MASTER_NODE_METRICS) + this.metrics = in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readEnumSet(Metric.class) : EnumSet.of(Metric.ALLOCATIONS, Metric.FS); } @@ -126,7 +127,7 @@ public Request(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { assert out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0); super.writeTo(out); - if (out.getTransportVersion().onOrAfter(TransportVersions.MASTER_NODE_METRICS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeEnumSet(metrics); } } @@ -155,7 +156,7 @@ public Response(Map nodeAllocationStats, DiskThresh public Response(StreamInput in) throws IOException { super(in); this.nodeAllocationStats = in.readImmutableMap(StreamInput::readString, NodeAllocationStats::new); - if (in.getTransportVersion().onOrAfter(TransportVersions.WATERMARK_THRESHOLDS_STATS)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { this.diskThresholdSettings = in.readOptionalWriteable(DiskThresholdSettings::readFrom); } else { this.diskThresholdSettings = null; @@ -165,7 +166,7 @@ public Response(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { out.writeMap(nodeAllocationStats, StreamOutput::writeString, StreamOutput::writeWriteable); - if (out.getTransportVersion().onOrAfter(TransportVersions.WATERMARK_THRESHOLDS_STATS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { out.writeOptionalWriteable(diskThresholdSettings); } else { assert diskThresholdSettings == null; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceAction.java index 8337353e1f40d..3731ca7de28ee 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.allocation; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java index 139ef58e4292e..ed25f6d531f52 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.configuration; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/ClearVotingConfigExclusionsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/ClearVotingConfigExclusionsRequest.java index f63c383d67cac..1a0d7af33dc29 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/ClearVotingConfigExclusionsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/ClearVotingConfigExclusionsRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.configuration; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java index 12db5e664a784..3528b7af38a51 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.configuration; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java index 29aa93cd7d513..a688b8105009f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.configuration; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/coordination/ClusterFormationInfoAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/coordination/ClusterFormationInfoAction.java index ff73bc0b9c56a..848dc8639c699 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/coordination/ClusterFormationInfoAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/coordination/ClusterFormationInfoAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/coordination/CoordinationDiagnosticsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/coordination/CoordinationDiagnosticsAction.java index bd41f5933e058..a2ac23ecec4b3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/coordination/CoordinationDiagnosticsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/coordination/CoordinationDiagnosticsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/coordination/MasterHistoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/coordination/MasterHistoryAction.java index 82a2ede90c945..40c634968581d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/coordination/MasterHistoryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/coordination/MasterHistoryAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/GetDesiredNodesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/GetDesiredNodesAction.java index 4af2c64c67ee5..d3780f47f4ae7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/GetDesiredNodesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/GetDesiredNodesAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.desirednodes; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportDeleteDesiredNodesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportDeleteDesiredNodesAction.java index f71def58820a0..dbd0250844fc3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportDeleteDesiredNodesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportDeleteDesiredNodesAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.desirednodes; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportGetDesiredNodesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportGetDesiredNodesAction.java index 6909ce1f9366b..d8460c3aabbc5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportGetDesiredNodesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportGetDesiredNodesAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.desirednodes; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesAction.java index 9ec8feeb5d405..bed2815f5a895 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.desirednodes; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesAction.java index 37f7baa3a30d9..99d42e8931861 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.desirednodes; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesRequest.java index 550db4892a673..a94401fdd66f3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.desirednodes; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesResponse.java index 1b9b26e978838..8e12f079fbfc7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.desirednodes; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java index 2344c8f99ceda..ecedbced245a1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.health; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestBuilder.java index 2b94b3f674a2c..9b280a957c546 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.health; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java index 8067f92a5a908..18cf15e60e1cb 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.health; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java index 522bbb4eee629..fc541fe7108c6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.health; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusAction.java index 4a57620d61e81..23b094fc72b6f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.migration; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusRequest.java index dfb229c4d5336..959197f1e12b6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.migration; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponse.java index ffbd6b70e9ed9..f65152a8b72e9 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.migration; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeAction.java index ffb90001fb42e..d5f881cef9e5c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.migration; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeRequest.java index 84321de66a339..5681043148821 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.migration; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeResponse.java index ca3771654de81..56c0d2eaad900 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.migration; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java index ae538e7c72334..e99ad020b15cc 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.migration; @@ -18,7 +19,6 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.indices.SystemIndices; @@ -52,18 +52,13 @@ public class TransportGetFeatureUpgradeStatusAction extends TransportMasterNodeA GetFeatureUpgradeStatusRequest, GetFeatureUpgradeStatusResponse> { - /** - * Once all feature migrations for 8.x -> 9.x have been tested, we can bump this to Version.V_8_0_0 - */ - @UpdateForV9 - public static final Version NO_UPGRADE_REQUIRED_VERSION = Version.V_7_0_0; - public static final IndexVersion NO_UPGRADE_REQUIRED_INDEX_VERSION = IndexVersions.V_7_0_0; + public static final Version NO_UPGRADE_REQUIRED_VERSION = Version.V_8_0_0; + public static final IndexVersion NO_UPGRADE_REQUIRED_INDEX_VERSION = IndexVersions.V_8_0_0; private final SystemIndices systemIndices; PersistentTasksService persistentTasksService; @Inject - @UpdateForV9 // Once we begin working on 9.x, we need to update our migration classes public TransportGetFeatureUpgradeStatusAction( TransportService transportService, ThreadPool threadPool, @@ -148,7 +143,6 @@ static GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus getFeatureUpgradeSta .map(idxInfo -> ERROR) .map(idxStatus -> GetFeatureUpgradeStatusResponse.UpgradeStatus.combine(idxStatus, initialStatus)) .orElse(initialStatus); - return new GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus(featureName, minimumVersion, status, indexInfos); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportPostFeatureUpgradeAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportPostFeatureUpgradeAction.java index 485f91f0fd4a2..cc5d2e95fc2ea 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportPostFeatureUpgradeAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportPostFeatureUpgradeAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.migration; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/NodeCapability.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/NodeCapability.java index c26aa673d13fd..7d70e83f6558c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/NodeCapability.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/NodeCapability.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.capabilities; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/NodesCapabilitiesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/NodesCapabilitiesRequest.java index 7adcff9f19ccb..114c6518da9fd 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/NodesCapabilitiesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/NodesCapabilitiesRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.capabilities; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/NodesCapabilitiesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/NodesCapabilitiesResponse.java index 3527b8cc46840..f51d440a6eb33 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/NodesCapabilitiesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/NodesCapabilitiesResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.capabilities; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/TransportNodesCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/TransportNodesCapabilitiesAction.java index 3691c07fe95e2..8df34d882941a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/TransportNodesCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/TransportNodesCapabilitiesAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.capabilities; @@ -37,7 +38,8 @@ public class TransportNodesCapabilitiesAction extends TransportNodesAction< NodesCapabilitiesRequest, NodesCapabilitiesResponse, TransportNodesCapabilitiesAction.NodeCapabilitiesRequest, - NodeCapability> { + NodeCapability, + Void> { public static final ActionType TYPE = new ActionType<>("cluster:monitor/nodes/capabilities"); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/NodeFeatures.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/NodeFeatures.java index b33520624d114..d6bff676505a2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/NodeFeatures.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/NodeFeatures.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.features; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/NodesFeaturesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/NodesFeaturesRequest.java index 83b6fff7cf2b2..7ebd4023919e2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/NodesFeaturesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/NodesFeaturesRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.features; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/NodesFeaturesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/NodesFeaturesResponse.java index 0fca588216b15..d53d70cff97d4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/NodesFeaturesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/NodesFeaturesResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.features; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/TransportNodesFeaturesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/TransportNodesFeaturesAction.java index 723f3f5720ff9..e5e04c8490c8e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/TransportNodesFeaturesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/TransportNodesFeaturesAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.features; @@ -32,7 +33,8 @@ public class TransportNodesFeaturesAction extends TransportNodesAction< NodesFeaturesRequest, NodesFeaturesResponse, TransportNodesFeaturesAction.NodeFeaturesRequest, - NodeFeatures> { + NodeFeatures, + Void> { public static final ActionType TYPE = new ActionType<>("cluster:monitor/nodes/features"); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodeHotThreads.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodeHotThreads.java index ef5d7f5e74ef8..de7ec27bb8821 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodeHotThreads.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodeHotThreads.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.hotthreads; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java index 467d2561f364e..7a5be51e3c06b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.hotthreads; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsResponse.java index 892629dbe46f7..971f36129898b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.hotthreads; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java index 811e2d8152067..25f3a53aff0a0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.hotthreads; @@ -38,7 +39,8 @@ public class TransportNodesHotThreadsAction extends TransportNodesAction< NodesHotThreadsRequest, NodesHotThreadsResponse, TransportNodesHotThreadsAction.NodeRequest, - NodeHotThreads> { + NodeHotThreads, + Void> { public static final ActionType TYPE = new ActionType<>("cluster:monitor/nodes/hot_threads"); @@ -115,14 +117,14 @@ public static class NodeRequest extends TransportRequest { NodeRequest(StreamInput in) throws IOException { super(in); - skipLegacyNodesRequestHeader(TransportVersions.MORE_LIGHTER_NODES_REQUESTS, in); + skipLegacyNodesRequestHeader(TransportVersions.V_8_15_0, in); requestOptions = HotThreads.RequestOptions.readFrom(in); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - sendLegacyNodesRequestHeader(TransportVersions.MORE_LIGHTER_NODES_REQUESTS, out); + sendLegacyNodesRequestHeader(TransportVersions.V_8_15_0, out); requestOptions.writeTo(out); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/ComponentVersionNumber.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/ComponentVersionNumber.java index 3e2572410a172..f06e7dac68112 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/ComponentVersionNumber.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/ComponentVersionNumber.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.info; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java index 97fbd2fa8c008..a7d92682b763c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.info; @@ -14,6 +15,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.support.nodes.BaseNodeResponse; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.version.CompatibilityVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; @@ -21,6 +23,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.http.HttpInfo; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.ingest.IngestInfo; import org.elasticsearch.monitor.jvm.JvmInfo; import org.elasticsearch.monitor.os.OsInfo; @@ -41,7 +44,7 @@ public class NodeInfo extends BaseNodeResponse { private final String version; - private final TransportVersion transportVersion; + private final CompatibilityVersions compatibilityVersions; private final IndexVersion indexVersion; private final Map componentVersions; private final Build build; @@ -63,15 +66,20 @@ public NodeInfo(StreamInput in) throws IOException { super(in); if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) { version = in.readString(); - transportVersion = TransportVersion.readVersion(in); + if (in.getTransportVersion().isPatchFrom(TransportVersions.ADD_COMPATIBILITY_VERSIONS_TO_NODE_INFO_BACKPORT_8_16) + || in.getTransportVersion().onOrAfter(TransportVersions.ADD_COMPATIBILITY_VERSIONS_TO_NODE_INFO)) { + compatibilityVersions = CompatibilityVersions.readVersion(in); + } else { + compatibilityVersions = new CompatibilityVersions(TransportVersion.readVersion(in), Map.of()); // unknown mappings versions + } indexVersion = IndexVersion.readVersion(in); } else { Version legacyVersion = Version.readVersion(in); version = legacyVersion.toString(); if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_8_0)) { - transportVersion = TransportVersion.readVersion(in); + compatibilityVersions = new CompatibilityVersions(TransportVersion.readVersion(in), Map.of()); // unknown mappings versions } else { - transportVersion = TransportVersion.fromId(legacyVersion.id); + compatibilityVersions = new CompatibilityVersions(TransportVersion.fromId(legacyVersion.id), Map.of()); } if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_11_X)) { indexVersion = IndexVersion.readVersion(in); @@ -113,7 +121,7 @@ public NodeInfo(StreamInput in) throws IOException { public NodeInfo( String version, - TransportVersion transportVersion, + CompatibilityVersions compatibilityVersions, IndexVersion indexVersion, Map componentVersions, Build build, @@ -133,7 +141,7 @@ public NodeInfo( ) { super(node); this.version = version; - this.transportVersion = transportVersion; + this.compatibilityVersions = compatibilityVersions; this.indexVersion = indexVersion; this.componentVersions = componentVersions; this.build = build; @@ -170,7 +178,7 @@ public String getVersion() { * The most recent transport version that can be used by this node */ public TransportVersion getTransportVersion() { - return transportVersion; + return compatibilityVersions.transportVersion(); } /** @@ -187,6 +195,13 @@ public Map getComponentVersions() { return componentVersions; } + /** + * A map of system index names to versions for their mappings supported by this node. + */ + public Map getCompatibilityVersions() { + return compatibilityVersions.systemIndexMappingsVersion(); + } + /** * The build version of the node. */ @@ -239,8 +254,11 @@ public void writeTo(StreamOutput out) throws IOException { } else { Version.writeVersion(Version.fromString(version), out); } - if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_8_0)) { - TransportVersion.writeVersion(transportVersion, out); + if (out.getTransportVersion().isPatchFrom(TransportVersions.ADD_COMPATIBILITY_VERSIONS_TO_NODE_INFO_BACKPORT_8_16) + || out.getTransportVersion().onOrAfter(TransportVersions.ADD_COMPATIBILITY_VERSIONS_TO_NODE_INFO)) { + compatibilityVersions.writeTo(out); + } else if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_8_0)) { + TransportVersion.writeVersion(compatibilityVersions.transportVersion(), out); } if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_11_X)) { IndexVersion.writeVersion(indexVersion, out); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoMetrics.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoMetrics.java index 39e210571f37b..8b588c92bed0a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoMetrics.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoMetrics.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.info; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java index cef9e880a8e70..7413ef90ef324 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.info; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java index 4f5860b4ba50d..7f644d3f06e5e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.info; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java index fbf285d40f698..d3d4fc4810946 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.info; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsAndModules.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsAndModules.java index 3691b3b432f3a..99b8dd85425b3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsAndModules.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsAndModules.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.info; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java index 542dea59e7197..65bf763197598 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.info; @@ -33,7 +34,8 @@ public class TransportNodesInfoAction extends TransportNodesAction< NodesInfoRequest, NodesInfoResponse, TransportNodesInfoAction.NodeInfoRequest, - NodeInfo> { + NodeInfo, + Void> { public static final ActionType TYPE = new ActionType<>("cluster:monitor/nodes/info"); private final NodeService nodeService; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java index 6894e68b49ed1..dba50efe54401 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.reload; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsResponse.java index a4a2263f2dcc9..8c2f5dc76073e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.reload; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java index fc75608317011..c84df0ddfe644 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.reload; @@ -38,7 +39,8 @@ public class TransportNodesReloadSecureSettingsAction extends TransportNodesActi NodesReloadSecureSettingsRequest, NodesReloadSecureSettingsResponse, NodesReloadSecureSettingsRequest.NodeRequest, - NodesReloadSecureSettingsResponse.NodeResponse> { + NodesReloadSecureSettingsResponse.NodeResponse, + Void> { public static final ActionType TYPE = new ActionType<>("cluster:admin/nodes/reload_secure_settings"); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodePrevalidateShardPathRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodePrevalidateShardPathRequest.java index f04610ea28a7d..467b99b4b7955 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodePrevalidateShardPathRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodePrevalidateShardPathRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.shutdown; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodePrevalidateShardPathResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodePrevalidateShardPathResponse.java index cc0728260b30e..5198d6cc3463b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodePrevalidateShardPathResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodePrevalidateShardPathResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.shutdown; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodesRemovalPrevalidation.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodesRemovalPrevalidation.java index e87222e2f914a..0e93b8006cbc1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodesRemovalPrevalidation.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodesRemovalPrevalidation.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.shutdown; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateNodeRemovalAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateNodeRemovalAction.java index 4e996055a1962..08eadca79663b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateNodeRemovalAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateNodeRemovalAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.shutdown; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateNodeRemovalRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateNodeRemovalRequest.java index 1d70d25478b12..92ebab8e7b030 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateNodeRemovalRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateNodeRemovalRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.shutdown; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateNodeRemovalResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateNodeRemovalResponse.java index f7c402bf8f55d..ea3ddc092f2c9 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateNodeRemovalResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateNodeRemovalResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.shutdown; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateShardPathRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateShardPathRequest.java index b8070bef1e32b..4998f537a9ac5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateShardPathRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateShardPathRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.shutdown; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateShardPathResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateShardPathResponse.java index d036d653962f5..646c3bd58d8cf 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateShardPathResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateShardPathResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.shutdown; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportPrevalidateNodeRemovalAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportPrevalidateNodeRemovalAction.java index 052263a00811d..560cb391476c2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportPrevalidateNodeRemovalAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportPrevalidateNodeRemovalAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.shutdown; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportPrevalidateShardPathAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportPrevalidateShardPathAction.java index 401a415756abe..8c49175c320fd 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportPrevalidateShardPathAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportPrevalidateShardPathAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.shutdown; @@ -43,7 +44,8 @@ public class TransportPrevalidateShardPathAction extends TransportNodesAction< PrevalidateShardPathRequest, PrevalidateShardPathResponse, NodePrevalidateShardPathRequest, - NodePrevalidateShardPathResponse> { + NodePrevalidateShardPathResponse, + Void> { public static final String ACTION_NAME = "internal:admin/indices/prevalidate_shard_path"; public static final ActionType TYPE = new ActionType<>(ACTION_NAME); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java index 1a53cec1bdbd7..e194101738489 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.stats; @@ -14,7 +15,6 @@ import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings; import org.elasticsearch.cluster.routing.allocation.NodeAllocationStats; -import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ChunkedToXContent; @@ -42,8 +42,6 @@ import java.util.Map; import java.util.Objects; -import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.singleChunk; - /** * Node statistics (dynamic, changes depending on when created). */ @@ -344,58 +342,46 @@ public void writeTo(StreamOutput out) throws IOException { @Override public Iterator toXContentChunked(ToXContent.Params outerParams) { - - return Iterators.concat( - - singleChunk((builder, params) -> { - builder.field("name", getNode().getName()); - builder.field("transport_address", getNode().getAddress().toString()); - builder.field("host", getNode().getHostName()); - builder.field("ip", getNode().getAddress()); - - builder.startArray("roles"); - for (DiscoveryNodeRole role : getNode().getRoles()) { - builder.value(role.roleName()); - } - builder.endArray(); - - if (getNode().getAttributes().isEmpty() == false) { - builder.startObject("attributes"); - for (Map.Entry attrEntry : getNode().getAttributes().entrySet()) { - builder.field(attrEntry.getKey(), attrEntry.getValue()); - } - builder.endObject(); + return ChunkedToXContent.builder(outerParams).append((builder, params) -> { + builder.field("name", getNode().getName()); + builder.field("transport_address", getNode().getAddress().toString()); + builder.field("host", getNode().getHostName()); + builder.field("ip", getNode().getAddress()); + + builder.startArray("roles"); + for (DiscoveryNodeRole role : getNode().getRoles()) { + builder.value(role.roleName()); + } + builder.endArray(); + + if (getNode().getAttributes().isEmpty() == false) { + builder.startObject("attributes"); + for (Map.Entry attrEntry : getNode().getAttributes().entrySet()) { + builder.field(attrEntry.getKey(), attrEntry.getValue()); } - - return builder; - }), - - ifPresent(getIndices()).toXContentChunked(outerParams), - - singleChunk( - (builder, p) -> builder.value(ifPresent(getOs()), p).value(ifPresent(getProcess()), p).value(ifPresent(getJvm()), p) - ), - - ifPresent(getThreadPool()).toXContentChunked(outerParams), - singleChunk(ifPresent(getFs())), - ifPresent(getTransport()).toXContentChunked(outerParams), - ifPresent(getHttp()).toXContentChunked(outerParams), - singleChunk(ifPresent(getBreaker())), - ifPresent(getScriptStats()).toXContentChunked(outerParams), - singleChunk(ifPresent(getDiscoveryStats())), - ifPresent(getIngestStats()).toXContentChunked(outerParams), - singleChunk(ifPresent(getAdaptiveSelectionStats())), - ifPresent(getScriptCacheStats()).toXContentChunked(outerParams), - singleChunk( + builder.endObject(); + } + return builder; + }) + + .appendIfPresent(getIndices()) + .append((builder, p) -> builder.value(ifPresent(getOs()), p).value(ifPresent(getProcess()), p).value(ifPresent(getJvm()), p)) + + .appendIfPresent(getThreadPool()) + .appendIfPresent(getFs()) + .appendIfPresent(getTransport()) + .appendIfPresent(getHttp()) + .appendIfPresent(getBreaker()) + .appendIfPresent(getScriptStats()) + .appendIfPresent(getDiscoveryStats()) + .appendIfPresent(getIngestStats()) + .appendIfPresent(getAdaptiveSelectionStats()) + .appendIfPresent(getScriptCacheStats()) + .append( (builder, p) -> builder.value(ifPresent(getIndexingPressureStats()), p) .value(ifPresent(getRepositoriesStats()), p) .value(ifPresent(getNodeAllocationStats()), p) - ) - ); - } - - private static ChunkedToXContent ifPresent(@Nullable ChunkedToXContent chunkedToXContent) { - return Objects.requireNonNullElse(chunkedToXContent, ChunkedToXContent.EMPTY); + ); } private static ToXContent ifPresent(@Nullable ToXContent toXContent) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java index b1fbe3f16a18b..1b3b3fd473b19 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.stats; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java index 3304edcce0831..ca5e397d1d95e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.stats; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestParameters.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestParameters.java index 42d6655d1706b..c5e8f37ed3a96 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestParameters.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestParameters.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.stats; @@ -116,7 +117,7 @@ public static Metric get(String name) { } public static void writeSetTo(StreamOutput out, EnumSet metrics) throws IOException { - if (out.getTransportVersion().onOrAfter(TransportVersions.NODES_STATS_ENUM_SET)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeEnumSet(metrics); } else { out.writeCollection(metrics, (output, metric) -> output.writeString(metric.metricName)); @@ -124,7 +125,7 @@ public static void writeSetTo(StreamOutput out, EnumSet metrics) throws } public static EnumSet readSetFrom(StreamInput in) throws IOException { - if (in.getTransportVersion().onOrAfter(TransportVersions.NODES_STATS_ENUM_SET)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { return in.readEnumSet(Metric.class); } else { return in.readCollection((i) -> EnumSet.noneOf(Metric.class), (is, out) -> { diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsResponse.java index 09bb6909191d1..1f0e0daf77d7b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.stats; @@ -13,10 +14,9 @@ import org.elasticsearch.action.support.nodes.BaseNodesXContentResponse; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.xcontent.ToXContent; import java.io.IOException; @@ -41,15 +41,12 @@ protected void writeNodesTo(StreamOutput out, List nodes) throws IOEx @Override protected Iterator xContentChunks(ToXContent.Params outerParams) { - return Iterators.concat( - ChunkedToXContentHelper.startObject("nodes"), - Iterators.flatMap(getNodes().iterator(), nodeStats -> Iterators.concat(Iterators.single((builder, params) -> { - builder.startObject(nodeStats.getNode().getId()); - builder.field("timestamp", nodeStats.getTimestamp()); - return builder; - }), nodeStats.toXContentChunked(outerParams), ChunkedToXContentHelper.endObject())), - ChunkedToXContentHelper.endObject() - ); + return ChunkedToXContent.builder(outerParams) + .object( + "nodes", + getNodes().iterator(), + (b, ns) -> b.object(ns.getNode().getId(), ob -> ob.field("timestamp", ns.getTimestamp()).append(ns)) + ); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java index be2cd5fb187a3..8021b0d583642 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.stats; @@ -45,7 +46,8 @@ public class TransportNodesStatsAction extends TransportNodesAction< NodesStatsRequest, NodesStatsResponse, TransportNodesStatsAction.NodeStatsRequest, - NodeStats> { + NodeStats, + Void> { public static final ActionType TYPE = new ActionType<>("cluster:monitor/nodes/stats"); @@ -82,6 +84,7 @@ protected NodesStatsResponse newResponse(NodesStatsRequest request, List responses, List failures, ActionListener listener @@ -164,8 +167,7 @@ public NodeStatsRequest(StreamInput in) throws IOException { super(in); skipLegacyNodesRequestHeader(TransportVersions.V_8_13_0, in); this.nodesStatsRequestParameters = new NodesStatsRequestParameters(in); - if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0) - && in.getTransportVersion().before(TransportVersions.DROP_UNUSED_NODES_IDS)) { + if (in.getTransportVersion().between(TransportVersions.V_8_13_0, TransportVersions.V_8_15_0)) { in.readStringArray(); // formerly nodeIds, now unused } } @@ -193,8 +195,7 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); sendLegacyNodesRequestHeader(TransportVersions.V_8_13_0, out); nodesStatsRequestParameters.writeTo(out); - if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0) - && out.getTransportVersion().before(TransportVersions.DROP_UNUSED_NODES_IDS)) { + if (out.getTransportVersion().between(TransportVersions.V_8_13_0, TransportVersions.V_8_15_0)) { out.writeStringArray(Strings.EMPTY_ARRAY); // formerly nodeIds, now unused } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/CancelTasksRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/CancelTasksRequest.java index d3596f2a652bf..97c68c4ffe732 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/CancelTasksRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/CancelTasksRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.tasks.cancel; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/CancelTasksRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/CancelTasksRequestBuilder.java index 39c19e225d175..e97f7c2b1ac68 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/CancelTasksRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/CancelTasksRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.tasks.cancel; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java index 1ff0a78575cac..8264ef10d713d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.tasks.cancel; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskRequest.java index 7f1a7cd00a8b1..0b36091cf27e3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.tasks.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskRequestBuilder.java index 6d8d4eceed3f5..e74e7d6b60013 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.tasks.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskResponse.java index 505a450eaf80a..5f14cdcf8f31a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.tasks.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java index 36d31f9ec59fc..0f5c519ecd1f6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.tasks.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksRequest.java index f42f138288de0..e1daea3ff1e2e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.tasks.list; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksRequestBuilder.java index f12b5cf6e202e..80806207eb6ac 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.tasks.list; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java index 6116a96faf60b..14dacb23e8a4b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.tasks.list; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskGroup.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskGroup.java index 7be064e597ef1..fbb917cff1d34 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskGroup.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskGroup.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.tasks.list; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java index 83b4b2fc45626..8910676094601 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.tasks.list; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodeUsage.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodeUsage.java index 2fccab17a02fd..f55c9857c4222 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodeUsage.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodeUsage.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.usage; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodesUsageRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodesUsageRequest.java index 31fd93685e2c6..4c6c428539437 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodesUsageRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodesUsageRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.usage; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodesUsageResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodesUsageResponse.java index 715d8d09d9098..463b607dd62e7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodesUsageResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodesUsageResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.usage; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/TransportNodesUsageAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/TransportNodesUsageAction.java index 1f8a049a678e5..8ec451f4bc4a6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/TransportNodesUsageAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/TransportNodesUsageAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.node.usage; @@ -33,7 +34,8 @@ public class TransportNodesUsageAction extends TransportNodesAction< NodesUsageRequest, NodesUsageResponse, TransportNodesUsageAction.NodeUsageRequest, - NodeUsage> { + NodeUsage, + Void> { public static final ActionType TYPE = new ActionType<>("cluster:monitor/nodes/usage"); private final UsageService restUsageService; @@ -91,7 +93,7 @@ public static class NodeUsageRequest extends TransportRequest { public NodeUsageRequest(StreamInput in) throws IOException { super(in); - skipLegacyNodesRequestHeader(TransportVersions.MORE_LIGHTER_NODES_REQUESTS, in); + skipLegacyNodesRequestHeader(TransportVersions.V_8_15_0, in); restActions = in.readBoolean(); aggregations = in.readBoolean(); } @@ -104,7 +106,7 @@ public NodeUsageRequest(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - sendLegacyNodesRequestHeader(TransportVersions.MORE_LIGHTER_NODES_REQUESTS, out); + sendLegacyNodesRequestHeader(TransportVersions.V_8_15_0, out); out.writeBoolean(restActions); out.writeBoolean(aggregations); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteClusterNodesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteClusterNodesAction.java index 71a5614cf5b25..9bdef4bfe4a13 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteClusterNodesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteClusterNodesAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.remote; @@ -114,8 +115,7 @@ protected void doExecute(Task task, Request request, ActionListener li } private void executeWithSystemContext(Request request, ThreadContext threadContext, ActionListener listener) { - try (var ignore = threadContext.stashContext()) { - threadContext.markAsSystemContext(); + try (var ignore = threadContext.newEmptySystemContext()) { if (request.remoteClusterServer) { final NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().clear() .addMetrics(NodesInfoMetrics.Metric.REMOTE_CLUSTER_SERVER.metricName()); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoRequest.java index f1294f2ece024..5a95b122f331c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.remote; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoResponse.java index 7a19e7b277a08..514d33107e5e1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.remote; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/TransportRemoteInfoAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/TransportRemoteInfoAction.java index b72b573796b1e..c79025cc99f18 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/TransportRemoteInfoAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/TransportRemoteInfoAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.remote; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryRequest.java index e4615b28af749..cdcf4bdad7b1a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.repositories.cleanup; @@ -28,7 +29,7 @@ public CleanupRepositoryRequest(TimeValue masterNodeTimeout, TimeValue ackTimeou } public static CleanupRepositoryRequest readFrom(StreamInput in) throws IOException { - if (in.getTransportVersion().onOrAfter(TransportVersions.SNAPSHOT_REQUEST_TIMEOUTS)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { return new CleanupRepositoryRequest(in); } else { return new CleanupRepositoryRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS, in); @@ -47,7 +48,7 @@ public CleanupRepositoryRequest(TimeValue masterNodeTimeout, TimeValue ackTimeou @Override public void writeTo(StreamOutput out) throws IOException { - if (out.getTransportVersion().onOrAfter(TransportVersions.SNAPSHOT_REQUEST_TIMEOUTS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { super.writeTo(out); } out.writeString(repository); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryRequestBuilder.java index 5e694cd92a46f..19d44d157e94d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.repositories.cleanup; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryResponse.java index a7ba179c22ef7..d7f6a15c0e3e5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.repositories.cleanup; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java index e6c5b63b5c2cf..891ebaa1ff7a3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.repositories.cleanup; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequest.java index cf2317fc143e6..e3bf63a5adf60 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.repositories.delete; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequestBuilder.java index 22983504e33a2..db7e11b751e8d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.repositories.delete; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java index 0b3645cd47364..104111e38645e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.repositories.delete; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesAction.java index 11e31a539fffd..822b5a95aea18 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.repositories.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesRequest.java index c8670c40f6aed..649af2b844730 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.repositories.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesRequestBuilder.java index 04a7655a60e4f..387dbdc334956 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.repositories.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java index 4c153c7331dba..95c95486ad458 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.repositories.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java index df7e235c00a3f..447908bec84c5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.repositories.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java index cda1df3dc1c2c..9b23eff5e1c61 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.repositories.put; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java index 78ad0dbdfa999..d2b317fa62e83 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.repositories.put; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java index 29999f98b553c..2992a017f8174 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.repositories.put; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryAction.java index d3dc7c916f066..79d9ac4c24a6c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.repositories.reservedstate; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java index 84e3ed77c64fc..f2e0f8e416071 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.repositories.verify; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryAction.java index fe66441c41844..c67d7314ab887 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.repositories.verify; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryRequest.java index 0876e48f929c8..42a70756a012c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.repositories.verify; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryRequestBuilder.java index d756ca93133c3..3f6dd3a2c6ff4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.repositories.verify; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java index 8d48141f9e268..d6984c8f90b87 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.repositories.verify; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequest.java index b245a752524ec..2ce2ccb5c20cb 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.reroute; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java index 7111f64d050de..6c6ca40a23aa7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.reroute; @@ -12,19 +13,17 @@ import org.elasticsearch.action.support.master.IsAcknowledgedSupplier; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.routing.allocation.RoutingExplanations; -import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.common.xcontent.ChunkedToXContentObject; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.xcontent.ToXContent; import java.io.IOException; -import java.util.Collections; import java.util.Iterator; import java.util.Objects; @@ -95,20 +94,15 @@ public Iterator toXContentChunked(ToXContent.Params outerP } @Override - public Iterator toXContentChunkedV7(ToXContent.Params outerParams) { - return Iterators.concat( - Iterators.single((builder, params) -> builder.startObject().field(ACKNOWLEDGED_KEY, isAcknowledged())), - emitState(outerParams) - ? ChunkedToXContentHelper.wrapWithObject("state", state.toXContentChunked(outerParams)) - : Collections.emptyIterator(), - Iterators.single((builder, params) -> { - if (params.paramAsBoolean("explain", false)) { - explanations.toXContent(builder, params); - } - - builder.endObject(); - return builder; - }) - ); + public Iterator toXContentChunkedV7(ToXContent.Params params) { + return ChunkedToXContent.builder(params).object(b -> { + b.field(ACKNOWLEDGED_KEY, isAcknowledged()); + if (emitState(params)) { + b.xContentObject("state", state); + } + if (params.paramAsBoolean("explain", false)) { + b.append(explanations); + } + }); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java index 3d90b4061f194..00bd90f4f59a0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.reroute; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsAction.java index 40511641baeaa..7e3c38c735091 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.settings; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsAction.java index a68e460b60529..de35cb1bb1e8f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.settings; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java index bf874c3c38870..f3f76c0d44723 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.settings; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java index a7682add8b092..e55256634e1b0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.settings; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsResponse.java index d99cc0b0ef8df..de754260dddbf 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.settings; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/RestClusterGetSettingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/RestClusterGetSettingsResponse.java index dbafefa1f1073..983aec7173776 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/RestClusterGetSettingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/RestClusterGetSettingsResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.settings; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterGetSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterGetSettingsAction.java index 41d010a6a9f5b..dce6a38001392 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterGetSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterGetSettingsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.settings; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java index de53ef70ff54a..e8dd773a054a6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.settings; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java index d912035c71070..56631fd41525a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.shards; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequest.java index 1f53840c20c06..5329128dfdeba 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.shards; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsResponse.java index 4781c06c1e2eb..c994cb3ea6dce 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.shards; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java index e727809ec56c1..b855f2cee7613 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.shards; @@ -16,6 +17,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardIterator; @@ -83,7 +85,7 @@ protected void masterOperation( String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request); Map> routingMap = indexNameExpressionResolver.resolveSearchRouting(state, request.routing(), request.indices()); Map indicesAndFilters = new HashMap<>(); - Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(clusterState, request.indices()); + Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(clusterState, request.indices()); for (String index : concreteIndices) { final AliasFilter aliasFilter = indicesService.buildAliasFilter(clusterState, index, indicesAndAliases); final String[] aliases = indexNameExpressionResolver.indexAliases( diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequest.java index dafd35f09f12f..ae5c36eab5d0d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.clone; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequestBuilder.java index a2726505dd834..ab7a1c1b3e68b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.clone; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java index 99e2ee6ca6e15..e01e9f4b90c67 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.clone; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java index d17ad9674a19d..b6ced06623306 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.create; @@ -117,7 +118,7 @@ public CreateSnapshotRequest(StreamInput in) throws IOException { waitForCompletion = in.readBoolean(); partial = in.readBoolean(); userMetadata = in.readGenericMap(); - uuid = in.getTransportVersion().onOrAfter(TransportVersions.REGISTER_SLM_STATS) ? in.readOptionalString() : null; + uuid = in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readOptionalString() : null; } @Override @@ -135,7 +136,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(waitForCompletion); out.writeBoolean(partial); out.writeGenericMap(userMetadata); - if (out.getTransportVersion().onOrAfter(TransportVersions.REGISTER_SLM_STATS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeOptionalString(uuid); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java index 983bb6e5d3a7c..2d2ebb72759f6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.create; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java index 4e04506d03d6a..d0919437d493f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.create; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java index 9752d764bd6e6..437dce47a5767 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.create; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequest.java index 771bab8d4d6b2..ab073f83e14da 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.delete; @@ -50,7 +51,7 @@ public DeleteSnapshotRequest(StreamInput in) throws IOException { super(in); repository = in.readString(); snapshots = in.readStringArray(); - if (in.getTransportVersion().onOrAfter(TransportVersions.DELETE_SNAPSHOTS_ASYNC_ADDED)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { waitForCompletion = in.readBoolean(); } } @@ -60,7 +61,7 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(repository); out.writeStringArray(snapshots); - if (out.getTransportVersion().onOrAfter(TransportVersions.DELETE_SNAPSHOTS_ASYNC_ADDED)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { out.writeBoolean(waitForCompletion); } else { assert waitForCompletion : "Using wait_for_completion parameter when it should have been disallowed"; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequestBuilder.java index ffe2a735cdd54..1e74401e00c15 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.delete; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java index 5c6259d133b44..1dfd83aee0407 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.delete; @@ -63,8 +64,7 @@ protected ClusterBlockException checkBlock(DeleteSnapshotRequest request, Cluste @Override protected void doExecute(Task task, DeleteSnapshotRequest request, ActionListener listener) { - if (clusterService.state().getMinTransportVersion().before(TransportVersions.DELETE_SNAPSHOTS_ASYNC_ADDED) - && request.waitForCompletion() == false) { + if (clusterService.state().getMinTransportVersion().before(TransportVersions.V_8_15_0) && request.waitForCompletion() == false) { throw new UnsupportedOperationException("wait_for_completion parameter is not supported by all nodes in this cluster"); } super.doExecute(task, request, listener); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesRequest.java index 695fb412d5735..144c25c8a4902 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.features; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java index 522ce39647464..3449d004e3447 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.features; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/ResetFeatureStateAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/ResetFeatureStateAction.java index ce1af6abd7732..e502ae558ab93 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/ResetFeatureStateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/ResetFeatureStateAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.features; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/ResetFeatureStateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/ResetFeatureStateRequest.java index ca867625272f9..f9d510a1a6a8f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/ResetFeatureStateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/ResetFeatureStateRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.features; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/ResetFeatureStateResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/ResetFeatureStateResponse.java index d0deab97b35e1..53d7f62d80960 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/ResetFeatureStateResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/ResetFeatureStateResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.features; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/SnapshottableFeaturesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/SnapshottableFeaturesAction.java index 7917c8eeb7da3..b8ae6271a035c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/SnapshottableFeaturesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/SnapshottableFeaturesAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.features; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportResetFeatureStateAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportResetFeatureStateAction.java index bb3fbd401b3a9..8d971597bd42d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportResetFeatureStateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportResetFeatureStateAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.features; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportSnapshottableFeaturesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportSnapshottableFeaturesAction.java index c12b12a18dd2d..a16a462b012d1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportSnapshottableFeaturesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportSnapshottableFeaturesAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.features; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java index 7c797444fc458..b99059e6711ca 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java index 54583a3ba23da..4ace7536a86e1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java index f7dedc21f93b6..1111d8d99f28b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/SnapshotNamePredicate.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/SnapshotNamePredicate.java index 44c75e95cbd6d..f1f9239e390dd 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/SnapshotNamePredicate.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/SnapshotNamePredicate.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/SnapshotSortKey.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/SnapshotSortKey.java index 14735d13ae68e..77f1584bf08d7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/SnapshotSortKey.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/SnapshotSortKey.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index 063fe49160d49..91bac0dfe3d17 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.get; @@ -321,7 +322,6 @@ public void onFailure(Exception e) { }) ), getSnapshotInfoExecutor.getMaxRunningTasks(), - () -> {}, () -> {} ); })); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/GetShardSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/GetShardSnapshotRequest.java index 96bedfdbfd108..61b9b00e1a3f2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/GetShardSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/GetShardSnapshotRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.get.shard; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/GetShardSnapshotResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/GetShardSnapshotResponse.java index 596d101595361..012a57e046b8d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/GetShardSnapshotResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/GetShardSnapshotResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.get.shard; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/TransportGetShardSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/TransportGetShardSnapshotAction.java index 6b31dd0c2d358..070d50e7a366e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/TransportGetShardSnapshotAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/TransportGetShardSnapshotAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.get.shard; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreClusterStateListener.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreClusterStateListener.java index b14f5a09408ec..e3907a9a276a6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreClusterStateListener.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreClusterStateListener.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.restore; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index f9ee2d84f8732..b7a513b3cb08e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.restore; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java index 7e862788ce7d3..03b83cb0c7704 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.restore; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponse.java index a0ef9fc1d3db1..a2ce7b0533d33 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.restore; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java index ba34b8cab1021..b21c8153d136d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.restore; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStage.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStage.java index 6065b44f4e7ad..c5db815852bfb 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStage.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStage.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.status; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java index 56bea66f1bdf3..ca3611d279e81 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.status; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexStatus.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexStatus.java index 4a98ff62f6293..102cadffa441e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexStatus.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexStatus.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.status; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotShardsStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotShardsStats.java index 28806b0aca87e..1be20e8a589da 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotShardsStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotShardsStats.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.status; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java index 4fe0c48831dd5..07689add763d3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.status; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatus.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatus.java index e228ad18641fe..36d502052969e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatus.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatus.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.status; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java index e9e91d6a72a8a..e74cbe9997da6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.status; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequestBuilder.java index d94b36e1055ba..f42b9b3085469 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.status; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java index 941f1c8d30b2c..cdb6622488caf 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.status; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java index 7c5d17ced1de5..42b71e275bb1b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.status; @@ -46,7 +47,8 @@ public class TransportNodesSnapshotsStatus extends TransportNodesAction< TransportNodesSnapshotsStatus.Request, TransportNodesSnapshotsStatus.NodesSnapshotStatus, TransportNodesSnapshotsStatus.NodeRequest, - TransportNodesSnapshotsStatus.NodeSnapshotStatus> { + TransportNodesSnapshotsStatus.NodeSnapshotStatus, + Void> { public static final String ACTION_NAME = TransportSnapshotsStatusAction.TYPE.name() + "[nodes]"; public static final ActionType TYPE = new ActionType<>(ACTION_NAME); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java index 7bd0c929773a8..13aa4aed1a5b6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.snapshots.status; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateAction.java index 96cb85d241b52..7ee9375c39ad2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.state; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java index d64de6e452e7c..6f9714b7551fc 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.state; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestBuilder.java index d93c1a172171e..30cf6f6cf4e33 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.state; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponse.java index fca9121a3a858..3b3218b95d874 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.state; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java index 0e5d7cda4a0d4..50839fc639d98 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.state; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/AnalysisStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/AnalysisStats.java index 9105c20044223..ea857215cf514 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/AnalysisStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/AnalysisStats.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.stats; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshot.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshot.java index 68fd4c2a1529a..8500302e4f755 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshot.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshot.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.stats; @@ -40,7 +41,6 @@ *
    */ public final class CCSTelemetrySnapshot implements Writeable, ToXContentFragment { - public static final String CCS_TELEMETRY_FIELD_NAME = "_search"; private long totalCount; private long successCount; private final Map failureReasons; @@ -65,6 +65,9 @@ public final class CCSTelemetrySnapshot implements Writeable, ToXContentFragment private final Map clientCounts; private final Map byRemoteCluster; + // Whether we should use per-MRT (minimize roundtrips) metrics. + // ES|QL does not have "minimize_roundtrips" option, so we don't collect those metrics for ES|QL usage. + private boolean useMRT = true; /** * Creates a new stats instance with the provided info. @@ -190,6 +193,11 @@ public Map getByRemoteCluster() { return Collections.unmodifiableMap(byRemoteCluster); } + public CCSTelemetrySnapshot setUseMRT(boolean useMRT) { + this.useMRT = useMRT; + return this; + } + public static class PerClusterCCSTelemetry implements Writeable, ToXContentFragment { private long count; private long skippedCount; @@ -269,6 +277,11 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(count, skippedCount, took); } + + @Override + public String toString() { + return Strings.toString(this, true, true); + } } /** @@ -290,8 +303,10 @@ public void add(CCSTelemetrySnapshot stats) { stats.featureCounts.forEach((k, v) -> featureCounts.merge(k, v, Long::sum)); stats.clientCounts.forEach((k, v) -> clientCounts.merge(k, v, Long::sum)); took.add(stats.took); - tookMrtTrue.add(stats.tookMrtTrue); - tookMrtFalse.add(stats.tookMrtFalse); + if (useMRT) { + tookMrtTrue.add(stats.tookMrtTrue); + tookMrtFalse.add(stats.tookMrtFalse); + } remotesPerSearchMax = Math.max(remotesPerSearchMax, stats.remotesPerSearchMax); if (totalCount > 0 && oldCount > 0) { // Weighted average @@ -327,30 +342,28 @@ private static void publishLatency(XContentBuilder builder, String name, LongMet @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(CCS_TELEMETRY_FIELD_NAME); - { - builder.field("total", totalCount); - builder.field("success", successCount); - builder.field("skipped", skippedRemotes); - publishLatency(builder, "took", took); + builder.field("total", totalCount); + builder.field("success", successCount); + builder.field("skipped", skippedRemotes); + publishLatency(builder, "took", took); + if (useMRT) { publishLatency(builder, "took_mrt_true", tookMrtTrue); publishLatency(builder, "took_mrt_false", tookMrtFalse); - builder.field("remotes_per_search_max", remotesPerSearchMax); - builder.field("remotes_per_search_avg", remotesPerSearchAvg); - builder.field("failure_reasons", failureReasons); - builder.field("features", featureCounts); - builder.field("clients", clientCounts); - builder.startObject("clusters"); - { - for (var entry : byRemoteCluster.entrySet()) { - String remoteName = entry.getKey(); - if (RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY.equals(remoteName)) { - remoteName = SearchResponse.LOCAL_CLUSTER_NAME_REPRESENTATION; - } - builder.field(remoteName, entry.getValue()); + } + builder.field("remotes_per_search_max", remotesPerSearchMax); + builder.field("remotes_per_search_avg", remotesPerSearchAvg); + builder.field("failure_reasons", failureReasons); + builder.field("features", featureCounts); + builder.field("clients", clientCounts); + builder.startObject("clusters"); + { + for (var entry : byRemoteCluster.entrySet()) { + String remoteName = entry.getKey(); + if (RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY.equals(remoteName)) { + remoteName = SearchResponse.LOCAL_CLUSTER_NAME_REPRESENTATION; } + builder.field(remoteName, entry.getValue()); } - builder.endObject(); } builder.endObject(); return builder; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsage.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsage.java index b2d75ac8f61f3..29a7dcb5d07d8 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsage.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsage.java @@ -1,14 +1,16 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.stats; import org.elasticsearch.ElasticsearchSecurityException; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ShardOperationFailedException; @@ -19,10 +21,8 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.query.SearchTimeoutException; +import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskCancelledException; -import org.elasticsearch.transport.ConnectTransportException; -import org.elasticsearch.transport.NoSeedNodeLeftException; -import org.elasticsearch.transport.NoSuchRemoteClusterException; import java.util.Arrays; import java.util.HashMap; @@ -86,6 +86,15 @@ public Builder setClient(String client) { return this; } + public Builder setClientFromTask(Task task) { + String client = task.getHeader(Task.X_ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER); + if (client != null) { + return setClient(client); + } else { + return this; + } + } + public Builder skippedRemote(String remote) { this.skippedRemotes.add(remote); return this; @@ -117,7 +126,7 @@ public static Result getFailureType(Exception e) { if (unwrapped instanceof Exception) { e = (Exception) unwrapped; } - if (isRemoteUnavailable(e)) { + if (ExceptionsHelper.isRemoteUnavailableException(e)) { return Result.REMOTES_UNAVAILABLE; } if (ExceptionsHelper.unwrap(e, ResourceNotFoundException.class) != null) { @@ -135,6 +144,10 @@ public static Result getFailureType(Exception e) { if (ExceptionsHelper.unwrapCorruption(e) != null) { return Result.CORRUPTION; } + ElasticsearchStatusException se = (ElasticsearchStatusException) ExceptionsHelper.unwrap(e, ElasticsearchStatusException.class); + if (se != null && se.getDetailedMessage().contains("license")) { + return Result.LICENSE; + } // This is kind of last resort check - if we still don't know the reason but all shard failures are remote, // we assume it's remote's fault somehow. if (e instanceof SearchPhaseExecutionException spe) { @@ -148,27 +161,6 @@ public static Result getFailureType(Exception e) { return Result.UNKNOWN; } - /** - * Is this failure exception because remote was unavailable? - * See also: TransportResolveClusterAction#notConnectedError - */ - static boolean isRemoteUnavailable(Exception e) { - if (ExceptionsHelper.unwrap( - e, - ConnectTransportException.class, - NoSuchRemoteClusterException.class, - NoSeedNodeLeftException.class - ) != null) { - return true; - } - Throwable ill = ExceptionsHelper.unwrap(e, IllegalStateException.class, IllegalArgumentException.class); - if (ill != null && (ill.getMessage().contains("Unable to open any connections") || ill.getMessage().contains("unknown host"))) { - return true; - } - // Ok doesn't look like any of the known remote exceptions - return false; - } - /** * Is this failure coming from a remote cluster? */ diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetry.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetry.java index 6016378aa8867..3f04eceed7eb5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetry.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetry.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.stats; @@ -46,6 +47,7 @@ public enum Result { TIMEOUT("timeout"), CORRUPTION("corruption"), SECURITY("security"), + LICENSE("license"), // May be helpful if there's a lot of other reasons, and it may be hard to calculate the unknowns for some clients. UNKNOWN("other"); @@ -64,6 +66,7 @@ public String getName() { public static final String MRT_FEATURE = "mrt_on"; public static final String ASYNC_FEATURE = "async"; public static final String WILDCARD_FEATURE = "wildcards"; + public static final String PIT_FEATURE = "pit"; // The list of known Elastic clients. May be incomplete. public static final Set KNOWN_CLIENTS = Set.of( @@ -104,8 +107,14 @@ public String getName() { private final Map clientCounts; private final Map byRemoteCluster; + // Should we calculate separate metrics per MRT? + private final boolean useMRT; public CCSUsageTelemetry() { + this(true); + } + + public CCSUsageTelemetry(boolean useMRT) { this.byRemoteCluster = new ConcurrentHashMap<>(); totalCount = new LongAdder(); successCount = new LongAdder(); @@ -117,6 +126,7 @@ public CCSUsageTelemetry() { skippedRemotes = new LongAdder(); featureCounts = new ConcurrentHashMap<>(); clientCounts = new ConcurrentHashMap<>(); + this.useMRT = useMRT; } public void updateUsage(CCSUsage ccsUsage) { @@ -132,10 +142,12 @@ private void doUpdate(CCSUsage ccsUsage) { if (isSuccess(ccsUsage)) { successCount.increment(); took.record(searchTook); - if (isMRT(ccsUsage)) { - tookMrtTrue.record(searchTook); - } else { - tookMrtFalse.record(searchTook); + if (useMRT) { + if (isMRT(ccsUsage)) { + tookMrtTrue.record(searchTook); + } else { + tookMrtFalse.record(searchTook); + } } ccsUsage.getPerClusterUsage().forEach((r, u) -> byRemoteCluster.computeIfAbsent(r, PerClusterCCSTelemetry::new).update(u)); } else { @@ -241,6 +253,6 @@ public CCSTelemetrySnapshot getCCSTelemetrySnapshot() { Collections.unmodifiableMap(Maps.transformValues(featureCounts, LongAdder::longValue)), Collections.unmodifiableMap(Maps.transformValues(clientCounts, LongAdder::longValue)), Collections.unmodifiableMap(Maps.transformValues(byRemoteCluster, PerClusterCCSTelemetry::getSnapshot)) - ); + ).setUseMRT(useMRT); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsFeatures.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsFeatures.java new file mode 100644 index 0000000000000..6e85093a52cdd --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsFeatures.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.admin.cluster.stats; + +import org.elasticsearch.features.FeatureSpecification; +import org.elasticsearch.features.NodeFeature; + +import java.util.Set; + +/** + * Spec for cluster stats features. + */ +public class ClusterStatsFeatures implements FeatureSpecification { + + @Override + public Set getFeatures() { + return Set.of(MappingStats.SOURCE_MODES_FEATURE); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java index 8e3b41a4876d4..28d90451ca503 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.stats; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java index 732eb2ec2dcc2..48b4e967742cd 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.stats; @@ -30,7 +31,8 @@ public class ClusterStatsNodeResponse extends BaseNodeResponse { private final ClusterHealthStatus clusterStatus; private final SearchUsageStats searchUsageStats; private final RepositoryUsageStats repositoryUsageStats; - private final CCSTelemetrySnapshot ccsMetrics; + private final CCSTelemetrySnapshot searchCcsMetrics; + private final CCSTelemetrySnapshot esqlCcsMetrics; public ClusterStatsNodeResponse(StreamInput in) throws IOException { super(in); @@ -43,15 +45,17 @@ public ClusterStatsNodeResponse(StreamInput in) throws IOException { } else { searchUsageStats = new SearchUsageStats(); } - if (in.getTransportVersion().onOrAfter(TransportVersions.REPOSITORIES_TELEMETRY)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { repositoryUsageStats = RepositoryUsageStats.readFrom(in); + searchCcsMetrics = new CCSTelemetrySnapshot(in); } else { repositoryUsageStats = RepositoryUsageStats.EMPTY; + searchCcsMetrics = new CCSTelemetrySnapshot(); } - if (in.getTransportVersion().onOrAfter(TransportVersions.CCS_TELEMETRY_STATS)) { - ccsMetrics = new CCSTelemetrySnapshot(in); + if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_CCS_TELEMETRY_STATS)) { + esqlCcsMetrics = new CCSTelemetrySnapshot(in); } else { - ccsMetrics = new CCSTelemetrySnapshot(); + esqlCcsMetrics = new CCSTelemetrySnapshot(); } } @@ -63,7 +67,8 @@ public ClusterStatsNodeResponse( ShardStats[] shardsStats, SearchUsageStats searchUsageStats, RepositoryUsageStats repositoryUsageStats, - CCSTelemetrySnapshot ccsTelemetrySnapshot + CCSTelemetrySnapshot ccsTelemetrySnapshot, + CCSTelemetrySnapshot esqlTelemetrySnapshot ) { super(node); this.nodeInfo = nodeInfo; @@ -72,7 +77,8 @@ public ClusterStatsNodeResponse( this.clusterStatus = clusterStatus; this.searchUsageStats = Objects.requireNonNull(searchUsageStats); this.repositoryUsageStats = Objects.requireNonNull(repositoryUsageStats); - this.ccsMetrics = ccsTelemetrySnapshot; + this.searchCcsMetrics = ccsTelemetrySnapshot; + this.esqlCcsMetrics = esqlTelemetrySnapshot; } public NodeInfo nodeInfo() { @@ -103,8 +109,12 @@ public RepositoryUsageStats repositoryUsageStats() { return repositoryUsageStats; } - public CCSTelemetrySnapshot getCcsMetrics() { - return ccsMetrics; + public CCSTelemetrySnapshot getSearchCcsMetrics() { + return searchCcsMetrics; + } + + public CCSTelemetrySnapshot getEsqlCcsMetrics() { + return esqlCcsMetrics; } @Override @@ -117,11 +127,12 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_6_0)) { searchUsageStats.writeTo(out); } - if (out.getTransportVersion().onOrAfter(TransportVersions.REPOSITORIES_TELEMETRY)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { repositoryUsageStats.writeTo(out); + searchCcsMetrics.writeTo(out); } // else just drop these stats, ok for bwc - if (out.getTransportVersion().onOrAfter(TransportVersions.CCS_TELEMETRY_STATS)) { - ccsMetrics.writeTo(out); + if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_CCS_TELEMETRY_STATS)) { + esqlCcsMetrics.writeTo(out); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java index 70060fc834452..5c4be62723e07 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.stats; @@ -792,6 +793,8 @@ static class IndexPressureStats implements ToXContentFragment { long currentCoordinatingOps = 0; long currentPrimaryOps = 0; long currentReplicaOps = 0; + long lowWaterMarkSplits = 0; + long highWaterMarkSplits = 0; for (NodeStats nodeStat : nodeStats) { IndexingPressureStats nodeStatIndexingPressureStats = nodeStat.getIndexingPressureStats(); if (nodeStatIndexingPressureStats != null) { @@ -815,6 +818,8 @@ static class IndexPressureStats implements ToXContentFragment { currentReplicaOps += nodeStatIndexingPressureStats.getCurrentReplicaOps(); primaryDocumentRejections += nodeStatIndexingPressureStats.getPrimaryDocumentRejections(); totalCoordinatingRequests += nodeStatIndexingPressureStats.getTotalCoordinatingRequests(); + lowWaterMarkSplits += nodeStatIndexingPressureStats.getLowWaterMarkSplits(); + highWaterMarkSplits += nodeStatIndexingPressureStats.getHighWaterMarkSplits(); } } indexingPressureStats = new IndexingPressureStats( @@ -837,7 +842,9 @@ static class IndexPressureStats implements ToXContentFragment { currentPrimaryOps, currentReplicaOps, primaryDocumentRejections, - totalCoordinatingRequests + totalCoordinatingRequests, + lowWaterMarkSplits, + highWaterMarkSplits ); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRequest.java index 77652eeb7d94e..632a2dc725e0f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.stats; @@ -19,16 +20,51 @@ * A request to get cluster level stats. */ public class ClusterStatsRequest extends BaseNodesRequest { + /** + * Should the remote cluster stats be included in the response. + */ + private final boolean doRemotes; + /** + * Return stripped down stats for remote clusters. + */ + private boolean remoteStats; + /** * Get stats from nodes based on the nodes ids specified. If none are passed, stats * based on all nodes will be returned. */ public ClusterStatsRequest(String... nodesIds) { + this(false, nodesIds); + } + + public ClusterStatsRequest(boolean doRemotes, String... nodesIds) { super(nodesIds); + this.doRemotes = doRemotes; + this.remoteStats = false; } @Override public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { return new CancellableTask(id, type, action, "", parentTaskId, headers); } + + public static ClusterStatsRequest newRemoteClusterStatsRequest() { + final var request = new ClusterStatsRequest(); + request.remoteStats = true; + return request; + } + + /** + * Should the remote cluster stats be included in the response. + */ + public boolean doRemotes() { + return doRemotes; + } + + /** + * Should the response be a stripped down version of the stats for remote clusters. + */ + public boolean isRemoteStats() { + return remoteStats; + } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java index d6a28eaadd6c7..43f56ade29634 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.stats; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java index 267db92496f76..ed8ca2f94a78b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.stats; @@ -17,14 +18,15 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; import java.util.List; import java.util.Locale; - -import static org.elasticsearch.action.search.TransportSearchAction.CCS_TELEMETRY_FEATURE_FLAG; +import java.util.Map; +import java.util.Set; public class ClusterStatsResponse extends BaseNodesResponse implements ToXContentFragment { @@ -33,10 +35,14 @@ public class ClusterStatsResponse extends BaseNodesResponse remoteClustersStats; + + public static final String CCS_TELEMETRY_FIELD_NAME = "_search"; + public static final String ESQL_TELEMETRY_FIELD_NAME = "_esql"; public ClusterStatsResponse( long timestamp, @@ -47,7 +53,8 @@ public ClusterStatsResponse( MappingStats mappingStats, AnalysisStats analysisStats, VersionStats versionStats, - ClusterSnapshotStats clusterSnapshotStats + ClusterSnapshotStats clusterSnapshotStats, + Map remoteClustersStats ) { super(clusterName, nodes, failures); this.clusterUUID = clusterUUID; @@ -55,6 +62,7 @@ public ClusterStatsResponse( nodesStats = new ClusterStatsNodes(nodes); indicesStats = new ClusterStatsIndices(nodes, mappingStats, analysisStats, versionStats); ccsMetrics = new CCSTelemetrySnapshot(); + esqlMetrics = new CCSTelemetrySnapshot().setUseMRT(false); ClusterHealthStatus status = null; for (ClusterStatsNodeResponse response : nodes) { // only the master node populates the status @@ -63,7 +71,10 @@ public ClusterStatsResponse( break; } } - nodes.forEach(node -> ccsMetrics.add(node.getCcsMetrics())); + nodes.forEach(node -> { + ccsMetrics.add(node.getSearchCcsMetrics()); + esqlMetrics.add(node.getEsqlCcsMetrics()); + }); this.status = status; this.clusterSnapshotStats = clusterSnapshotStats; @@ -74,6 +85,7 @@ public ClusterStatsResponse( // stats should be the same on every node so just pick one of them .findAny() .orElse(RepositoryUsageStats.EMPTY); + this.remoteClustersStats = remoteClustersStats; } public String getClusterUUID() { @@ -100,6 +112,10 @@ public CCSTelemetrySnapshot getCcsMetrics() { return ccsMetrics; } + public Map getRemoteClustersStats() { + return remoteClustersStats; + } + @Override public void writeTo(StreamOutput out) throws IOException { TransportAction.localOnly(); @@ -135,12 +151,22 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("repositories"); repositoryUsageStats.toXContent(builder, params); - if (CCS_TELEMETRY_FEATURE_FLAG.isEnabled()) { - builder.startObject("ccs"); - ccsMetrics.toXContent(builder, params); + builder.startObject("ccs"); + if (remoteClustersStats != null) { + builder.field("clusters", remoteClustersStats); + } + builder.startObject(CCS_TELEMETRY_FIELD_NAME); + ccsMetrics.toXContent(builder, params); + builder.endObject(); + + if (esqlMetrics.getTotalCount() > 0) { + builder.startObject(ESQL_TELEMETRY_FIELD_NAME); + esqlMetrics.toXContent(builder, params); builder.endObject(); } + builder.endObject(); + return builder; } @@ -149,4 +175,74 @@ public String toString() { return Strings.toString(this, true, true); } + /** + * Represents the information about a remote cluster. + */ + public record RemoteClusterStats( + String clusterUUID, + String mode, + boolean skipUnavailable, + String transportCompress, + Set versions, + String status, + long nodesCount, + long shardsCount, + long indicesCount, + long indicesBytes, + long heapBytes, + long memBytes + ) implements ToXContentFragment { + public RemoteClusterStats(String mode, boolean skipUnavailable, String transportCompress) { + this( + "unavailable", + mode, + skipUnavailable, + transportCompress.toLowerCase(Locale.ROOT), + Set.of(), + "unavailable", + 0, + 0, + 0, + 0, + 0, + 0 + ); + } + + public RemoteClusterStats acceptResponse(RemoteClusterStatsResponse remoteResponse) { + return new RemoteClusterStats( + remoteResponse.getClusterUUID(), + mode, + skipUnavailable, + transportCompress, + remoteResponse.getVersions(), + remoteResponse.getStatus().name().toLowerCase(Locale.ROOT), + remoteResponse.getNodesCount(), + remoteResponse.getShardsCount(), + remoteResponse.getIndicesCount(), + remoteResponse.getIndicesBytes(), + remoteResponse.getHeapBytes(), + remoteResponse.getMemBytes() + ); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("cluster_uuid", clusterUUID); + builder.field("mode", mode); + builder.field("skip_unavailable", skipUnavailable); + builder.field("transport.compress", transportCompress); + builder.field("status", status); + builder.field("version", versions); + builder.field("nodes_count", nodesCount); + builder.field("shards_count", shardsCount); + builder.field("indices_count", indicesCount); + builder.humanReadableField("indices_total_size_in_bytes", "indices_total_size", ByteSizeValue.ofBytes(indicesBytes)); + builder.humanReadableField("max_heap_in_bytes", "max_heap", ByteSizeValue.ofBytes(heapBytes)); + builder.humanReadableField("mem_total_in_bytes", "mem_total", ByteSizeValue.ofBytes(memBytes)); + builder.endObject(); + return builder; + } + } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/DenseVectorFieldStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/DenseVectorFieldStats.java index 099c299e0114e..ef91a921e83ab 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/DenseVectorFieldStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/DenseVectorFieldStats.java @@ -1,18 +1,20 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.stats; -import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import java.util.Objects; /** @@ -20,7 +22,10 @@ */ public final class DenseVectorFieldStats extends FieldStats { static final int UNSET = -1; - + static final String NOT_INDEXED = "not_indexed"; + Map vectorIndexTypeCount; // count of mappings by index type + Map vectorSimilarityTypeCount; // count of mappings by similarity + Map vectorElementTypeCount; // count of mappings by element type int indexedVectorCount; // number of times vectors with index:true are used in mappings of this cluster int indexedVectorDimMin; // minimum dimension of indexed vectors in this cluster int indexedVectorDimMax; // maximum dimension of indexed vectors in this cluster @@ -30,21 +35,14 @@ public final class DenseVectorFieldStats extends FieldStats { indexedVectorCount = 0; indexedVectorDimMin = UNSET; indexedVectorDimMax = UNSET; - } - - DenseVectorFieldStats(StreamInput in) throws IOException { - super(in); - indexedVectorCount = in.readVInt(); - indexedVectorDimMin = in.readVInt(); - indexedVectorDimMax = in.readVInt(); + vectorIndexTypeCount = new HashMap<>(); + vectorSimilarityTypeCount = new HashMap<>(); + vectorElementTypeCount = new HashMap<>(); } @Override public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeVInt(indexedVectorCount); - out.writeVInt(indexedVectorDimMin); - out.writeVInt(indexedVectorDimMax); + assert false : "writeTo should not be called on DenseVectorFieldStats"; } @Override @@ -52,6 +50,21 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.field("indexed_vector_count", indexedVectorCount); builder.field("indexed_vector_dim_min", indexedVectorDimMin); builder.field("indexed_vector_dim_max", indexedVectorDimMax); + if (vectorIndexTypeCount.isEmpty() == false) { + builder.startObject("vector_index_type_count"); + builder.mapContents(vectorIndexTypeCount); + builder.endObject(); + } + if (vectorSimilarityTypeCount.isEmpty() == false) { + builder.startObject("vector_similarity_type_count"); + builder.mapContents(vectorSimilarityTypeCount); + builder.endObject(); + } + if (vectorElementTypeCount.isEmpty() == false) { + builder.startObject("vector_element_type_count"); + builder.mapContents(vectorElementTypeCount); + builder.endObject(); + } } @Override @@ -68,11 +81,53 @@ public boolean equals(Object o) { DenseVectorFieldStats that = (DenseVectorFieldStats) o; return indexedVectorCount == that.indexedVectorCount && indexedVectorDimMin == that.indexedVectorDimMin - && indexedVectorDimMax == that.indexedVectorDimMax; + && indexedVectorDimMax == that.indexedVectorDimMax + && Objects.equals(vectorIndexTypeCount, that.vectorIndexTypeCount) + && Objects.equals(vectorSimilarityTypeCount, that.vectorSimilarityTypeCount) + && Objects.equals(vectorElementTypeCount, that.vectorElementTypeCount); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), indexedVectorCount, indexedVectorDimMin, indexedVectorDimMax); + return Objects.hash( + super.hashCode(), + indexedVectorCount, + indexedVectorDimMin, + indexedVectorDimMax, + vectorIndexTypeCount, + vectorSimilarityTypeCount, + vectorElementTypeCount + ); + } + + @Override + public String toString() { + return "DenseVectorFieldStats{" + + "vectorIndexTypeCount=" + + vectorIndexTypeCount + + ", vectorSimilarityTypeCount=" + + vectorSimilarityTypeCount + + ", vectorElementTypeCount=" + + vectorElementTypeCount + + ", indexedVectorCount=" + + indexedVectorCount + + ", indexedVectorDimMin=" + + indexedVectorDimMin + + ", indexedVectorDimMax=" + + indexedVectorDimMax + + ", scriptCount=" + + scriptCount + + ", scriptLangs=" + + scriptLangs + + ", fieldScriptStats=" + + fieldScriptStats + + ", name='" + + name + + '\'' + + ", count=" + + count + + ", indexCount=" + + indexCount + + '}'; } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/FieldScriptStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/FieldScriptStats.java index d3ed4203cb2fd..0947542ee9874 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/FieldScriptStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/FieldScriptStats.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.stats; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/FieldStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/FieldStats.java index 1341297526abf..dbd57d2bb40b0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/FieldStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/FieldStats.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.stats; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/IndexFeatureStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/IndexFeatureStats.java index 863426e86fb36..55cb2a4332076 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/IndexFeatureStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/IndexFeatureStats.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.stats; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/LongMetric.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/LongMetric.java index f3bb936b108c0..07d9c11ae4c07 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/LongMetric.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/LongMetric.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.stats; @@ -73,6 +74,7 @@ public static LongMetricValue fromStream(StreamInput in) throws IOException { try { // TODO: not sure what is the good value for minBarForHighestToLowestValueRatio here? Histogram dh = Histogram.decodeFromCompressedByteBuffer(bb, 1); + dh.setAutoResize(true); return new LongMetricValue(dh); } catch (DataFormatException e) { throw new IOException(e); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java index afd13b02ab3f2..1bc2e1d13c864 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java @@ -1,13 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.stats; +import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MappingMetadata; @@ -18,6 +20,8 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.Nullable; +import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.XContentBuilder; @@ -30,6 +34,7 @@ import java.util.HashSet; import java.util.IdentityHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.OptionalLong; @@ -43,6 +48,8 @@ */ public final class MappingStats implements ToXContentFragment, Writeable { + static final NodeFeature SOURCE_MODES_FEATURE = new NodeFeature("cluster.stats.source_modes"); + private static final Pattern DOC_PATTERN = Pattern.compile("doc[\\[.]"); private static final Pattern SOURCE_PATTERN = Pattern.compile("params\\._source"); @@ -52,6 +59,8 @@ public final class MappingStats implements ToXContentFragment, Writeable { public static MappingStats of(Metadata metadata, Runnable ensureNotCancelled) { Map fieldTypes = new HashMap<>(); Set concreteFieldNames = new HashSet<>(); + // Account different source modes based on index.mapping.source.mode setting: + Map sourceModeUsageCount = new HashMap<>(); Map runtimeFieldTypes = new HashMap<>(); final Map mappingCounts = new IdentityHashMap<>(metadata.getMappingsByHash().size()); for (IndexMetadata indexMetadata : metadata) { @@ -61,6 +70,9 @@ public static MappingStats of(Metadata metadata, Runnable ensureNotCancelled) { continue; } AnalysisStats.countMapping(mappingCounts, indexMetadata); + + var sourceMode = SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexMetadata.getSettings()); + sourceModeUsageCount.merge(sourceMode.toString().toLowerCase(Locale.ENGLISH), 1, Integer::sum); } final AtomicLong totalFieldCount = new AtomicLong(); final AtomicLong totalDeduplicatedFieldCount = new AtomicLong(); @@ -85,9 +97,30 @@ public static MappingStats of(Metadata metadata, Runnable ensureNotCancelled) { FieldStats stats; if (type.equals("dense_vector")) { stats = fieldTypes.computeIfAbsent(type, DenseVectorFieldStats::new); - boolean indexed = fieldMapping.containsKey("index") ? (boolean) fieldMapping.get("index") : false; + DenseVectorFieldStats vStats = (DenseVectorFieldStats) stats; + if (fieldMapping.containsKey("similarity")) { + Object similarity = fieldMapping.get("similarity"); + vStats.vectorSimilarityTypeCount.compute(similarity.toString(), (t, c) -> c == null ? count : c + count); + } + String elementTypeStr = "float"; + if (fieldMapping.containsKey("element_type")) { + Object elementType = fieldMapping.get("element_type"); + elementTypeStr = elementType.toString(); + } + vStats.vectorElementTypeCount.compute(elementTypeStr, (t, c) -> c == null ? count : c + count); + boolean indexed = fieldMapping.containsKey("index") && (boolean) fieldMapping.get("index"); if (indexed) { - DenseVectorFieldStats vStats = (DenseVectorFieldStats) stats; + Object indexOptions = fieldMapping.get("index_options"); + // NOTE, while the default for `float` is now `int8_hnsw`, that is actually added to the mapping + // if the value is truly missing & we are indexed, we default to hnsw. + String indexTypeStr = "hnsw"; + if (indexOptions instanceof Map indexOptionsMap) { + Object indexType = indexOptionsMap.get("type"); + if (indexType != null) { + indexTypeStr = indexType.toString(); + } + } + vStats.vectorIndexTypeCount.compute(indexTypeStr, (t, c) -> c == null ? count : c + count); vStats.indexedVectorCount += count; Object obj = fieldMapping.get("dims"); if (obj != null) { @@ -99,6 +132,8 @@ public static MappingStats of(Metadata metadata, Runnable ensureNotCancelled) { vStats.indexedVectorDimMax = dims; } } + } else { + vStats.vectorIndexTypeCount.compute(DenseVectorFieldStats.NOT_INDEXED, (t, c) -> c == null ? 1 : c + 1); } } else { stats = fieldTypes.computeIfAbsent(type, FieldStats::new); @@ -151,12 +186,14 @@ public static MappingStats of(Metadata metadata, Runnable ensureNotCancelled) { for (MappingMetadata mappingMetadata : metadata.getMappingsByHash().values()) { totalMappingSizeBytes += mappingMetadata.source().compressed().length; } + return new MappingStats( totalFieldCount.get(), totalDeduplicatedFieldCount.get(), totalMappingSizeBytes, fieldTypes.values(), - runtimeFieldTypes.values() + runtimeFieldTypes.values(), + sourceModeUsageCount ); } @@ -191,17 +228,20 @@ private static int countOccurrences(String script, Pattern pattern) { private final List fieldTypeStats; private final List runtimeFieldStats; + private final Map sourceModeUsageCount; MappingStats( long totalFieldCount, long totalDeduplicatedFieldCount, long totalMappingSizeBytes, Collection fieldTypeStats, - Collection runtimeFieldStats + Collection runtimeFieldStats, + Map sourceModeUsageCount ) { this.totalFieldCount = totalFieldCount; this.totalDeduplicatedFieldCount = totalDeduplicatedFieldCount; this.totalMappingSizeBytes = totalMappingSizeBytes; + this.sourceModeUsageCount = sourceModeUsageCount; List stats = new ArrayList<>(fieldTypeStats); stats.sort(Comparator.comparing(IndexFeatureStats::getName)); this.fieldTypeStats = Collections.unmodifiableList(stats); @@ -222,6 +262,10 @@ private static int countOccurrences(String script, Pattern pattern) { } fieldTypeStats = in.readCollectionAsImmutableList(FieldStats::new); runtimeFieldStats = in.readCollectionAsImmutableList(RuntimeFieldStats::new); + var transportVersion = in.getTransportVersion(); + sourceModeUsageCount = canReadOrWriteSourceModeTelemetry(transportVersion) + ? in.readImmutableMap(StreamInput::readString, StreamInput::readVInt) + : Map.of(); } @Override @@ -233,6 +277,15 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeCollection(fieldTypeStats); out.writeCollection(runtimeFieldStats); + var transportVersion = out.getTransportVersion(); + if (canReadOrWriteSourceModeTelemetry(transportVersion)) { + out.writeMap(sourceModeUsageCount, StreamOutput::writeVInt); + } + } + + private static boolean canReadOrWriteSourceModeTelemetry(TransportVersion version) { + return version.isPatchFrom(TransportVersions.SOURCE_MODE_TELEMETRY_FIX_8_17) + || version.onOrAfter(TransportVersions.SOURCE_MODE_TELEMETRY); } private static OptionalLong ofNullable(Long l) { @@ -276,6 +329,10 @@ public List getRuntimeFieldStats() { return runtimeFieldStats; } + public Map getSourceModeUsageCount() { + return sourceModeUsageCount; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject("mappings"); @@ -302,6 +359,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws st.toXContent(builder, params); } builder.endArray(); + builder.startObject("source_modes"); + var entries = sourceModeUsageCount.entrySet().stream().sorted(Map.Entry.comparingByKey()).toList(); + for (var entry : entries) { + builder.field(entry.getKey(), entry.getValue()); + } + builder.endObject(); builder.endObject(); return builder; } @@ -320,11 +383,19 @@ public boolean equals(Object o) { && Objects.equals(totalDeduplicatedFieldCount, that.totalDeduplicatedFieldCount) && Objects.equals(totalMappingSizeBytes, that.totalMappingSizeBytes) && fieldTypeStats.equals(that.fieldTypeStats) - && runtimeFieldStats.equals(that.runtimeFieldStats); + && runtimeFieldStats.equals(that.runtimeFieldStats) + && sourceModeUsageCount.equals(that.sourceModeUsageCount); } @Override public int hashCode() { - return Objects.hash(totalFieldCount, totalDeduplicatedFieldCount, totalMappingSizeBytes, fieldTypeStats, runtimeFieldStats); + return Objects.hash( + totalFieldCount, + totalDeduplicatedFieldCount, + totalMappingSizeBytes, + fieldTypeStats, + runtimeFieldStats, + sourceModeUsageCount + ); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingVisitor.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingVisitor.java index a451c224b6252..a4b7411867188 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingVisitor.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingVisitor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.stats; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RemoteClusterStatsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RemoteClusterStatsRequest.java new file mode 100644 index 0000000000000..6c3c5cbb50ece --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RemoteClusterStatsRequest.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.admin.cluster.stats; + +import org.elasticsearch.TransportVersions; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; + +/** + * A request to get cluster level stats from the remote cluster. + */ +public class RemoteClusterStatsRequest extends ActionRequest { + public RemoteClusterStatsRequest(StreamInput in) throws IOException { + super(in); + } + + public RemoteClusterStatsRequest() { + super(); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + assert out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) + : "RemoteClusterStatsRequest is not supported by the remote cluster"; + if (out.getTransportVersion().before(TransportVersions.V_8_16_0)) { + throw new UnsupportedOperationException("RemoteClusterStatsRequest is not supported by the remote cluster"); + } + super.writeTo(out); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RemoteClusterStatsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RemoteClusterStatsResponse.java new file mode 100644 index 0000000000000..9a140b6b7424e --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RemoteClusterStatsResponse.java @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.action.admin.cluster.stats; + +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.Set; + +/** + * Trimmed down cluster stats response for reporting to a remote cluster. + */ +public class RemoteClusterStatsResponse extends ActionResponse { + final String clusterUUID; + final ClusterHealthStatus status; + private final Set versions; + private final long nodesCount; + private final long shardsCount; + private final long indicesCount; + private final long indicesBytes; + private final long heapBytes; + private final long memBytes; + + public Set getVersions() { + return versions; + } + + public long getNodesCount() { + return nodesCount; + } + + public long getShardsCount() { + return shardsCount; + } + + public long getIndicesCount() { + return indicesCount; + } + + public long getIndicesBytes() { + return indicesBytes; + } + + public long getHeapBytes() { + return heapBytes; + } + + public long getMemBytes() { + return memBytes; + } + + public RemoteClusterStatsResponse( + String clusterUUID, + ClusterHealthStatus status, + Set versions, + long nodesCount, + long shardsCount, + long indicesCount, + long indicesBytes, + long heapBytes, + long memBytes + ) { + this.clusterUUID = clusterUUID; + this.status = status; + this.versions = versions; + this.nodesCount = nodesCount; + this.shardsCount = shardsCount; + this.indicesCount = indicesCount; + this.indicesBytes = indicesBytes; + this.heapBytes = heapBytes; + this.memBytes = memBytes; + } + + public String getClusterUUID() { + return this.clusterUUID; + } + + public ClusterHealthStatus getStatus() { + return this.status; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(clusterUUID); + status.writeTo(out); + out.writeStringCollection(versions); + out.writeLong(nodesCount); + out.writeLong(shardsCount); + out.writeLong(indicesCount); + out.writeLong(indicesBytes); + out.writeLong(heapBytes); + out.writeLong(memBytes); + } + + public RemoteClusterStatsResponse(StreamInput in) throws IOException { + super(in); + this.clusterUUID = in.readString(); + this.status = ClusterHealthStatus.readFrom(in); + this.versions = in.readCollectionAsSet(StreamInput::readString); + this.nodesCount = in.readLong(); + this.shardsCount = in.readLong(); + this.indicesCount = in.readLong(); + this.indicesBytes = in.readLong(); + this.heapBytes = in.readLong(); + this.memBytes = in.readLong(); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RepositoryUsageStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RepositoryUsageStats.java index 771aa0fbef842..9c8b6eb558065 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RepositoryUsageStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RepositoryUsageStats.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.stats; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RuntimeFieldStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RuntimeFieldStats.java index 6c182eaf2ad05..4c3c535f6c9c4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RuntimeFieldStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RuntimeFieldStats.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.stats; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/SearchUsageStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/SearchUsageStats.java index 7903b3f3dffde..a6e80b5efd08c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/SearchUsageStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/SearchUsageStats.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.stats; @@ -22,6 +23,7 @@ import java.util.Objects; import static org.elasticsearch.TransportVersions.V_8_12_0; +import static org.elasticsearch.TransportVersions.V_8_16_0; /** * Holds a snapshot of the search usage statistics. @@ -33,6 +35,7 @@ public final class SearchUsageStats implements Writeable, ToXContentFragment { private final Map queries; private final Map rescorers; private final Map sections; + private final Map retrievers; /** * Creates a new empty stats instance, that will get additional stats added through {@link #add(SearchUsageStats)} @@ -42,17 +45,25 @@ public SearchUsageStats() { this.queries = new HashMap<>(); this.sections = new HashMap<>(); this.rescorers = new HashMap<>(); + this.retrievers = new HashMap<>(); } /** * Creates a new stats instance with the provided info. The expectation is that when a new instance is created using * this constructor, the provided stats are final and won't be modified further. */ - public SearchUsageStats(Map queries, Map rescorers, Map sections, long totalSearchCount) { + public SearchUsageStats( + Map queries, + Map rescorers, + Map sections, + Map retrievers, + long totalSearchCount + ) { this.totalSearchCount = totalSearchCount; this.queries = queries; this.sections = sections; this.rescorers = rescorers; + this.retrievers = retrievers; } public SearchUsageStats(StreamInput in) throws IOException { @@ -60,6 +71,7 @@ public SearchUsageStats(StreamInput in) throws IOException { this.sections = in.readMap(StreamInput::readLong); this.totalSearchCount = in.readVLong(); this.rescorers = in.getTransportVersion().onOrAfter(V_8_12_0) ? in.readMap(StreamInput::readLong) : Map.of(); + this.retrievers = in.getTransportVersion().onOrAfter(V_8_16_0) ? in.readMap(StreamInput::readLong) : Map.of(); } @Override @@ -71,6 +83,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(V_8_12_0)) { out.writeMap(rescorers, StreamOutput::writeLong); } + if (out.getTransportVersion().onOrAfter(V_8_16_0)) { + out.writeMap(retrievers, StreamOutput::writeLong); + } } /** @@ -80,6 +95,7 @@ public void add(SearchUsageStats stats) { stats.queries.forEach((query, count) -> queries.merge(query, count, Long::sum)); stats.rescorers.forEach((rescorer, count) -> rescorers.merge(rescorer, count, Long::sum)); stats.sections.forEach((query, count) -> sections.merge(query, count, Long::sum)); + stats.retrievers.forEach((query, count) -> retrievers.merge(query, count, Long::sum)); this.totalSearchCount += stats.totalSearchCount; } @@ -94,6 +110,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.map(rescorers); builder.field("sections"); builder.map(sections); + builder.field("retrievers"); + builder.map(retrievers); } builder.endObject(); return builder; @@ -111,6 +129,10 @@ public Map getSectionsUsage() { return Collections.unmodifiableMap(sections); } + public Map getRetrieversUsage() { + return Collections.unmodifiableMap(retrievers); + } + public long getTotalSearchCount() { return totalSearchCount; } @@ -127,12 +149,13 @@ public boolean equals(Object o) { return totalSearchCount == that.totalSearchCount && queries.equals(that.queries) && rescorers.equals(that.rescorers) - && sections.equals(that.sections); + && sections.equals(that.sections) + && retrievers.equals(that.retrievers); } @Override public int hashCode() { - return Objects.hash(totalSearchCount, queries, rescorers, sections); + return Objects.hash(totalSearchCount, queries, rescorers, sections, retrievers); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/SynonymsStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/SynonymsStats.java index ceab7758e516d..4a5c8ec135f36 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/SynonymsStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/SynonymsStats.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.stats; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java index 66cf627ce066e..f074a6ed94b87 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -1,25 +1,34 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.stats; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.lucene.store.AlreadyClosedException; import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; +import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse.RemoteClusterStats; import org.elasticsearch.action.admin.indices.stats.CommonStats; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.CancellableFanOut; +import org.elasticsearch.action.support.RefCountingListener; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.action.support.nodes.TransportNodesAction; +import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterSnapshotStats; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.health.ClusterHealthStatus; @@ -29,8 +38,8 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CancellableSingleObjectCache; -import org.elasticsearch.common.util.concurrent.ListenableFuture; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.index.IndexService; @@ -42,10 +51,14 @@ import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.node.NodeService; import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.search.SearchService; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.RemoteClusterConnection; +import org.elasticsearch.transport.RemoteClusterService; +import org.elasticsearch.transport.RemoteConnectionInfo; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportService; import org.elasticsearch.transport.Transports; @@ -54,18 +67,26 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.concurrent.Executor; import java.util.function.BiFunction; import java.util.function.BooleanSupplier; +import java.util.stream.Collectors; +/** + * Transport action implementing _cluster/stats API. + */ public class TransportClusterStatsAction extends TransportNodesAction< ClusterStatsRequest, ClusterStatsResponse, TransportClusterStatsAction.ClusterStatsNodeRequest, - ClusterStatsNodeResponse> { + ClusterStatsNodeResponse, + SubscribableListener> { public static final ActionType TYPE = new ActionType<>("cluster:monitor/stats"); + private static final CommonStatsFlags SHARD_STATS_FLAGS = new CommonStatsFlags( CommonStatsFlags.Flag.Docs, CommonStatsFlags.Flag.Store, @@ -76,26 +97,33 @@ public class TransportClusterStatsAction extends TransportNodesAction< CommonStatsFlags.Flag.DenseVector, CommonStatsFlags.Flag.SparseVector ); + private static final Logger logger = LogManager.getLogger(TransportClusterStatsAction.class); + private final Settings settings; private final NodeService nodeService; private final IndicesService indicesService; private final RepositoriesService repositoriesService; private final SearchUsageHolder searchUsageHolder; private final CCSUsageTelemetry ccsUsageHolder; + private final CCSUsageTelemetry esqlUsageHolder; + private final Executor clusterStateStatsExecutor; private final MetadataStatsCache mappingStatsCache; private final MetadataStatsCache analysisStatsCache; + private final RemoteClusterService remoteClusterService; @Inject public TransportClusterStatsAction( ThreadPool threadPool, ClusterService clusterService, TransportService transportService, + Client client, NodeService nodeService, IndicesService indicesService, RepositoriesService repositoriesService, UsageService usageService, - ActionFilters actionFilters + ActionFilters actionFilters, + Settings settings ) { super( TYPE.name(), @@ -110,14 +138,37 @@ public TransportClusterStatsAction( this.repositoriesService = repositoriesService; this.searchUsageHolder = usageService.getSearchUsageHolder(); this.ccsUsageHolder = usageService.getCcsUsageHolder(); + this.esqlUsageHolder = usageService.getEsqlUsageHolder(); + this.clusterStateStatsExecutor = threadPool.executor(ThreadPool.Names.MANAGEMENT); this.mappingStatsCache = new MetadataStatsCache<>(threadPool.getThreadContext(), MappingStats::of); this.analysisStatsCache = new MetadataStatsCache<>(threadPool.getThreadContext(), AnalysisStats::of); + this.remoteClusterService = transportService.getRemoteClusterService(); + this.settings = settings; + + // register remote-cluster action with transport service only and not as a local-node Action that the Client can invoke + new TransportRemoteClusterStatsAction(client, transportService, actionFilters); + } + + @Override + protected SubscribableListener createActionContext(Task task, ClusterStatsRequest request) { + assert task instanceof CancellableTask; + final var cancellableTask = (CancellableTask) task; + final var additionalStatsListener = new SubscribableListener(); + if (request.isRemoteStats() == false) { + final AdditionalStats additionalStats = new AdditionalStats(); + additionalStats.compute(cancellableTask, request, additionalStatsListener); + } else { + // For remote stats request, we don't need to compute anything + additionalStatsListener.onResponse(null); + } + return additionalStatsListener; } @Override protected void newResponseAsync( final Task task, final ClusterStatsRequest request, + final SubscribableListener additionalStatsListener, final List responses, final List failures, final ActionListener listener @@ -127,41 +178,35 @@ protected void newResponseAsync( + "the cluster state that are too slow for a transport thread" ); assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.MANAGEMENT); - assert task instanceof CancellableTask; - final CancellableTask cancellableTask = (CancellableTask) task; - final ClusterState state = clusterService.state(); - final Metadata metadata = state.metadata(); - final ClusterSnapshotStats clusterSnapshotStats = ClusterSnapshotStats.of( - state, - clusterService.threadPool().absoluteTimeInMillis() - ); - final ListenableFuture mappingStatsStep = new ListenableFuture<>(); - final ListenableFuture analysisStatsStep = new ListenableFuture<>(); - mappingStatsCache.get(metadata, cancellableTask::isCancelled, mappingStatsStep); - analysisStatsCache.get(metadata, cancellableTask::isCancelled, analysisStatsStep); - mappingStatsStep.addListener( - listener.delegateFailureAndWrap( - (l, mappingStats) -> analysisStatsStep.addListener( - l.delegateFailureAndWrap( - (ll, analysisStats) -> ActionListener.completeWith( - ll, - () -> new ClusterStatsResponse( - System.currentTimeMillis(), - metadata.clusterUUID(), - clusterService.getClusterName(), - responses, - failures, - mappingStats, - analysisStats, - VersionStats.of(metadata, responses), - clusterSnapshotStats - ) - ) - ) + additionalStatsListener.andThenApply( + additionalStats -> request.isRemoteStats() + // Return stripped down stats for remote clusters + ? new ClusterStatsResponse( + System.currentTimeMillis(), + clusterService.state().metadata().clusterUUID(), + clusterService.getClusterName(), + responses, + List.of(), + null, + null, + null, + null, + Map.of() ) - ) - ); + : new ClusterStatsResponse( + System.currentTimeMillis(), + additionalStats.clusterUUID(), + clusterService.getClusterName(), + responses, + failures, + additionalStats.mappingStats(), + additionalStats.analysisStats(), + VersionStats.of(clusterService.state().metadata(), responses), + additionalStats.clusterSnapshotStats(), + additionalStats.getRemoteStats() + ) + ).addListener(listener); } @Override @@ -252,6 +297,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq final RepositoryUsageStats repositoryUsageStats = repositoriesService.getUsageStats(); final CCSTelemetrySnapshot ccsTelemetry = ccsUsageHolder.getCCSTelemetrySnapshot(); + final CCSTelemetrySnapshot esqlTelemetry = esqlUsageHolder.getCCSTelemetrySnapshot(); return new ClusterStatsNodeResponse( nodeInfo.getNode(), @@ -261,7 +307,8 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq shardsStats.toArray(new ShardStats[shardsStats.size()]), searchUsageStats, repositoryUsageStats, - ccsTelemetry + ccsTelemetry, + esqlTelemetry ); } @@ -272,7 +319,7 @@ public static class ClusterStatsNodeRequest extends TransportRequest { public ClusterStatsNodeRequest(StreamInput in) throws IOException { super(in); - skipLegacyNodesRequestHeader(TransportVersions.DROP_UNUSED_NODES_REQUESTS, in); + skipLegacyNodesRequestHeader(TransportVersions.V_8_15_0, in); } @Override @@ -283,7 +330,7 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId, @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - sendLegacyNodesRequestHeader(TransportVersions.DROP_UNUSED_NODES_REQUESTS, out); + sendLegacyNodesRequestHeader(TransportVersions.V_8_15_0, out); } } @@ -315,4 +362,150 @@ protected boolean isFresh(Long currentKey, Long newKey) { return newKey <= currentKey; } } + + public final class AdditionalStats { + + private String clusterUUID; + private MappingStats mappingStats; + private AnalysisStats analysisStats; + private ClusterSnapshotStats clusterSnapshotStats; + private Map remoteStats; + + void compute(CancellableTask task, ClusterStatsRequest request, ActionListener listener) { + clusterStateStatsExecutor.execute(ActionRunnable.wrap(listener, l -> { + task.ensureNotCancelled(); + internalCompute( + task, + request, + clusterService.state(), + mappingStatsCache, + analysisStatsCache, + task::isCancelled, + clusterService.threadPool().absoluteTimeInMillis(), + l.map(ignored -> this) + ); + })); + } + + private void internalCompute( + CancellableTask task, + ClusterStatsRequest request, + ClusterState clusterState, + MetadataStatsCache mappingStatsCache, + MetadataStatsCache analysisStatsCache, + BooleanSupplier isCancelledSupplier, + long absoluteTimeInMillis, + ActionListener listener + ) { + try (var listeners = new RefCountingListener(listener)) { + final var metadata = clusterState.metadata(); + clusterUUID = metadata.clusterUUID(); + mappingStatsCache.get(metadata, isCancelledSupplier, listeners.acquire(s -> mappingStats = s)); + analysisStatsCache.get(metadata, isCancelledSupplier, listeners.acquire(s -> analysisStats = s)); + clusterSnapshotStats = ClusterSnapshotStats.of(clusterState, absoluteTimeInMillis); + if (doRemotes(request)) { + var remotes = remoteClusterService.getRegisteredRemoteClusterNames(); + if (remotes.isEmpty()) { + remoteStats = Map.of(); + } else { + new RemoteStatsFanout(task, transportService.getThreadPool().executor(ThreadPool.Names.SEARCH_COORDINATION)).start( + task, + remotes, + listeners.acquire(s -> remoteStats = s) + ); + } + } + } + } + + String clusterUUID() { + return clusterUUID; + } + + MappingStats mappingStats() { + return mappingStats; + } + + AnalysisStats analysisStats() { + return analysisStats; + } + + ClusterSnapshotStats clusterSnapshotStats() { + return clusterSnapshotStats; + } + + public Map getRemoteStats() { + return remoteStats; + } + } + + private boolean doRemotes(ClusterStatsRequest request) { + return SearchService.CCS_COLLECT_TELEMETRY.get(settings) && request.doRemotes(); + } + + private class RemoteStatsFanout extends CancellableFanOut> { + private final Executor requestExecutor; + private final TaskId taskId; + private Map remoteClustersStats; + + RemoteStatsFanout(Task task, Executor requestExecutor) { + this.requestExecutor = requestExecutor; + this.taskId = new TaskId(clusterService.getNodeName(), task.getId()); + } + + @Override + protected void sendItemRequest(String clusterAlias, ActionListener listener) { + var remoteClusterClient = remoteClusterService.getRemoteClusterClient( + clusterAlias, + requestExecutor, + RemoteClusterService.DisconnectedStrategy.RECONNECT_IF_DISCONNECTED + ); + var remoteRequest = new RemoteClusterStatsRequest(); + remoteRequest.setParentTask(taskId); + remoteClusterClient.getConnection(remoteRequest, listener.delegateFailureAndWrap((responseListener, connection) -> { + if (connection.getTransportVersion().before(TransportVersions.V_8_16_0)) { + responseListener.onResponse(null); + } else { + remoteClusterClient.execute(connection, TransportRemoteClusterStatsAction.REMOTE_TYPE, remoteRequest, responseListener); + } + })); + } + + @Override + protected void onItemResponse(String clusterAlias, RemoteClusterStatsResponse response) { + if (response != null) { + remoteClustersStats.computeIfPresent(clusterAlias, (k, v) -> v.acceptResponse(response)); + } + } + + @Override + protected void onItemFailure(String clusterAlias, Exception e) { + logger.warn("Failed to get remote cluster stats for [{}]: {}", clusterAlias, e); + } + + void start(Task task, Collection remotes, ActionListener> listener) { + this.remoteClustersStats = remotes.stream().collect(Collectors.toConcurrentMap(r -> r, this::makeRemoteClusterStats)); + super.run(task, remotes.iterator(), listener); + } + + /** + * Create static portion of RemoteClusterStats for a given cluster alias. + */ + RemoteClusterStats makeRemoteClusterStats(String clusterAlias) { + RemoteClusterConnection remoteConnection = remoteClusterService.getRemoteClusterConnection(clusterAlias); + RemoteConnectionInfo remoteConnectionInfo = remoteConnection.getConnectionInfo(); + var compression = RemoteClusterService.REMOTE_CLUSTER_COMPRESS.getConcreteSettingForNamespace(clusterAlias).get(settings); + return new RemoteClusterStats( + remoteConnectionInfo.getModeInfo().modeName(), + remoteConnection.isSkipUnavailable(), + compression.toString() + ); + } + + @Override + protected Map onCompletion() { + return remoteClustersStats; + } + } + } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportRemoteClusterStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportRemoteClusterStatsAction.java new file mode 100644 index 0000000000000..882aaa8b18e15 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportRemoteClusterStatsAction.java @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.admin.cluster.stats; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.RemoteClusterActionType; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.client.internal.ParentTaskAssigningClient; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.transport.TransportService; + +/** + * Handler action for incoming {@link RemoteClusterStatsRequest}. + * Will pass the work to {@link TransportClusterStatsAction} and return the response. + */ +public class TransportRemoteClusterStatsAction extends HandledTransportAction { + + public static final String NAME = "cluster:monitor/stats/remote"; + public static final RemoteClusterActionType REMOTE_TYPE = new RemoteClusterActionType<>( + NAME, + RemoteClusterStatsResponse::new + ); + + private final Client client; + private final TransportService transportService; + + @Inject + public TransportRemoteClusterStatsAction(Client client, TransportService transportService, ActionFilters actionFilters) { + super(NAME, transportService, actionFilters, RemoteClusterStatsRequest::new, EsExecutors.DIRECT_EXECUTOR_SERVICE); + this.client = client; + this.transportService = transportService; + } + + @Override + protected void doExecute(Task task, RemoteClusterStatsRequest request, ActionListener listener) { + new ParentTaskAssigningClient(client, transportService.getLocalNode(), task).execute( + TransportClusterStatsAction.TYPE, + ClusterStatsRequest.newRemoteClusterStatsRequest(), + listener.map( + clusterStatsResponse -> new RemoteClusterStatsResponse( + clusterStatsResponse.getClusterUUID(), + clusterStatsResponse.getStatus(), + clusterStatsResponse.getNodesStats().getVersions(), + clusterStatsResponse.getNodesStats().getCounts().getTotal(), + clusterStatsResponse.getIndicesStats().getShards().getTotal(), + clusterStatsResponse.getIndicesStats().getIndexCount(), + clusterStatsResponse.getIndicesStats().getStore().sizeInBytes(), + clusterStatsResponse.getNodesStats().getJvm().getHeapMax().getBytes(), + clusterStatsResponse.getNodesStats().getOs().getMem().getTotal().getBytes() + ) + ) + ); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/VersionStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/VersionStats.java index 12fe846a9e68b..3645cea1d0187 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/VersionStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/VersionStats.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.stats; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequest.java index a893a77f660f3..b4ce32b13e908 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.storedscripts; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptContextAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptContextAction.java index f1b1e481aea2b..a7db2163af7ea 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptContextAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptContextAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.storedscripts; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptContextRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptContextRequest.java index f66ee121054a2..63fd3c0b8d1c3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptContextRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptContextRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.storedscripts; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptContextResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptContextResponse.java index 73cfeb48b96bc..845774a578e83 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptContextResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptContextResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.storedscripts; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageAction.java index e8655b3022abd..4d2f4308e25ed 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.storedscripts; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageRequest.java index 7abedcf69d308..5e9c923ef2684 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.storedscripts; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageResponse.java index f32cd3f7e0197..532e86db56ff2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.storedscripts; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptAction.java index 3a3756553bd76..7a29dc4509eae 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.storedscripts; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequest.java index f2f651b127573..712eb79422507 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.storedscripts; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptResponse.java index b33a718daee8a..b578aadb0f57a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.storedscripts; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java index 1ac151f0f4971..8e453cd5bac3a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java @@ -1,19 +1,22 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.storedscripts; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.script.StoredScriptSource; import org.elasticsearch.xcontent.ToXContentFragment; @@ -27,11 +30,15 @@ public class PutStoredScriptRequest extends AcknowledgedRequest implements ToXContentFragment { - private String id; - private String context; - private BytesReference content; - private XContentType xContentType; - private StoredScriptSource source; + @Nullable + private final String id; + + @Nullable + private final String context; + + private final BytesReference content; + private final XContentType xContentType; + private final StoredScriptSource source; public PutStoredScriptRequest(StreamInput in) throws IOException { super(in); @@ -42,15 +49,11 @@ public PutStoredScriptRequest(StreamInput in) throws IOException { source = new StoredScriptSource(in); } - public PutStoredScriptRequest(TimeValue masterNodeTimeout, TimeValue ackTimeout) { - super(masterNodeTimeout, ackTimeout); - } - public PutStoredScriptRequest( TimeValue masterNodeTimeout, TimeValue ackTimeout, - String id, - String context, + @Nullable String id, + @Nullable String context, BytesReference content, XContentType xContentType, StoredScriptSource source @@ -58,9 +61,9 @@ public PutStoredScriptRequest( super(masterNodeTimeout, ackTimeout); this.id = id; this.context = context; - this.content = content; + this.content = Objects.requireNonNull(content); this.xContentType = Objects.requireNonNull(xContentType); - this.source = source; + this.source = Objects.requireNonNull(source); } @Override @@ -73,10 +76,6 @@ public ActionRequestValidationException validate() { validationException = addValidationError("id cannot contain '#' for stored script", validationException); } - if (content == null) { - validationException = addValidationError("must specify code for stored script", validationException); - } - return validationException; } @@ -84,20 +83,10 @@ public String id() { return id; } - public PutStoredScriptRequest id(String id) { - this.id = id; - return this; - } - public String context() { return context; } - public PutStoredScriptRequest context(String context) { - this.context = context; - return this; - } - public BytesReference content() { return content; } @@ -110,16 +99,6 @@ public StoredScriptSource source() { return source; } - /** - * Set the script source and the content type of the bytes. - */ - public PutStoredScriptRequest content(BytesReference content, XContentType xContentType) { - this.content = content; - this.xContentType = Objects.requireNonNull(xContentType); - this.source = StoredScriptSource.parse(content, xContentType); - return this; - } - @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); @@ -132,28 +111,16 @@ public void writeTo(StreamOutput out) throws IOException { @Override public String toString() { - String source = "_na_"; - - try { - source = XContentHelper.convertToJson(content, false, xContentType); - } catch (Exception e) { - // ignore - } - - return "put stored script {id [" - + id - + "]" - + (context != null ? ", context [" + context + "]" : "") - + ", content [" - + source - + "]}"; + return Strings.format( + "put stored script {id [%s]%s, content [%s]}", + id, + context != null ? ", context [" + context + "]" : "", + source + ); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field("script"); - source.toXContent(builder, params); - - return builder; + return builder.field("script", source, params); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportDeleteStoredScriptAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportDeleteStoredScriptAction.java index 9362564e0855d..58361303fdcd4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportDeleteStoredScriptAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportDeleteStoredScriptAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.storedscripts; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetScriptContextAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetScriptContextAction.java index 58a68c558f4c7..260fe89d66356 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetScriptContextAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetScriptContextAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.storedscripts; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetScriptLanguageAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetScriptLanguageAction.java index f8fe7fb9b85a1..9766aa04c804e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetScriptLanguageAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetScriptLanguageAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.storedscripts; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetStoredScriptAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetStoredScriptAction.java index daf8411d0808f..609e8923558e2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetStoredScriptAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetStoredScriptAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.storedscripts; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java index 2ce5fa942c948..de77675901941 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.storedscripts; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksRequest.java index 976948dc722fc..79a04e724af74 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.tasks; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksResponse.java index d9eeb07a53c2c..c50f2b8011b1b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.tasks; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java index 7c4d77be1364b..708d0fbf64b64 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.cluster.tasks; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/Alias.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/Alias.java index bb202dc6c3772..a6959e5b59e06 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/Alias.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/Alias.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.alias; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesClusterStateUpdateRequest.java index 1f87cf618dfcf..c6d60f34eddeb 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesClusterStateUpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesClusterStateUpdateRequest.java @@ -1,39 +1,33 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.alias; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse.AliasActionResult; -import org.elasticsearch.cluster.ack.ClusterStateUpdateRequest; import org.elasticsearch.cluster.metadata.AliasAction; +import org.elasticsearch.core.TimeValue; import java.util.List; +import java.util.Objects; /** * Cluster state update request that allows to add or remove aliases */ -public class IndicesAliasesClusterStateUpdateRequest extends ClusterStateUpdateRequest { - private final List actions; - - private final List actionResults; - - public IndicesAliasesClusterStateUpdateRequest(List actions, List actionResults) { - this.actions = actions; - this.actionResults = actionResults; - } - - /** - * Returns the alias actions to be performed - */ - public List actions() { - return actions; - } - - public List getActionResults() { - return actionResults; +public record IndicesAliasesClusterStateUpdateRequest( + TimeValue masterNodeTimeout, + TimeValue ackTimeout, + List actions, + List actionResults +) { + public IndicesAliasesClusterStateUpdateRequest { + Objects.requireNonNull(masterNodeTimeout); + Objects.requireNonNull(ackTimeout); + Objects.requireNonNull(actions); + Objects.requireNonNull(actionResults); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java index f223d7fb2762f..d66cab1d2d717 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.alias; @@ -57,7 +58,26 @@ public class IndicesAliasesRequest extends AcknowledgedRequest { logger.debug("failed to perform aliases", e); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/BaseAliasesRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/BaseAliasesRequestBuilder.java index 4f80d933fa460..eb5045917fb98 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/BaseAliasesRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/BaseAliasesRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.alias.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesAction.java index a033abfd0a107..7b0a39a87e055 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.alias.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java index 09071f2e6ea3a..0cd32d7a8185f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.alias.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequestBuilder.java index c64ef8a9b5437..1a0aef78a604a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.alias.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponse.java index edb05b0fcef75..6bb6148cf3781 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.alias.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java index 77f2aca410e68..79850cc71d946 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.alias.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeAction.java index d19aacb306414..2ac738c3759d6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.analyze; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java index 708ba01019146..8bc3f021838d1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.analyze; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/ReloadAnalyzersRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/ReloadAnalyzersRequest.java index be33fada9c934..033725a5c9dcf 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/ReloadAnalyzersRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/ReloadAnalyzersRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.analyze; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/ReloadAnalyzersResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/ReloadAnalyzersResponse.java index 98b11de5bffc9..39836568bbf8f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/ReloadAnalyzersResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/ReloadAnalyzersResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.analyze; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java index 76a972f5b900b..fb672b49c2f5a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.analyze; @@ -17,6 +18,7 @@ import org.apache.lucene.analysis.tokenattributes.TypeAttribute; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.single.shard.TransportSingleShardAction; import org.elasticsearch.cluster.ClusterState; @@ -43,6 +45,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -454,11 +457,12 @@ private TokenCounter(int maxTokenCount) { private void increment() { tokenCount++; if (tokenCount > maxTokenCount) { - throw new IllegalStateException( + throw new ElasticsearchStatusException( "The number of tokens produced by calling _analyze has exceeded the allowed maximum of [" + maxTokenCount + "]." - + " This limit can be set by changing the [index.analyze.max_token_count] index level setting." + + " This limit can be set by changing the [index.analyze.max_token_count] index level setting.", + RestStatus.BAD_REQUEST ); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportReloadAnalyzersAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportReloadAnalyzersAction.java index 5b9810eb466d7..2d511f493aba7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportReloadAnalyzersAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportReloadAnalyzersAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.analyze; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheRequest.java index 8e0e4d62e611d..243ab111ae3c1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.cache.clear; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheRequestBuilder.java index 43ad9bff9af8d..8b8244d135621 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.cache.clear; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java index 51402bf8c80b5..fe81dce5eaa83 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.cache.clear; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexClusterStateUpdateRequest.java index fdecc4aaf8707..de8db5c025d13 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexClusterStateUpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexClusterStateUpdateRequest.java @@ -1,42 +1,33 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.close; import org.elasticsearch.action.support.ActiveShardCount; -import org.elasticsearch.cluster.ack.IndicesClusterStateUpdateRequest; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.Index; + +import java.util.Objects; /** * Cluster state update request that allows to close one or more indices */ -public class CloseIndexClusterStateUpdateRequest extends IndicesClusterStateUpdateRequest { - - private long taskId; - private ActiveShardCount waitForActiveShards = ActiveShardCount.DEFAULT; - - public CloseIndexClusterStateUpdateRequest(final long taskId) { - this.taskId = taskId; - } - - public long taskId() { - return taskId; - } - - public CloseIndexClusterStateUpdateRequest taskId(final long taskId) { - this.taskId = taskId; - return this; - } - - public ActiveShardCount waitForActiveShards() { - return waitForActiveShards; - } - - public CloseIndexClusterStateUpdateRequest waitForActiveShards(final ActiveShardCount waitForActiveShards) { - this.waitForActiveShards = waitForActiveShards; - return this; +public record CloseIndexClusterStateUpdateRequest( + TimeValue masterNodeTimeout, + TimeValue ackTimeout, + long taskId, + ActiveShardCount waitForActiveShards, + Index[] indices +) { + public CloseIndexClusterStateUpdateRequest { + Objects.requireNonNull(masterNodeTimeout); + Objects.requireNonNull(ackTimeout); + Objects.requireNonNull(waitForActiveShards); + Objects.requireNonNull(indices); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequest.java index 9a722f1bce2a9..5c292a1c8e056 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.close; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestBuilder.java index 0b19a9f6eb69a..fef6399111f86 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.close; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexResponse.java index 0a5f02bf591f6..c3bb8ac150920 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.close; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java index 9b6331a9d320d..5a4292804fd6c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.close; @@ -119,9 +120,13 @@ protected void masterOperation( return; } - final CloseIndexClusterStateUpdateRequest closeRequest = new CloseIndexClusterStateUpdateRequest(task.getId()).ackTimeout( - request.ackTimeout() - ).masterNodeTimeout(request.masterNodeTimeout()).waitForActiveShards(request.waitForActiveShards()).indices(concreteIndices); + final CloseIndexClusterStateUpdateRequest closeRequest = new CloseIndexClusterStateUpdateRequest( + request.masterNodeTimeout(), + request.ackTimeout(), + task.getId(), + request.waitForActiveShards(), + concreteIndices + ); indexStateService.closeIndices(closeRequest, listener.delegateResponse((delegatedListener, t) -> { logger.debug(() -> "failed to close indices [" + Arrays.toString(concreteIndices) + "]", t); delegatedListener.onFailure(t); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseAction.java index cfeab14d05e32..7bb638212c273 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.close; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java index fed8044477cc0..9bc088f944be0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.create; @@ -304,11 +305,14 @@ ClusterState execute( final CreateIndexClusterStateUpdateRequest updateRequest; if (isManagedSystemIndex) { - final SystemIndexDescriptor descriptor = mainDescriptor.getDescriptorCompatibleWith( - currentState.getMinSystemIndexMappingVersions().get(mainDescriptor.getPrimaryIndex()) - ); + final var requiredMinimumMappingVersion = currentState.getMinSystemIndexMappingVersions() + .get(mainDescriptor.getPrimaryIndex()); + final SystemIndexDescriptor descriptor = mainDescriptor.getDescriptorCompatibleWith(requiredMinimumMappingVersion); if (descriptor == null) { - final String message = mainDescriptor.getMinimumMappingsVersionMessage("auto-create index"); + final String message = mainDescriptor.getMinimumMappingsVersionMessage( + "auto-create index", + requiredMinimumMappingVersion + ); logger.warn(message); throw new IllegalStateException(message); } @@ -349,7 +353,7 @@ private CreateIndexClusterStateUpdateRequest buildUpdateRequest(String indexName request.cause(), indexName, request.index() - ).ackTimeout(request.ackTimeout()).performReroute(false).masterNodeTimeout(request.masterNodeTimeout()); + ).performReroute(false); logger.debug("Auto-creating index {}", indexName); return updateRequest; } @@ -366,7 +370,7 @@ private CreateIndexClusterStateUpdateRequest buildSystemIndexUpdateRequest(Strin request.cause(), concreteIndexName, request.index() - ).ackTimeout(request.ackTimeout()).masterNodeTimeout(request.masterNodeTimeout()).performReroute(false); + ).performReroute(false); updateRequest.waitForActiveShards(ActiveShardCount.ALL); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java index 948199fbe74f4..553f784d23a87 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.create; @@ -11,7 +12,6 @@ import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.support.ActiveShardCount; -import org.elasticsearch.cluster.ack.ClusterStateUpdateRequest; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; @@ -24,7 +24,7 @@ /** * Cluster state update request that allows to create an index */ -public class CreateIndexClusterStateUpdateRequest extends ClusterStateUpdateRequest { +public class CreateIndexClusterStateUpdateRequest { private final String cause; private final String index; @@ -35,6 +35,7 @@ public class CreateIndexClusterStateUpdateRequest extends ClusterStateUpdateRequ private ResizeType resizeType; private boolean copySettings; private SystemDataStreamDescriptor systemDataStreamDescriptor; + private boolean isFailureIndex = false; private Settings settings = Settings.EMPTY; @@ -102,6 +103,11 @@ public CreateIndexClusterStateUpdateRequest systemDataStreamDescriptor(SystemDat return this; } + public CreateIndexClusterStateUpdateRequest isFailureIndex(boolean isFailureIndex) { + this.isFailureIndex = isFailureIndex; + return this; + } + public String cause() { return cause; } @@ -168,6 +174,10 @@ public String dataStreamName() { return dataStreamName; } + public boolean isFailureIndex() { + return isFailureIndex; + } + public CreateIndexClusterStateUpdateRequest dataStreamName(String dataStreamName) { this.dataStreamName = dataStreamName; return this; @@ -228,6 +238,8 @@ public String toString() { + systemDataStreamDescriptor + ", matchingTemplate=" + matchingTemplate + + ", isFailureIndex=" + + isFailureIndex + '}'; } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java index 5d1b7264ebf81..cc96954c8a8e4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.create; @@ -67,8 +68,9 @@ public class CreateIndexRequest extends AcknowledgedRequest private boolean initializeFailureStore; private Settings settings = Settings.EMPTY; + public static final String EMPTY_MAPPINGS = "{}"; - private String mappings = "{}"; + private String mappings = EMPTY_MAPPINGS; private final Set aliases = new HashSet<>(); @@ -111,7 +113,7 @@ public CreateIndexRequest(StreamInput in) throws IOException { } else { requireDataStream = false; } - if (in.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_LAZY_CREATION)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { initializeFailureStore = in.readBoolean(); } else { initializeFailureStore = true; @@ -284,8 +286,11 @@ private CreateIndexRequest mapping(BytesReference source, XContentType xContentT } private CreateIndexRequest mapping(String type, Map source) { - // wrap it in a type map if its not - if (source.size() != 1 || source.containsKey(type) == false) { + if (source.isEmpty()) { + // If no source is provided we return empty mappings + return mapping(EMPTY_MAPPINGS); + } else if (source.size() != 1 || source.containsKey(type) == false) { + // wrap it in a type map if its not source = Map.of(MapperService.SINGLE_MAPPING_NAME, source); } else if (MapperService.SINGLE_MAPPING_NAME.equals(type) == false) { // if it has a different type name, then unwrap and rewrap with _doc @@ -475,13 +480,19 @@ public CreateIndexRequest requireDataStream(boolean requireDataStream) { return this; } + /** + * Returns whether the failure store should be initialized. N.B. If true, failure store index creation will be performed regardless of + * whether the template indicates that the failure store is enabled. + */ public boolean isInitializeFailureStore() { return initializeFailureStore; } /** * Set whether this CreateIndexRequest should initialize the failure store on data stream creation. This can be necessary when, for - * example, a failure occurs while trying to ingest a document into a data stream that has to be auto-created. + * example, a failure occurs while trying to ingest a document into a data stream that has to be auto-created. N.B. If true, failure + * store index creation will be performed regardless of whether the template indicates that the failure store is enabled. It is the + * caller's responsibility to ensure that this is correct. */ public CreateIndexRequest initializeFailureStore(boolean initializeFailureStore) { this.initializeFailureStore = initializeFailureStore; @@ -513,7 +524,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) { out.writeBoolean(this.requireDataStream); } - if (out.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_LAZY_CREATION)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { out.writeBoolean(this.initializeFailureStore); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java index 4e265f4052e72..1191dcef63316 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.create; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponse.java index f0596d061aeb3..a17c998230a31 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.create; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java index 438f3930dee7e..73f35e283f7d8 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.create; @@ -134,11 +135,10 @@ protected void masterOperation( // We check this via the request's origin. Eventually, `SystemIndexManager` will reconfigure // the index to the latest settings. if (isManagedSystemIndex && Strings.isNullOrEmpty(request.origin())) { - final SystemIndexDescriptor descriptor = mainDescriptor.getDescriptorCompatibleWith( - state.getMinSystemIndexMappingVersions().get(mainDescriptor.getPrimaryIndex()) - ); + final var requiredMinimumMappingVersion = state.getMinSystemIndexMappingVersions().get(mainDescriptor.getPrimaryIndex()); + final SystemIndexDescriptor descriptor = mainDescriptor.getDescriptorCompatibleWith(requiredMinimumMappingVersion); if (descriptor == null) { - final String message = mainDescriptor.getMinimumMappingsVersionMessage("create index"); + final String message = mainDescriptor.getMinimumMappingsVersionMessage("create index", requiredMinimumMappingVersion); logger.warn(message); listener.onFailure(new IllegalStateException(message)); return; @@ -149,6 +149,9 @@ protected void masterOperation( } createIndexService.createIndex( + request.masterNodeTimeout(), + request.ackTimeout(), + request.ackTimeout(), updateRequest, listener.map(response -> new CreateIndexResponse(response.isAcknowledged(), response.isShardsAcknowledged(), indexName)) ); @@ -165,9 +168,7 @@ private CreateIndexClusterStateUpdateRequest buildUpdateRequest( alias.isHidden(true); } }).collect(Collectors.toSet()); - return new CreateIndexClusterStateUpdateRequest(cause, indexName, request.index()).ackTimeout(request.ackTimeout()) - .masterNodeTimeout(request.masterNodeTimeout()) - .settings(request.settings()) + return new CreateIndexClusterStateUpdateRequest(cause, indexName, request.index()).settings(request.settings()) .mappings(request.mappings()) .aliases(aliases) .nameResolvedInstant(nameResolvedAt) @@ -195,15 +196,7 @@ private static CreateIndexClusterStateUpdateRequest buildSystemIndexUpdateReques ); } - final CreateIndexClusterStateUpdateRequest updateRequest = new CreateIndexClusterStateUpdateRequest( - cause, - descriptor.getPrimaryIndex(), - request.index() - ); - - return updateRequest.ackTimeout(request.ackTimeout()) - .masterNodeTimeout(request.masterNodeTimeout()) - .aliases(aliases) + return new CreateIndexClusterStateUpdateRequest(cause, descriptor.getPrimaryIndex(), request.index()).aliases(aliases) .waitForActiveShards(ActiveShardCount.ALL) .mappings(descriptor.getMappings()) .settings(settings); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/package-info.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/package-info.java index dc0fa891f4e49..84d5f83d91b14 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/package-info.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/DanglingIndexInfo.java b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/DanglingIndexInfo.java index 60eb3bf772969..a9b8a6f2985a0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/DanglingIndexInfo.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/DanglingIndexInfo.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.dangling; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/delete/DeleteDanglingIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/delete/DeleteDanglingIndexRequest.java index daceeece4f97b..5c083c7d343cb 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/delete/DeleteDanglingIndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/delete/DeleteDanglingIndexRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.dangling.delete; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/delete/TransportDeleteDanglingIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/delete/TransportDeleteDanglingIndexAction.java index 7b8d70693f40d..34aa8f4532270 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/delete/TransportDeleteDanglingIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/delete/TransportDeleteDanglingIndexAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.dangling.delete; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/find/FindDanglingIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/find/FindDanglingIndexRequest.java index 1f6d6e65b9128..f8abbf502565b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/find/FindDanglingIndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/find/FindDanglingIndexRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.dangling.find; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/find/FindDanglingIndexResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/find/FindDanglingIndexResponse.java index db2e433a4b9d0..92251c1c8c0b1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/find/FindDanglingIndexResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/find/FindDanglingIndexResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.dangling.find; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/find/NodeFindDanglingIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/find/NodeFindDanglingIndexRequest.java index b2cc484714174..bc1e172f77ce0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/find/NodeFindDanglingIndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/find/NodeFindDanglingIndexRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.dangling.find; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/find/NodeFindDanglingIndexResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/find/NodeFindDanglingIndexResponse.java index 30d9bb701c65c..5b65fd0772be3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/find/NodeFindDanglingIndexResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/find/NodeFindDanglingIndexResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.dangling.find; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/find/TransportFindDanglingIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/find/TransportFindDanglingIndexAction.java index ff2b32ac5cc6c..a181b059b82e3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/find/TransportFindDanglingIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/find/TransportFindDanglingIndexAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.dangling.find; @@ -33,7 +34,8 @@ public class TransportFindDanglingIndexAction extends TransportNodesAction< FindDanglingIndexRequest, FindDanglingIndexResponse, NodeFindDanglingIndexRequest, - NodeFindDanglingIndexResponse> { + NodeFindDanglingIndexResponse, + Void> { public static final ActionType TYPE = new ActionType<>("cluster:admin/indices/dangling/find"); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/import_index/ImportDanglingIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/import_index/ImportDanglingIndexRequest.java index be2fb10821662..4e493c154deac 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/import_index/ImportDanglingIndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/import_index/ImportDanglingIndexRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.dangling.import_index; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/import_index/TransportImportDanglingIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/import_index/TransportImportDanglingIndexAction.java index 4c90ce9875266..475649471768c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/import_index/TransportImportDanglingIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/import_index/TransportImportDanglingIndexAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.dangling.import_index; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/ListDanglingIndicesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/ListDanglingIndicesRequest.java index 450c45bf0742c..51cabaa5b24e2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/ListDanglingIndicesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/ListDanglingIndicesRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.dangling.list; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/ListDanglingIndicesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/ListDanglingIndicesResponse.java index bab73a072fe55..d942c4347960a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/ListDanglingIndicesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/ListDanglingIndicesResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.dangling.list; @@ -78,7 +79,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("index_name", info.indexName); builder.field("index_uuid", info.indexUUID); - builder.timeField("creation_date_millis", "creation_date", info.creationDateMillis); + builder.timestampFieldsFromUnixEpochMillis("creation_date_millis", "creation_date", info.creationDateMillis); builder.array("node_ids", info.nodeIds.toArray(new String[0])); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/NodeListDanglingIndicesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/NodeListDanglingIndicesRequest.java index bc548bc1ad58e..6baac2dbc8174 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/NodeListDanglingIndicesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/NodeListDanglingIndicesRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.dangling.list; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/NodeListDanglingIndicesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/NodeListDanglingIndicesResponse.java index f56dadd6a2482..c94f1fc1fc000 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/NodeListDanglingIndicesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/NodeListDanglingIndicesResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.dangling.list; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/TransportListDanglingIndicesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/TransportListDanglingIndicesAction.java index 40ca4e6de9435..3410e617e3ed4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/TransportListDanglingIndicesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/TransportListDanglingIndicesAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.dangling.list; @@ -35,7 +36,8 @@ public class TransportListDanglingIndicesAction extends TransportNodesAction< ListDanglingIndicesRequest, ListDanglingIndicesResponse, NodeListDanglingIndicesRequest, - NodeListDanglingIndicesResponse> { + NodeListDanglingIndicesResponse, + Void> { public static final ActionType TYPE = new ActionType<>("cluster:admin/indices/dangling/list"); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/package-info.java b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/package-info.java index 03fa46a811677..33c6dbb216ec4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/package-info.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexClusterStateUpdateRequest.java deleted file mode 100644 index dbb3901be8500..0000000000000 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexClusterStateUpdateRequest.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -package org.elasticsearch.action.admin.indices.delete; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.cluster.ClusterStateAckListener; -import org.elasticsearch.cluster.ClusterStateTaskListener; -import org.elasticsearch.cluster.ack.IndicesClusterStateUpdateRequest; -import org.elasticsearch.cluster.node.DiscoveryNode; - -/** - * Cluster state update request that allows to close one or more indices - */ -public class DeleteIndexClusterStateUpdateRequest extends IndicesClusterStateUpdateRequest - implements - ClusterStateAckListener, - ClusterStateTaskListener { - - private final ActionListener listener; - - public DeleteIndexClusterStateUpdateRequest(ActionListener listener) { - this.listener = listener; - } - - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - - @Override - public boolean mustAck(DiscoveryNode discoveryNode) { - return true; - } - - @Override - public void onAllNodesAcked() { - listener.onResponse(AcknowledgedResponse.TRUE); - } - - @Override - public void onAckFailure(Exception e) { - listener.onResponse(AcknowledgedResponse.FALSE); - } - - @Override - public void onAckTimeout() { - listener.onResponse(AcknowledgedResponse.FALSE); - } -} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequest.java index 2cb431577242d..96fae42b05505 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.delete; @@ -27,16 +28,25 @@ */ public class DeleteIndexRequest extends AcknowledgedRequest implements IndicesRequest.Replaceable { - public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.fromOptions( - false, - true, - true, - true, - false, - false, - true, - false - ); + public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.builder() + .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) + .wildcardOptions( + IndicesOptions.WildcardOptions.builder() + .matchOpen(true) + .matchClosed(true) + .allowEmptyExpressions(true) + .resolveAliases(false) + .build() + ) + .gatekeeperOptions( + IndicesOptions.GatekeeperOptions.builder() + .allowAliasToMultipleIndices(false) + .allowClosedIndices(true) + .ignoreThrottled(false) + .allowFailureIndices(true) + .build() + ) + .build(); private String[] indices; // Delete index should work by default on both open and closed indices. diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequestBuilder.java index 5c0aec258176a..b80c8c00bc965 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.delete; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java index a2cda7fc1ac33..fbf95699dd03a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.delete; @@ -91,11 +92,14 @@ protected void masterOperation( return; } - DeleteIndexClusterStateUpdateRequest deleteRequest = new DeleteIndexClusterStateUpdateRequest(listener.delegateResponse((l, e) -> { - logger.debug(() -> "failed to delete indices [" + concreteIndices + "]", e); - listener.onFailure(e); - })).ackTimeout(request.ackTimeout()).masterNodeTimeout(request.masterNodeTimeout()).indices(concreteIndices.toArray(new Index[0])); - - deleteIndexService.deleteIndices(deleteRequest); + deleteIndexService.deleteIndices( + request.masterNodeTimeout(), + request.ackTimeout(), + concreteIndices, + listener.delegateResponse((l, e) -> { + logger.debug(() -> "failed to delete indices [" + concreteIndices + "]", e); + listener.onFailure(e); + }) + ); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/package-info.java b/server/src/main/java/org/elasticsearch/action/admin/indices/delete/package-info.java index 08f59d45087bf..a136f4401763a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/package-info.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/delete/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/AnalyzeDiskUsageShardRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/AnalyzeDiskUsageShardRequest.java index 9eb8b5b4b6b4e..0864d755f95da 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/AnalyzeDiskUsageShardRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/AnalyzeDiskUsageShardRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.diskusage; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/AnalyzeDiskUsageShardResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/AnalyzeDiskUsageShardResponse.java index 331b97830f36b..55dde843bb6cf 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/AnalyzeDiskUsageShardResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/AnalyzeDiskUsageShardResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.diskusage; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/AnalyzeIndexDiskUsageRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/AnalyzeIndexDiskUsageRequest.java index 119bb8d9dec61..ace884bbead09 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/AnalyzeIndexDiskUsageRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/AnalyzeIndexDiskUsageRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.diskusage; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/AnalyzeIndexDiskUsageResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/AnalyzeIndexDiskUsageResponse.java index f42101d2b3adc..21548f1f0c118 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/AnalyzeIndexDiskUsageResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/AnalyzeIndexDiskUsageResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.diskusage; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java index 2f57b59c165e2..666708ea6ffde 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.diskusage; @@ -12,6 +13,7 @@ import org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat; import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat; import org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat; +import org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.codecs.FieldsProducer; import org.apache.lucene.codecs.KnnVectorsReader; @@ -19,7 +21,7 @@ import org.apache.lucene.codecs.PointsReader; import org.apache.lucene.codecs.StoredFieldsReader; import org.apache.lucene.codecs.TermVectorsReader; -import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat; +import org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.ByteVectorValues; import org.apache.lucene.index.DirectoryReader; @@ -303,6 +305,9 @@ private static void readProximity(Terms terms, PostingsEnum postings) throws IOE private static BlockTermState getBlockTermState(TermsEnum termsEnum, BytesRef term) throws IOException { if (term != null && termsEnum.seekExact(term)) { final TermState termState = termsEnum.termState(); + if (termState instanceof final Lucene912PostingsFormat.IntBlockTermState blockTermState) { + return new BlockTermState(blockTermState.docStartFP, blockTermState.posStartFP, blockTermState.payStartFP); + } if (termState instanceof final ES812PostingsFormat.IntBlockTermState blockTermState) { return new BlockTermState(blockTermState.docStartFP, blockTermState.posStartFP, blockTermState.payStartFP); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageStats.java b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageStats.java index 12abb10be4aba..d14d9cc19a33b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageStats.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.diskusage; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/TransportAnalyzeIndexDiskUsageAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/TransportAnalyzeIndexDiskUsageAction.java index 8676daf0a55d9..6c0eb79714093 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/TransportAnalyzeIndexDiskUsageAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/TransportAnalyzeIndexDiskUsageAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.diskusage; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushAction.java index bdcdfdb164a41..950095b7fe7c8 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.flush; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushRequest.java index fc326f804ce8a..33575dea95d6e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.flush; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushRequestBuilder.java index f23e247428698..f1187be0fb518 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.flush; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/ShardFlushRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/ShardFlushRequest.java index a510deff94d98..62f01561fc644 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/ShardFlushRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/ShardFlushRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.flush; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java index bc34e3fb99e04..279417e1f25e9 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java @@ -1,10 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.flush; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java index 52662a79d7cbd..5d184f56dd748 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.flush; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/package-info.java b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/package-info.java index a0180828601db..dd9c75caf3d37 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/package-info.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeAction.java index df6a42a67ae54..48bc73908b6e1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.forcemerge; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeRequest.java index 37075dd896b80..3a3b25f1c05de 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.forcemerge; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeRequestBuilder.java index d4c15ee799670..f955e4267daea 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.forcemerge; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/TransportForceMergeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/TransportForceMergeAction.java index c474a3d40cdc9..da08b78d711cf 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/TransportForceMergeAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/TransportForceMergeAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.forcemerge; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/package-info.java b/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/package-info.java index 9dc1cf95b0c4c..bf5a6aeae44ac 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/package-info.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexAction.java index ea2671eafe01f..c6104e6da5808 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java index 0b94e89fcc64d..801dbbdee0858 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.get; @@ -97,9 +98,7 @@ public GetIndexRequest() { super( DataStream.isFailureStoreFeatureFlagEnabled() ? IndicesOptions.builder(IndicesOptions.strictExpandOpen()) - .failureStoreOptions( - IndicesOptions.FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true) - ) + .selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE) .build() : IndicesOptions.strictExpandOpen() ); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequestBuilder.java index 032d64b6acf02..51088124b084d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java index 90fe22d0ee9d7..c3ed0c675c3c7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/get/TransportGetIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/get/TransportGetIndexAction.java index 7de14400ec2b5..72e5bb0085de6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/get/TransportGetIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/get/TransportGetIndexAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsAction.java index 0245457ac7588..585c006fa8913 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.mapping.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsIndexRequest.java index b4ab91a52c97f..6ac78e7c27aed 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsIndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsIndexRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.mapping.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java index 29461ea7455a4..415ccd5bf2a73 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.mapping.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequestBuilder.java index 98e0d626102cd..5fe64b6530096 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.mapping.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java index 923fc07ae14d2..4398b33cd798f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.mapping.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsAction.java index 2e5976f8239d5..ba877025429bc 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.mapping.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequest.java index 6554b2377c2a7..dd4114c947174 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.mapping.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequestBuilder.java index 1b1f52a7133f1..3f54138581398 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.mapping.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java index f013d2bb518d5..37edae05c22af 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.mapping.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java index 849977a39f2a6..d40f8e8de838b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.mapping.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java index 7b024737dc70a..a579abb66a3c7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.mapping.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java index 7691dac2e1dc4..8b5d15661faf0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.mapping.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/package-info.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/package-info.java index ddff6fdb6597a..d50bc6433007a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/package-info.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingClusterStateUpdateRequest.java index c8f80eae99a3d..2c469f4cae490 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingClusterStateUpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingClusterStateUpdateRequest.java @@ -1,40 +1,45 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.mapping.put; -import org.elasticsearch.cluster.ack.IndicesClusterStateUpdateRequest; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.Index; import java.io.IOException; +import java.util.Objects; /** * Cluster state update request that allows to put a mapping */ -public class PutMappingClusterStateUpdateRequest extends IndicesClusterStateUpdateRequest { - - private final CompressedXContent source; - private boolean autoUpdate; - - public PutMappingClusterStateUpdateRequest(String source) throws IOException { - this.source = CompressedXContent.fromJSON(source); - } - - public CompressedXContent source() { - return source; - } - - public PutMappingClusterStateUpdateRequest autoUpdate(boolean autoUpdate) { - this.autoUpdate = autoUpdate; - return this; +public record PutMappingClusterStateUpdateRequest( + TimeValue masterNodeTimeout, + TimeValue ackTimeout, + CompressedXContent source, + boolean autoUpdate, + Index[] indices +) { + public PutMappingClusterStateUpdateRequest { + Objects.requireNonNull(masterNodeTimeout); + Objects.requireNonNull(ackTimeout); + Objects.requireNonNull(source); + Objects.requireNonNull(indices); } - public boolean autoUpdate() { - return autoUpdate; + public PutMappingClusterStateUpdateRequest( + TimeValue masterNodeTimeout, + TimeValue ackTimeout, + String source, + boolean autoUpdate, + Index... indices + ) throws IOException { + this(masterNodeTimeout, ackTimeout, CompressedXContent.fromJSON(source), autoUpdate, indices); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java index 707286801cf66..7b782c6da5a84 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.mapping.put; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java index 85b758f1693d1..ebe0908231a85 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.mapping.put; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java index 6a3977bcc8d4a..4a01121ddfefe 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.mapping.put; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java index 737ee28bbf873..749470e181deb 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.mapping.put; @@ -156,10 +157,13 @@ static void performMappingUpdate( }); final PutMappingClusterStateUpdateRequest updateRequest; try { - updateRequest = new PutMappingClusterStateUpdateRequest(request.source()).indices(concreteIndices) - .ackTimeout(request.ackTimeout()) - .masterNodeTimeout(request.masterNodeTimeout()) - .autoUpdate(autoUpdate); + updateRequest = new PutMappingClusterStateUpdateRequest( + request.masterNodeTimeout(), + request.ackTimeout(), + request.source(), + autoUpdate, + concreteIndices + ); } catch (IOException e) { wrappedListener.onFailure(e); return; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/package-info.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/package-info.java index 54fe43d17d033..0f43e1753d7ed 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/package-info.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexAction.java index 8fb772496baf7..1fb78cfb244d8 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.open; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexClusterStateUpdateRequest.java index 50e02894990e8..92e2433a9b1cc 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexClusterStateUpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexClusterStateUpdateRequest.java @@ -1,32 +1,32 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.open; import org.elasticsearch.action.support.ActiveShardCount; -import org.elasticsearch.cluster.ack.IndicesClusterStateUpdateRequest; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.Index; + +import java.util.Objects; /** * Cluster state update request that allows to open one or more indices */ -public class OpenIndexClusterStateUpdateRequest extends IndicesClusterStateUpdateRequest { - - private ActiveShardCount waitForActiveShards = ActiveShardCount.DEFAULT; - - public OpenIndexClusterStateUpdateRequest() { - - } - - public ActiveShardCount waitForActiveShards() { - return waitForActiveShards; - } - - public OpenIndexClusterStateUpdateRequest waitForActiveShards(ActiveShardCount waitForActiveShards) { - this.waitForActiveShards = waitForActiveShards; - return this; +public record OpenIndexClusterStateUpdateRequest( + TimeValue masterNodeTimeout, + TimeValue ackTimeout, + ActiveShardCount waitForActiveShards, + Index[] indices +) { + public OpenIndexClusterStateUpdateRequest { + Objects.requireNonNull(masterNodeTimeout); + Objects.requireNonNull(ackTimeout); + Objects.requireNonNull(waitForActiveShards); + Objects.requireNonNull(indices); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexRequest.java index 4bb4578f24459..b3c8b7e9e795e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.open; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexRequestBuilder.java index 8784d7d924d75..4f5a27624cee3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.open; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponse.java index 605aab5ab02d9..6bb308df0f9ec 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.open; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java index bd89d124b3f76..ffaa50f7c0f29 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.open; @@ -89,23 +90,26 @@ protected void masterOperation( listener.onResponse(new OpenIndexResponse(true, true)); return; } - OpenIndexClusterStateUpdateRequest updateRequest = new OpenIndexClusterStateUpdateRequest().ackTimeout(request.ackTimeout()) - .masterNodeTimeout(request.masterNodeTimeout()) - .indices(concreteIndices) - .waitForActiveShards(request.waitForActiveShards()); - indexStateService.openIndices(updateRequest, new ActionListener<>() { + indexStateService.openIndices( + new OpenIndexClusterStateUpdateRequest( + request.masterNodeTimeout(), + request.ackTimeout(), + request.waitForActiveShards(), + concreteIndices + ), + new ActionListener<>() { + @Override + public void onResponse(ShardsAcknowledgedResponse response) { + listener.onResponse(new OpenIndexResponse(response.isAcknowledged(), response.isShardsAcknowledged())); + } - @Override - public void onResponse(ShardsAcknowledgedResponse response) { - listener.onResponse(new OpenIndexResponse(response.isAcknowledged(), response.isShardsAcknowledged())); + @Override + public void onFailure(Exception t) { + logger.debug(() -> "failed to open indices [" + Arrays.toString(concreteIndices) + "]", t); + listener.onFailure(t); + } } - - @Override - public void onFailure(Exception t) { - logger.debug(() -> "failed to open indices [" + Arrays.toString(concreteIndices) + "]", t); - listener.onFailure(t); - } - }); + ); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/package-info.java b/server/src/main/java/org/elasticsearch/action/admin/indices/package-info.java index 539d2cd33d963..7c73b08a2c8df 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/package-info.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/AddIndexBlockClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/AddIndexBlockClusterStateUpdateRequest.java index 94fea86b8df01..50bd3b37b4cb3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/AddIndexBlockClusterStateUpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/AddIndexBlockClusterStateUpdateRequest.java @@ -1,38 +1,33 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.readonly; -import org.elasticsearch.cluster.ack.IndicesClusterStateUpdateRequest; import org.elasticsearch.cluster.metadata.IndexMetadata.APIBlock; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.Index; + +import java.util.Objects; /** * Cluster state update request that allows to add a block to one or more indices */ -public class AddIndexBlockClusterStateUpdateRequest extends IndicesClusterStateUpdateRequest { - - private final APIBlock block; - private long taskId; - - public AddIndexBlockClusterStateUpdateRequest(final APIBlock block, final long taskId) { - this.block = block; - this.taskId = taskId; - } - - public long taskId() { - return taskId; - } - - public APIBlock getBlock() { - return block; - } - - public AddIndexBlockClusterStateUpdateRequest taskId(final long taskId) { - this.taskId = taskId; - return this; +public record AddIndexBlockClusterStateUpdateRequest( + TimeValue masterNodeTimeout, + TimeValue ackTimeout, + APIBlock block, + long taskId, + Index[] indices +) { + public AddIndexBlockClusterStateUpdateRequest { + Objects.requireNonNull(masterNodeTimeout); + Objects.requireNonNull(ackTimeout); + Objects.requireNonNull(block); + Objects.requireNonNull(indices); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/AddIndexBlockRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/AddIndexBlockRequest.java index 9331d7010a6e6..5f9bd6399fe7d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/AddIndexBlockRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/AddIndexBlockRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.readonly; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/AddIndexBlockRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/AddIndexBlockRequestBuilder.java index 4bcff8d1f6b16..ce3c7ba3c8d83 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/AddIndexBlockRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/AddIndexBlockRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.readonly; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/AddIndexBlockResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/AddIndexBlockResponse.java index 4605fb1a342ef..fef52c53f78f1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/AddIndexBlockResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/AddIndexBlockResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.readonly; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/TransportAddIndexBlockAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/TransportAddIndexBlockAction.java index f5b12dbe209ac..867cd80fb68d0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/TransportAddIndexBlockAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/TransportAddIndexBlockAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.readonly; @@ -101,13 +102,18 @@ protected void masterOperation( return; } - final AddIndexBlockClusterStateUpdateRequest addBlockRequest = new AddIndexBlockClusterStateUpdateRequest( - request.getBlock(), - task.getId() - ).ackTimeout(request.ackTimeout()).masterNodeTimeout(request.masterNodeTimeout()).indices(concreteIndices); - indexStateService.addIndexBlock(addBlockRequest, listener.delegateResponse((delegatedListener, t) -> { - logger.debug(() -> "failed to mark indices as readonly [" + Arrays.toString(concreteIndices) + "]", t); - delegatedListener.onFailure(t); - })); + indexStateService.addIndexBlock( + new AddIndexBlockClusterStateUpdateRequest( + request.masterNodeTimeout(), + request.ackTimeout(), + request.getBlock(), + task.getId(), + concreteIndices + ), + listener.delegateResponse((delegatedListener, t) -> { + logger.debug(() -> "failed to mark indices as readonly [" + Arrays.toString(concreteIndices) + "]", t); + delegatedListener.onFailure(t); + }) + ); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/TransportVerifyShardIndexBlockAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/TransportVerifyShardIndexBlockAction.java index 27234976f780b..1a7a536123ed4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/TransportVerifyShardIndexBlockAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/TransportVerifyShardIndexBlockAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.readonly; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryAction.java index 0f1ad96a6ad0f..6a05373b962db 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.recovery; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryRequest.java index a09c43fd6c4b7..595ffab2ad7bf 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.recovery; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryRequestBuilder.java index 1d559eca8c186..4f950f7c25514 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.recovery; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryResponse.java index 4e58fbab4bd7c..bdb7433f11427 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.recovery; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java index cec8e26ded364..4dbbb317cd1f8 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.recovery; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshAction.java index d484346e1b8fc..232f5f1c87e6c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.refresh; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshRequest.java index 1f703e59980d6..d6168d2228643 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.refresh; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshRequestBuilder.java index c503ff6ca6930..9a4245966643d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.refresh; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/ShardRefreshReplicaRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/ShardRefreshReplicaRequest.java index 7e72f6bcad545..e87732b2606cb 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/ShardRefreshReplicaRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/ShardRefreshReplicaRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.refresh; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java index a5d0ec36a3bfd..96be045daac7e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.refresh; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java index 5fa9ba6662c2e..cb667400240f0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.refresh; @@ -22,7 +23,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.injection.guice.Inject; @@ -119,27 +119,18 @@ public void onPrimaryOperationComplete( ActionListener listener ) { assert replicaRequest.primaryRefreshResult.refreshed() : "primary has not refreshed"; - boolean fastRefresh = IndexSettings.INDEX_FAST_REFRESH_SETTING.get( - clusterService.state().metadata().index(indexShardRoutingTable.shardId().getIndex()).getSettings() + UnpromotableShardRefreshRequest unpromotableReplicaRequest = new UnpromotableShardRefreshRequest( + indexShardRoutingTable, + replicaRequest.primaryRefreshResult.primaryTerm(), + replicaRequest.primaryRefreshResult.generation(), + false + ); + transportService.sendRequest( + transportService.getLocalNode(), + TransportUnpromotableShardRefreshAction.NAME, + unpromotableReplicaRequest, + new ActionListenerResponseHandler<>(listener.safeMap(r -> null), in -> ActionResponse.Empty.INSTANCE, refreshExecutor) ); - - // Indices marked with fast refresh do not rely on refreshing the unpromotables - if (fastRefresh) { - listener.onResponse(null); - } else { - UnpromotableShardRefreshRequest unpromotableReplicaRequest = new UnpromotableShardRefreshRequest( - indexShardRoutingTable, - replicaRequest.primaryRefreshResult.primaryTerm(), - replicaRequest.primaryRefreshResult.generation(), - false - ); - transportService.sendRequest( - transportService.getLocalNode(), - TransportUnpromotableShardRefreshAction.NAME, - unpromotableReplicaRequest, - new ActionListenerResponseHandler<>(listener.safeMap(r -> null), in -> ActionResponse.Empty.INSTANCE, refreshExecutor) - ); - } } } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportUnpromotableShardRefreshAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportUnpromotableShardRefreshAction.java index 29eaeb273a8a3..6c24ec2d17604 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportUnpromotableShardRefreshAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportUnpromotableShardRefreshAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.refresh; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/UnpromotableShardRefreshRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/UnpromotableShardRefreshRequest.java index 26d07b49ecef3..f0629bee5f72f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/UnpromotableShardRefreshRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/UnpromotableShardRefreshRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.refresh; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/package-info.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/package-info.java index 3c9df3ac57004..4e9f7016f3b9f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/package-info.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java index 224ea63150420..9c5b6097b11bd 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.resolve; @@ -14,13 +15,12 @@ import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.ValidateActions; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; -import org.elasticsearch.transport.RemoteClusterService; +import org.elasticsearch.transport.RemoteClusterAware; import java.io.IOException; import java.util.Arrays; @@ -165,10 +165,7 @@ public String getDescription() { boolean localIndicesPresent(String[] indices) { for (String index : indices) { - // ensure that `index` is a remote name and not a date math expression which includes ':' symbol - // since date math expression after evaluation should not contain ':' symbol - String indexExpression = IndexNameExpressionResolver.resolveDateMathExpression(index); - if (indexExpression.indexOf(RemoteClusterService.REMOTE_CLUSTER_INDEX_SEPARATOR) < 0) { + if (RemoteClusterAware.isRemoteIndexName(index) == false) { return true; } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionResponse.java index 892b34df2b863..ba2666a374a58 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.resolve; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterInfo.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterInfo.java index dc2416a1b1baa..b539c87adc484 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterInfo.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterInfo.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.resolve; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java index 1e5036a9bfc63..f5c100b7884bb 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.resolve; @@ -24,6 +25,7 @@ import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; @@ -564,8 +566,8 @@ static void resolveIndices( if (names.length == 1 && (Metadata.ALL.equals(names[0]) || Regex.isMatchAllPattern(names[0]))) { names = new String[] { "**" }; } - Set resolvedIndexAbstractions = resolver.resolveExpressions(clusterState, indicesOptions, true, names); - for (String s : resolvedIndexAbstractions) { + Set resolvedIndexAbstractions = resolver.resolveExpressions(clusterState, indicesOptions, true, names); + for (ResolvedExpression s : resolvedIndexAbstractions) { enrichIndexAbstraction(clusterState, s, indices, aliases, dataStreams); } indices.sort(Comparator.comparing(ResolvedIndexAbstraction::getName)); @@ -596,12 +598,12 @@ private static void mergeResults( private static void enrichIndexAbstraction( ClusterState clusterState, - String indexAbstraction, + ResolvedExpression indexAbstraction, List indices, List aliases, List dataStreams ) { - IndexAbstraction ia = clusterState.metadata().getIndicesLookup().get(indexAbstraction); + IndexAbstraction ia = clusterState.metadata().getIndicesLookup().get(indexAbstraction.resource()); if (ia != null) { switch (ia.getType()) { case CONCRETE_INDEX -> { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java index 1ac7def758ab5..e3e737595cac6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.resolve; @@ -34,9 +35,6 @@ import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.ConnectTransportException; -import org.elasticsearch.transport.NoSeedNodeLeftException; -import org.elasticsearch.transport.NoSuchRemoteClusterException; import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.transport.RemoteClusterService; import org.elasticsearch.transport.TransportService; @@ -143,7 +141,7 @@ protected void doExecuteForked(Task task, ResolveClusterActionRequest request, A RemoteClusterClient remoteClusterClient = remoteClusterService.getRemoteClusterClient( clusterAlias, searchCoordinationExecutor, - RemoteClusterService.DisconnectedStrategy.RECONNECT_IF_DISCONNECTED + RemoteClusterService.DisconnectedStrategy.FAIL_IF_DISCONNECTED ); var remoteRequest = new ResolveClusterActionRequest(originalIndices.indices(), request.indicesOptions()); // allow cancellation requests to propagate to remote clusters @@ -171,7 +169,7 @@ public void onFailure(Exception failure) { releaseResourcesOnCancel(clusterInfoMap); return; } - if (notConnectedError(failure)) { + if (ExceptionsHelper.isRemoteUnavailableException((failure))) { clusterInfoMap.put(clusterAlias, new ResolveClusterInfo(false, skipUnavailable)); } else if (ExceptionsHelper.unwrap( failure, @@ -245,27 +243,6 @@ public void onFailure(Exception e) { } } - /** - * Checks the exception against a known list of exceptions that indicate a remote cluster - * cannot be connected to. - */ - private boolean notConnectedError(Exception e) { - if (e instanceof ConnectTransportException || e instanceof NoSuchRemoteClusterException) { - return true; - } - if (e instanceof IllegalStateException && e.getMessage().contains("Unable to open any connections")) { - return true; - } - Throwable ill = ExceptionsHelper.unwrap(e, IllegalArgumentException.class); - if (ill != null && ill.getMessage().contains("unknown host")) { - return true; - } - if (ExceptionsHelper.unwrap(e, NoSeedNodeLeftException.class) != null) { - return true; - } - return false; - } - /** * Checks whether the local cluster has any matching indices (non-closed), aliases or data streams for * the index expression captured in localIndices. diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java index b4a466dc9aa1e..b018b872c930e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.rollover; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java index 65b768a1c629f..a677897d79633 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.rollover; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxAgeCondition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxAgeCondition.java index a2c2456f788e0..55204676267ab 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxAgeCondition.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxAgeCondition.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.rollover; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxDocsCondition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxDocsCondition.java index 4bb64c451b074..650b7c90340a8 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxDocsCondition.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxDocsCondition.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.rollover; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxPrimaryShardDocsCondition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxPrimaryShardDocsCondition.java index 40579b1acf361..778a200f34211 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxPrimaryShardDocsCondition.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxPrimaryShardDocsCondition.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.rollover; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxPrimaryShardSizeCondition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxPrimaryShardSizeCondition.java index a17c6daf87988..9cc2762c64f55 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxPrimaryShardSizeCondition.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxPrimaryShardSizeCondition.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.rollover; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxSizeCondition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxSizeCondition.java index 9dda96c600a53..c0e29c67fc88c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxSizeCondition.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxSizeCondition.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.rollover; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java index b8d975f82980d..0c22a17bb1f6b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.rollover; @@ -411,7 +412,7 @@ yield new DataStreamAutoShardingEvent( dataStream.rollover( indexMetadata.getIndex(), newGeneration, - metadata.isTimeSeriesTemplate(templateV2), + metadata.retrieveIndexModeFromTemplate(templateV2), dataStreamAutoShardingEvent ) ); @@ -547,11 +548,7 @@ static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest( if (settings != null) { b.put(settings); } - return new CreateIndexClusterStateUpdateRequest(cause, targetIndexName, providedIndexName).ackTimeout( - createIndexRequest.ackTimeout() - ) - .masterNodeTimeout(createIndexRequest.masterNodeTimeout()) - .settings(b.build()) + return new CreateIndexClusterStateUpdateRequest(cause, targetIndexName, providedIndexName).settings(b.build()) .aliases(createIndexRequest.aliases()) .waitForActiveShards(ActiveShardCount.NONE) // not waiting for shards here, will wait on the alias switch operation .mappings(createIndexRequest.mappings()) @@ -660,7 +657,7 @@ static void validate( } if ((request.settings().equals(Settings.EMPTY) == false) || (request.aliases().size() > 0) - || (request.mappings().equals("{}") == false)) { + || (request.mappings().equals(CreateIndexRequest.EMPTY_MAPPINGS) == false)) { throw new IllegalArgumentException( "aliases, mappings, and index settings may not be specified when rolling over a data stream" ); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinAgeCondition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinAgeCondition.java index 5f55d01a0ab43..8555033f64040 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinAgeCondition.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinAgeCondition.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.rollover; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinDocsCondition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinDocsCondition.java index 28af3df8c7e22..0b6ca8a7dcd58 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinDocsCondition.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinDocsCondition.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.rollover; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinPrimaryShardDocsCondition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinPrimaryShardDocsCondition.java index c89b34345db8c..7f03ad22b6e24 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinPrimaryShardDocsCondition.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinPrimaryShardDocsCondition.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.rollover; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinPrimaryShardSizeCondition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinPrimaryShardSizeCondition.java index 7ef63186c76a8..13634a3250225 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinPrimaryShardSizeCondition.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinPrimaryShardSizeCondition.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.rollover; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinSizeCondition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinSizeCondition.java index 318f385842707..4feb4cdabf0cf 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinSizeCondition.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinSizeCondition.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.rollover; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/OptimalShardCountCondition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/OptimalShardCountCondition.java index acd26f2984c99..3183fc3eafeed 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/OptimalShardCountCondition.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/OptimalShardCountCondition.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.rollover; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverAction.java index 84ca69bcde77d..c6a45f4b34675 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.rollover; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverConditions.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverConditions.java index d7cd7459d4821..7fd4c4f1c597c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverConditions.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverConditions.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.rollover; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverConfiguration.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverConfiguration.java index 5e0ca49c138bb..6fa9cd909ba47 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverConfiguration.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverConfiguration.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.rollover; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverInfo.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverInfo.java index 318d4b7eecb54..75aa963f8a140 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverInfo.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverInfo.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.rollover; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java index db4fad99d4f48..4240f6cb1d787 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.rollover; @@ -12,6 +13,7 @@ import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.support.ActiveShardCount; +import org.elasticsearch.action.support.IndexComponentSelector; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.cluster.metadata.DataStream; @@ -146,8 +148,8 @@ public ActionRequestValidationException validate() { ); } - var failureStoreOptions = indicesOptions.failureStoreOptions(); - if (failureStoreOptions.includeRegularIndices() && failureStoreOptions.includeFailureIndices()) { + var selector = indicesOptions.selectorOptions().defaultSelector(); + if (selector == IndexComponentSelector.ALL_APPLICABLE) { validationException = addValidationError( "rollover cannot be applied to both regular and failure indices at the same time", validationException @@ -187,7 +189,7 @@ public IndicesOptions indicesOptions() { * @return true of the rollover request targets the failure store, false otherwise. */ public boolean targetsFailureStore() { - return DataStream.isFailureStoreFeatureFlagEnabled() && indicesOptions.failureStoreOptions().includeFailureIndices(); + return DataStream.isFailureStoreFeatureFlagEnabled() && indicesOptions.includeFailureIndices(); } public void setIndicesOptions(IndicesOptions indicesOptions) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java index 5cbb3f4161298..ccb0f21f81b2e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.rollover; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java index 04b9f6498a3a9..72c568a3995ab 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.rollover; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java index c997795bb3b89..c5c874f9bcddf 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.rollover; @@ -149,7 +150,7 @@ protected ClusterBlockException checkBlock(RolloverRequest request, ClusterState .matchClosed(request.indicesOptions().expandWildcardsClosed()) .build(), IndicesOptions.GatekeeperOptions.DEFAULT, - request.indicesOptions().failureStoreOptions() + request.indicesOptions().selectorOptions() ); return state.blocks() @@ -246,7 +247,7 @@ protected void masterOperation( IndicesOptions.ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS, IndicesOptions.WildcardOptions.builder().matchClosed(true).allowEmptyExpressions(false).build(), IndicesOptions.GatekeeperOptions.DEFAULT, - rolloverRequest.indicesOptions().failureStoreOptions() + rolloverRequest.indicesOptions().selectorOptions() ); IndicesStatsRequest statsRequest = new IndicesStatsRequest().indices(rolloverRequest.getRolloverTarget()) .clear() diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndexSegments.java b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndexSegments.java index 3107727f5174d..2e062680572d5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndexSegments.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndexSegments.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.segments; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndexShardSegments.java b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndexShardSegments.java index 9c100fd7e2ee9..b751ecc9a8b3a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndexShardSegments.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndexShardSegments.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.segments; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java index 429ebe365bbe1..1071f120f929e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.segments; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsAction.java index 86df6b6d9a755..4e7d6cdc0d916 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.segments; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequest.java index 290396cdf8384..bdacfb0ab642d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.segments; @@ -33,7 +34,7 @@ public IndicesSegmentsRequest(StreamInput in) throws IOException { if (in.getTransportVersion().before(TransportVersions.V_8_0_0)) { in.readBoolean(); // old 'verbose' option, since removed } - if (in.getTransportVersion().onOrAfter(TransportVersions.INDEX_SEGMENTS_VECTOR_FORMATS)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { this.includeVectorFormatsInfo = in.readBoolean(); } } @@ -58,7 +59,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().before(TransportVersions.V_8_0_0)) { out.writeBoolean(false); } - if (out.getTransportVersion().onOrAfter(TransportVersions.INDEX_SEGMENTS_VECTOR_FORMATS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { out.writeBoolean(includeVectorFormatsInfo); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestBuilder.java index 7eb4d296a228c..75c14c1679d5e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.segments; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/ShardSegments.java b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/ShardSegments.java index bdb94c7b11f02..d29e5dc45c6bd 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/ShardSegments.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/ShardSegments.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.segments; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java index f5a79eec7ab33..be0bf76673f2f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.segments; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsAction.java index 4522519952ea8..369cddff18b96 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.settings.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsRequest.java index 42ff256579984..c0ee7d7407614 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.settings.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsRequestBuilder.java index 166e5cb497ad6..bd68ba03cafdd 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.settings.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsResponse.java index 8a106d1b43d3e..e7b8d40451d04 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.settings.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java index 7396e1d383bd5..a846526bf706e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.settings.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java index 25a22ae52ac61..1e7f32641b86f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.settings.put; @@ -123,19 +124,24 @@ protected void masterOperation( return; } - UpdateSettingsClusterStateUpdateRequest clusterStateUpdateRequest = new UpdateSettingsClusterStateUpdateRequest().indices( - concreteIndices - ) - .settings(requestSettings) - .setPreserveExisting(request.isPreserveExisting()) - .reopenShards(request.reopen()) - .ackTimeout(request.ackTimeout()) - .masterNodeTimeout(request.masterNodeTimeout()); - - updateSettingsService.updateSettings(clusterStateUpdateRequest, listener.delegateResponse((l, e) -> { - logger.debug(() -> "failed to update settings on indices [" + Arrays.toString(concreteIndices) + "]", e); - l.onFailure(e); - })); + updateSettingsService.updateSettings( + new UpdateSettingsClusterStateUpdateRequest( + request.masterNodeTimeout(), + request.ackTimeout(), + requestSettings, + request.isPreserveExisting() + ? UpdateSettingsClusterStateUpdateRequest.OnExisting.PRESERVE + : UpdateSettingsClusterStateUpdateRequest.OnExisting.OVERWRITE, + request.reopen() + ? UpdateSettingsClusterStateUpdateRequest.OnStaticSetting.REOPEN_INDICES + : UpdateSettingsClusterStateUpdateRequest.OnStaticSetting.REJECT, + concreteIndices + ), + listener.delegateResponse((l, e) -> { + logger.debug(() -> "failed to update settings on indices [" + Arrays.toString(concreteIndices) + "]", e); + l.onFailure(e); + }) + ); } /** diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsClusterStateUpdateRequest.java index 99a43c6594c62..fe8573da5fb68 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsClusterStateUpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsClusterStateUpdateRequest.java @@ -1,77 +1,68 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.settings.put; -import org.elasticsearch.cluster.ack.IndicesClusterStateUpdateRequest; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.Index; -import java.util.Arrays; +import java.util.Objects; /** * Cluster state update request that allows to update settings for some indices */ -public class UpdateSettingsClusterStateUpdateRequest extends IndicesClusterStateUpdateRequest { - - private Settings settings; - - private boolean preserveExisting = false; - - private boolean reopenShards = false; - - /** - * Returns true iff the settings update should only add but not update settings. If the setting already exists - * it should not be overwritten by this update. The default is false - */ - public boolean isPreserveExisting() { - return preserveExisting; - } +public record UpdateSettingsClusterStateUpdateRequest( + TimeValue masterNodeTimeout, + TimeValue ackTimeout, + Settings settings, + OnExisting onExisting, + OnStaticSetting onStaticSetting, + Index... indices +) { /** - * Returns true if non-dynamic setting updates should go through, by automatically unassigning shards in the same cluster - * state change as the setting update. The shards will be automatically reassigned after the cluster state update is made. The - * default is false. + * Specifies the behaviour of an update-settings action on existing settings. */ - public boolean reopenShards() { - return reopenShards; - } + public enum OnExisting { + /** + * Update all the specified settings, overwriting any settings which already exist. This is the API default. + */ + OVERWRITE, - public UpdateSettingsClusterStateUpdateRequest reopenShards(boolean reopenShards) { - this.reopenShards = reopenShards; - return this; + /** + * Only add new settings, preserving the values of any settings which are already set and ignoring the new values specified in the + * request. + */ + PRESERVE } /** - * Iff set to true this settings update will only add settings not already set on an index. Existing settings remain - * unchanged. + * Specifies the behaviour of an update-settings action which is trying to adjust a non-dynamic setting. */ - public UpdateSettingsClusterStateUpdateRequest setPreserveExisting(boolean preserveExisting) { - this.preserveExisting = preserveExisting; - return this; - } + public enum OnStaticSetting { + /** + * Reject attempts to update non-dynamic settings on open indices. This is the API default. + */ + REJECT, - /** - * Returns the {@link Settings} to update - */ - public Settings settings() { - return settings; - } - - /** - * Sets the {@link Settings} to update - */ - public UpdateSettingsClusterStateUpdateRequest settings(Settings settings) { - this.settings = settings; - return this; + /** + * Automatically close and reopen the shards of any open indices when updating a non-dynamic setting, forcing the shard to + * reinitialize from scratch. + */ + REOPEN_INDICES } - @Override - public String toString() { - return Arrays.toString(indices()) + settings; + public UpdateSettingsClusterStateUpdateRequest { + Objects.requireNonNull(masterNodeTimeout); + Objects.requireNonNull(ackTimeout); + Objects.requireNonNull(settings); + Objects.requireNonNull(indices); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java index c3e87f2f54cf0..fe5d688378bfe 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.settings.put; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java index 634223b22d37a..b6c9852d2dbfe 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.settings.put; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresRequest.java index 8cf2427e91c15..3ab427dbecdf7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.shards; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java index fc1ef9e011e62..60a8ae5ad6371 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.shards; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java index 08399cdd3f9cb..76aea70a65cbd 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.shards; @@ -190,7 +191,6 @@ void run() { Iterators.flatMap(Iterators.forArray(concreteIndices), this::getIndexIterator), this::doShardRequest, maxConcurrentShardRequests, - () -> {}, outerListener::close ); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeAction.java index e89ccac18ff2f..3c8ee569670dc 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.shrink; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeNumberOfShardsCalculator.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeNumberOfShardsCalculator.java index 076841e3efadc..d6207bc3151b4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeNumberOfShardsCalculator.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeNumberOfShardsCalculator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.shrink; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java index ef709fc4457a7..21013525fcc8c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.shrink; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestBuilder.java index a18de15037e49..8d14ca18c7c20 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.shrink; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeType.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeType.java index 8ccac9c221b01..a068ac3e727f0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeType.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeType.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.shrink; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java index 0fe8ee9670f00..4a6311e6400aa 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.shrink; @@ -135,6 +136,9 @@ protected void masterOperation( return; } createIndexService.createIndex( + resizeRequest.masterNodeTimeout(), + resizeRequest.ackTimeout(), + resizeRequest.ackTimeout(), updateRequest, delegatedListener.map( response -> new CreateIndexResponse( @@ -233,8 +237,6 @@ static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest( // mappings are updated on the node when creating in the shards, this prevents race-conditions since all mapping must be // applied once we took the snapshot and if somebody messes things up and switches the index read/write and adds docs we // miss the mappings for everything is corrupted and hard to debug - .ackTimeout(targetIndex.ackTimeout()) - .masterNodeTimeout(targetIndex.masterNodeTimeout()) .settings(targetIndex.settings()) .aliases(targetIndex.aliases()) .waitForActiveShards(targetIndex.waitForActiveShards()) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java index 2596e62c85259..1351674fffbfa 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.stats; @@ -46,12 +47,11 @@ import java.io.IOException; import java.util.Objects; -import static org.elasticsearch.TransportVersions.VERSION_SUPPORTING_SPARSE_VECTOR_STATS; - public class CommonStats implements Writeable, ToXContentFragment { private static final TransportVersion VERSION_SUPPORTING_NODE_MAPPINGS = TransportVersions.V_8_5_0; private static final TransportVersion VERSION_SUPPORTING_DENSE_VECTOR_STATS = TransportVersions.V_8_10_X; + private static final TransportVersion VERSION_SUPPORTING_SPARSE_VECTOR_STATS = TransportVersions.V_8_15_0; @Nullable public DocsStats docs; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java index 31dc6a744a757..ae6e1985d56c7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.stats; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageShardResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageShardResponse.java index 4018a89523594..347376a918d4c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageShardResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageShardResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.stats; @@ -68,7 +69,7 @@ public FieldUsageStats getStats() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(Fields.TRACKING_ID, trackingId); - builder.timeField(Fields.TRACKING_STARTED_AT_MILLIS, Fields.TRACKING_STARTED_AT, trackingStartTime); + builder.timestampFieldsFromUnixEpochMillis(Fields.TRACKING_STARTED_AT_MILLIS, Fields.TRACKING_STARTED_AT, trackingStartTime); builder.startObject(Fields.ROUTING) .field(Fields.STATE, shardRouting.state()) .field(Fields.PRIMARY, shardRouting.primary()) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsAction.java index e7d55aba26605..e4250c7a32b8d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.stats; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsRequest.java index 795241fe59b21..ea3d120569b8c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.stats; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponse.java index adb892b5561a3..50b46aa5e284f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.stats; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndexShardStats.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndexShardStats.java index 965d78a1cde2f..80bc0b1d6848a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndexShardStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndexShardStats.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.stats; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndexStats.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndexStats.java index 5affc6a9f39e4..6106e620521f7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndexStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndexStats.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.stats; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsAction.java index 08ed9df5d2455..61691763f9a63 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.stats; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java index 12c26f3b10ca4..d67516b0d1e62 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.stats; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java index 0c20869eeb906..8da95a95444ca 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.stats; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java index 1ea0820492b04..65f7e1969ea69 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.stats; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java index 477a0bd910719..dab46df50abc5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.stats; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportFieldUsageAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportFieldUsageAction.java index db49c199e05b0..d660f94d2c829 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportFieldUsageAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportFieldUsageAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.stats; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java index 4c042570704d9..d3274dab8a386 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.stats; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateRequest.java index 3c2416200ce61..4067c2dfafa13 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.template.delete; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateRequestBuilder.java index a1f5c6cdb4ebb..f249e7428fa74 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.template.delete; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComponentTemplateAction.java index 3fcdd96feaf5f..824f93e68b6c4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComponentTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComponentTemplateAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.template.delete; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java index 2c6a0a2945abb..daaef97d49732 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.template.delete; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java index 757930bc0505b..e7632ce9bfbb9 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.template.delete; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java index f0552cc3226f5..f729455edcc24 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.template.get; @@ -130,8 +131,7 @@ public Response(StreamInput in) throws IOException { } else { rolloverConfiguration = null; } - if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) - && in.getTransportVersion().before(TransportVersions.REMOVE_GLOBAL_RETENTION_FROM_TEMPLATES)) { + if (in.getTransportVersion().between(TransportVersions.V_8_14_0, TransportVersions.V_8_16_0)) { in.readOptionalWriteable(DataStreamGlobalRetention::read); } } @@ -189,8 +189,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { out.writeOptionalWriteable(rolloverConfiguration); } - if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) - && out.getTransportVersion().before(TransportVersions.REMOVE_GLOBAL_RETENTION_FROM_TEMPLATES)) { + if (out.getTransportVersion().between(TransportVersions.V_8_14_0, TransportVersions.V_8_16_0)) { out.writeOptionalWriteable(null); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java index ba07c87e753e6..67f87476ea6a5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.template.get; @@ -131,8 +132,7 @@ public Response(StreamInput in) throws IOException { } else { rolloverConfiguration = null; } - if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) - && in.getTransportVersion().before(TransportVersions.REMOVE_GLOBAL_RETENTION_FROM_TEMPLATES)) { + if (in.getTransportVersion().between(TransportVersions.V_8_14_0, TransportVersions.V_8_16_0)) { in.readOptionalWriteable(DataStreamGlobalRetention::read); } } @@ -190,8 +190,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { out.writeOptionalWriteable(rolloverConfiguration); } - if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) - && out.getTransportVersion().before(TransportVersions.REMOVE_GLOBAL_RETENTION_FROM_TEMPLATES)) { + if (out.getTransportVersion().between(TransportVersions.V_8_14_0, TransportVersions.V_8_16_0)) { out.writeOptionalWriteable(null); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesAction.java index 67198ccf0c528..5c39465ab20b7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.template.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesRequest.java index 19c89b0186733..e3f765eb8ae88 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.template.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesRequestBuilder.java index 63ec0ae677e9b..97aabbd4631b7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.template.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java index 9c34dbac79357..fa3ac8ae720c2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.template.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java index fcc053b8181fa..d04ccd39be04b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.template.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComposableIndexTemplateAction.java index e2ce172a1bf0b..608e7529d34bf 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComposableIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComposableIndexTemplateAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.template.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java index 40815e87d26f7..99763d4d76dc6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.template.get; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateAction.java index 3ccc34e4693a9..e053480f4884c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.template.post; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java index dfbebfb0e91c8..ce29d65ececf9 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.template.post; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java index a27defd2c655c..a521dac60e96a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.template.post; @@ -12,6 +13,7 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration; import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; +import org.elasticsearch.cluster.metadata.ResettableValue; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -81,8 +83,7 @@ public SimulateIndexTemplateResponse(StreamInput in) throws IOException { rolloverConfiguration = in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X) ? in.readOptionalWriteable(RolloverConfiguration::new) : null; - if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) - && in.getTransportVersion().before(TransportVersions.REMOVE_GLOBAL_RETENTION_FROM_TEMPLATES)) { + if (in.getTransportVersion().between(TransportVersions.V_8_14_0, TransportVersions.V_8_16_0)) { in.readOptionalWriteable(DataStreamGlobalRetention::read); } } @@ -103,8 +104,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { out.writeOptionalWriteable(rolloverConfiguration); } - if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) - && out.getTransportVersion().before(TransportVersions.REMOVE_GLOBAL_RETENTION_FROM_TEMPLATES)) { + if (out.getTransportVersion().between(TransportVersions.V_8_14_0, TransportVersions.V_8_16_0)) { out.writeOptionalWriteable(null); } } @@ -112,9 +112,9 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - if (this.resolvedTemplate != null) { + if (resolvedTemplate != null) { builder.field(TEMPLATE.getPreferredName()); - this.resolvedTemplate.toXContent(builder, params, rolloverConfiguration); + resolvedTemplate.toXContent(builder, ResettableValue.hideResetValues(params), rolloverConfiguration); } if (this.overlappingTemplates != null) { builder.startArray(OVERLAPPING.getPreferredName()); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateAction.java index 39440c09fa8ec..75cc72416a854 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.template.post; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index cd8ffea3d3824..5f98852148ed4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.template.post; @@ -47,6 +48,7 @@ import java.time.Instant; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; @@ -59,6 +61,7 @@ import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findConflictingV1Templates; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findConflictingV2Templates; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV2Template; +import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.resolveDataStreamOptions; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.resolveLifecycle; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.resolveSettings; @@ -269,20 +272,35 @@ public static Template resolveTemplate( // First apply settings sourced from index settings providers final var now = Instant.now(); Settings.Builder additionalSettings = Settings.builder(); + Set overrulingSettings = new HashSet<>(); for (var provider : indexSettingProviders) { Settings result = provider.getAdditionalIndexSettings( indexName, template.getDataStreamTemplate() != null ? indexName : null, - template.getDataStreamTemplate() != null && metadata.isTimeSeriesTemplate(template), + metadata.retrieveIndexModeFromTemplate(template), simulatedState.getMetadata(), now, templateSettings, mappings ); + MetadataCreateIndexService.validateAdditionalSettings(provider, result, additionalSettings); dummySettings.put(result); additionalSettings.put(result); + if (provider.overrulesTemplateAndRequestSettings()) { + overrulingSettings.addAll(result.keySet()); + } + } + + if (overrulingSettings.isEmpty() == false) { + // Filter any conflicting settings from overruling providers, to avoid overwriting their values from templates. + final Settings.Builder filtered = Settings.builder().put(templateSettings); + for (String setting : overrulingSettings) { + filtered.remove(setting); + } + templateSettings = filtered.build(); } - // Then apply settings resolved from templates: + + // Apply settings resolved from templates. dummySettings.put(templateSettings); final IndexMetadata indexMetadata = IndexMetadata.builder(indexName) @@ -331,7 +349,13 @@ public static Template resolveTemplate( if (template.getDataStreamTemplate() != null && lifecycle == null && isDslOnlyMode) { lifecycle = DataStreamLifecycle.DEFAULT; } - return new Template(settings, mergedMapping, aliasesByName, lifecycle); + return new Template( + settings, + mergedMapping, + aliasesByName, + lifecycle, + resolveDataStreamOptions(simulatedState.metadata(), matchingTemplate) + ); } private static IndexLongFieldRange getEventIngestedRange(String indexName, ClusterState simulatedState) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java index ead00dc858a47..30bbad0b57df0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.template.post; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutComponentTemplateAction.java index ebf1e9e74b793..2c34359dabbad 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutComponentTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutComponentTemplateAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.template.put; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java index 6ef887847c270..0f11cb45e08d5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.template.put; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java index d0b93d28ad157..9aa70f036c028 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.template.put; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java index 38e9918aae853..fb5267ba87b75 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.template.put; @@ -77,7 +78,7 @@ public static ComponentTemplate normalizeComponentTemplate( Settings.Builder builder = Settings.builder().put(template.settings()).normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX); Settings settings = builder.build(); indexScopedSettings.validate(settings, true); - template = new Template(settings, template.mappings(), template.aliases(), template.lifecycle()); + template = Template.builder(template).settings(settings).build(); componentTemplate = new ComponentTemplate( template, componentTemplate.version(), diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java index 34a0d2e358acf..097c17a98b728 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.template.put; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java index 8611d2688f5d2..a32fc521455a6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.template.put; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateAction.java index abd2c9f6872e8..804267c2a62b5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.template.reservedstate; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/QueryExplanation.java b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/QueryExplanation.java index e263e4c1451dd..932606d89864d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/QueryExplanation.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/QueryExplanation.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.validate.query; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java index 9ccebdc5b8ca8..f41cb6fbc0306 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.validate.query; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryResponse.java index d97cad3c2272d..15c932fc3ea11 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.validate.query; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java index 5e0d9c9338e0c..e01f364712676 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.validate.query; @@ -20,6 +21,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.cluster.routing.ShardRouting; @@ -132,7 +134,7 @@ protected void doExecute(Task task, ValidateQueryRequest request, ActionListener @Override protected ShardValidateQueryRequest newShardRequest(int numShards, ShardRouting shard, ValidateQueryRequest request) { final ClusterState clusterState = clusterService.state(); - final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(clusterState, request.indices()); + final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(clusterState, request.indices()); final AliasFilter aliasFilter = searchService.buildAliasFilter(clusterState, shard.getIndexName(), indicesAndAliases); return new ShardValidateQueryRequest(shard.shardId(), aliasFilter, request); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryAction.java index 323be06664f4c..d538022905d8f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.validate.query; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequest.java index 0505f41b27599..f30206c1d238a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.validate.query; @@ -31,7 +32,7 @@ */ public final class ValidateQueryRequest extends BroadcastRequest implements ToXContentObject { - public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.fromOptions(false, false, true, false); + public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpenAndForbidClosed(); private QueryBuilder query = new MatchAllQueryBuilder(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequestBuilder.java index d2a4f25dc6b3d..964d4f1206397 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.validate.query; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryResponse.java index aaa06908f72f0..947fa1b9a05c3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.admin.indices.validate.query; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/package-info.java b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/package-info.java index 9aad578a44111..f7243608765c9 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/package-info.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/server/src/main/java/org/elasticsearch/action/admin/package-info.java b/server/src/main/java/org/elasticsearch/action/admin/package-info.java index cfa12ccd38904..c26bede2c6fd3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/package-info.java +++ b/server/src/main/java/org/elasticsearch/action/admin/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BackoffPolicy.java b/server/src/main/java/org/elasticsearch/action/bulk/BackoffPolicy.java index 5c23a84d93963..b9f89d4a65a01 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BackoffPolicy.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BackoffPolicy.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java index b8dd0d1fe415e..998a3ada5d157 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; @@ -13,11 +14,24 @@ import java.util.Set; +import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS; +import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_IGNORED_FIELDS; +import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_INDEX_TEMPLATE_SUBSTITUTIONS; +import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_ADDITION; import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_VALIDATION; import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_VALIDATION_TEMPLATES; +import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_SUPPORT_NON_TEMPLATE_MAPPING; public class BulkFeatures implements FeatureSpecification { public Set getFeatures() { - return Set.of(SIMULATE_MAPPING_VALIDATION, SIMULATE_MAPPING_VALIDATION_TEMPLATES); + return Set.of( + SIMULATE_MAPPING_VALIDATION, + SIMULATE_MAPPING_VALIDATION_TEMPLATES, + SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS, + SIMULATE_INDEX_TEMPLATE_SUBSTITUTIONS, + SIMULATE_MAPPING_ADDITION, + SIMULATE_SUPPORT_NON_TEMPLATE_MAPPING, + SIMULATE_IGNORED_FIELDS + ); } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java index 7c1304f92eefd..6a85a7752b900 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java index d3e550eaf05b3..6ea2045315c34 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; @@ -62,6 +63,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(_ID, failure.getId()); builder.field(STATUS, failure.getStatus().getStatus()); + failure.getFailureStoreStatus().toXContent(builder, params); builder.startObject(ERROR); ElasticsearchException.generateThrowableXContent(builder, params, failure.getCause()); builder.endObject(); @@ -87,6 +89,7 @@ public static class Failure implements Writeable, ToXContentFragment { private final long seqNo; private final long term; private final boolean aborted; + private IndexDocFailureStoreStatus failureStoreStatus; /** * For write failures before operation was assigned a sequence number. @@ -102,7 +105,27 @@ public Failure(String index, String id, Exception cause) { ExceptionsHelper.status(cause), SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, - false + false, + IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN + ); + } + + /** + * For write failures before operation was assigned a sequence number. + * + * use @{link {@link #Failure(String, String, Exception, long, long)}} + * to record operation sequence no with failure + */ + public Failure(String index, String id, Exception cause, IndexDocFailureStoreStatus failureStoreStatus) { + this( + index, + id, + cause, + ExceptionsHelper.status(cause), + SequenceNumbers.UNASSIGNED_SEQ_NO, + SequenceNumbers.UNASSIGNED_PRIMARY_TERM, + false, + failureStoreStatus ); } @@ -114,20 +137,48 @@ public Failure(String index, String id, Exception cause, boolean aborted) { ExceptionsHelper.status(cause), SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, - aborted + aborted, + IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN ); } public Failure(String index, String id, Exception cause, RestStatus status) { - this(index, id, cause, status, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, false); + this( + index, + id, + cause, + status, + SequenceNumbers.UNASSIGNED_SEQ_NO, + SequenceNumbers.UNASSIGNED_PRIMARY_TERM, + false, + IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN + ); } /** For write failures after operation was assigned a sequence number. */ public Failure(String index, String id, Exception cause, long seqNo, long term) { - this(index, id, cause, ExceptionsHelper.status(cause), seqNo, term, false); + this( + index, + id, + cause, + ExceptionsHelper.status(cause), + seqNo, + term, + false, + IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN + ); } - private Failure(String index, String id, Exception cause, RestStatus status, long seqNo, long term, boolean aborted) { + private Failure( + String index, + String id, + Exception cause, + RestStatus status, + long seqNo, + long term, + boolean aborted, + IndexDocFailureStoreStatus failureStoreStatus + ) { this.index = index; this.id = id; this.cause = cause; @@ -135,6 +186,7 @@ private Failure(String index, String id, Exception cause, RestStatus status, lon this.seqNo = seqNo; this.term = term; this.aborted = aborted; + this.failureStoreStatus = failureStoreStatus; } /** @@ -153,6 +205,11 @@ public Failure(StreamInput in) throws IOException { seqNo = in.readZLong(); term = in.readVLong(); aborted = in.readBoolean(); + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { + failureStoreStatus = IndexDocFailureStoreStatus.read(in); + } else { + failureStoreStatus = IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN; + } } @Override @@ -166,6 +223,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeZLong(seqNo); out.writeVLong(term); out.writeBoolean(aborted); + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { + failureStoreStatus.writeTo(out); + } } /** @@ -230,6 +290,14 @@ public boolean isAborted() { return aborted; } + public IndexDocFailureStoreStatus getFailureStoreStatus() { + return failureStoreStatus; + } + + public void setFailureStoreStatus(IndexDocFailureStoreStatus failureStoreStatus) { + this.failureStoreStatus = failureStoreStatus; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field(INDEX_FIELD, index); @@ -243,6 +311,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws ElasticsearchException.generateThrowableXContent(builder, params, cause); builder.endObject(); builder.field(STATUS_FIELD, status.getStatus()); + failureStoreStatus.toXContent(builder, params); return builder; } @@ -340,6 +409,14 @@ public long getVersion() { return response.getVersion(); } + public IndexDocFailureStoreStatus getFailureStoreStatus() { + if (response != null) { + return response.getFailureStoreStatus(); + } else { + return failure.getFailureStoreStatus(); + } + } + /** * The actual response ({@link IndexResponse} or {@link DeleteResponse}). {@code null} in * case of failure. diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java index 64e0b80aca74a..6e5735ccfba98 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; @@ -22,6 +23,7 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.RefCountingRunnable; import org.elasticsearch.client.internal.OriginSettingClient; @@ -31,6 +33,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -85,7 +88,6 @@ final class BulkOperation extends ActionRunnable { private final ConcurrentLinkedQueue failureStoreRedirects = new ConcurrentLinkedQueue<>(); private final long startTimeNanos; private final ClusterStateObserver observer; - private final Map indicesThatCannotBeCreated; private final Executor executor; private final LongSupplier relativeTimeProvider; private final FailureStoreDocumentConverter failureStoreDocumentConverter; @@ -94,7 +96,9 @@ final class BulkOperation extends ActionRunnable { private final OriginSettingClient rolloverClient; private final Set failureStoresToBeRolledOver = ConcurrentCollections.newConcurrentSet(); private final Set failedRolloverRequests = ConcurrentCollections.newConcurrentSet(); + private final Map shortCircuitShardFailures = ConcurrentCollections.newConcurrentMap(); private final FailureStoreMetrics failureStoreMetrics; + private final DataStreamFailureStoreSettings dataStreamFailureStoreSettings; BulkOperation( Task task, @@ -104,12 +108,12 @@ final class BulkOperation extends ActionRunnable { BulkRequest bulkRequest, NodeClient client, AtomicArray responses, - Map indicesThatCannotBeCreated, IndexNameExpressionResolver indexNameExpressionResolver, LongSupplier relativeTimeProvider, long startTimeNanos, ActionListener listener, - FailureStoreMetrics failureStoreMetrics + FailureStoreMetrics failureStoreMetrics, + DataStreamFailureStoreSettings dataStreamFailureStoreSettings ) { this( task, @@ -119,14 +123,14 @@ final class BulkOperation extends ActionRunnable { bulkRequest, client, responses, - indicesThatCannotBeCreated, indexNameExpressionResolver, relativeTimeProvider, startTimeNanos, listener, new ClusterStateObserver(clusterService, bulkRequest.timeout(), logger, threadPool.getThreadContext()), new FailureStoreDocumentConverter(), - failureStoreMetrics + failureStoreMetrics, + dataStreamFailureStoreSettings ); } @@ -138,14 +142,14 @@ final class BulkOperation extends ActionRunnable { BulkRequest bulkRequest, NodeClient client, AtomicArray responses, - Map indicesThatCannotBeCreated, IndexNameExpressionResolver indexNameExpressionResolver, LongSupplier relativeTimeProvider, long startTimeNanos, ActionListener listener, ClusterStateObserver observer, FailureStoreDocumentConverter failureStoreDocumentConverter, - FailureStoreMetrics failureStoreMetrics + FailureStoreMetrics failureStoreMetrics, + DataStreamFailureStoreSettings dataStreamFailureStoreSettings ) { super(listener); this.task = task; @@ -155,7 +159,6 @@ final class BulkOperation extends ActionRunnable { this.bulkRequest = bulkRequest; this.listener = listener; this.startTimeNanos = startTimeNanos; - this.indicesThatCannotBeCreated = indicesThatCannotBeCreated; this.executor = executor; this.relativeTimeProvider = relativeTimeProvider; this.indexNameExpressionResolver = indexNameExpressionResolver; @@ -163,7 +166,9 @@ final class BulkOperation extends ActionRunnable { this.observer = observer; this.failureStoreDocumentConverter = failureStoreDocumentConverter; this.rolloverClient = new OriginSettingClient(client, LAZY_ROLLOVER_ORIGIN); + this.shortCircuitShardFailures.putAll(bulkRequest.incrementalState().shardLevelFailures()); this.failureStoreMetrics = failureStoreMetrics; + this.dataStreamFailureStoreSettings = dataStreamFailureStoreSettings; } @Override @@ -213,7 +218,7 @@ private void rollOverFailureStores(Runnable runnable) { RolloverRequest rolloverRequest = new RolloverRequest(dataStream, null); rolloverRequest.setIndicesOptions( IndicesOptions.builder(rolloverRequest.indicesOptions()) - .failureStoreOptions(new IndicesOptions.FailureStoreOptions(false, true)) + .selectorOptions(IndicesOptions.SelectorOptions.FAILURES) .build() ); // We are executing a lazy rollover because it is an action specialised for this situation, when we want an @@ -237,7 +242,13 @@ public void onFailure(Exception e) { if (failureStoreRedirect.index().equals(dataStream) == false) { continue; } - addFailure(failureStoreRedirect.request(), failureStoreRedirect.id(), failureStoreRedirect.index(), e); + addFailure( + failureStoreRedirect.request(), + failureStoreRedirect.id(), + failureStoreRedirect.index(), + e, + IndexDocFailureStoreStatus.FAILED + ); failedRolloverRequests.add(failureStoreRedirect.id()); } } @@ -288,9 +299,6 @@ private Map> groupRequestsByShards( if (addFailureIfRequiresAliasAndAliasIsMissing(docWriteRequest, bulkItemRequest.id(), metadata)) { continue; } - if (addFailureIfIndexCannotBeCreated(docWriteRequest, bulkItemRequest.id())) { - continue; - } if (addFailureIfRequiresDataStreamAndNoParentDataStream(docWriteRequest, bulkItemRequest.id(), metadata)) { continue; } @@ -311,16 +319,26 @@ private Map> groupRequestsByShards( continue; } IndexRouting indexRouting = concreteIndices.routing(concreteIndex); - docWriteRequest.process(indexRouting); + docWriteRequest.preRoutingProcess(indexRouting); int shardId = docWriteRequest.route(indexRouting); + docWriteRequest.postRoutingProcess(indexRouting); List shardRequests = requestsByShard.computeIfAbsent( new ShardId(concreteIndex, shardId), shard -> new ArrayList<>() ); shardRequests.add(bulkItemRequest); + } catch (DataStream.TimestampError timestampError) { + IndexDocFailureStoreStatus failureStoreStatus = processFailure(bulkItemRequest, clusterState, timestampError); + if (IndexDocFailureStoreStatus.USED.equals(failureStoreStatus) == false) { + String name = ia != null ? ia.getName() : docWriteRequest.index(); + addFailureAndDiscardRequest(docWriteRequest, bulkItemRequest.id(), name, timestampError, failureStoreStatus); + } } catch (ElasticsearchParseException | IllegalArgumentException | RoutingMissingException | ResourceNotFoundException e) { String name = ia != null ? ia.getName() : docWriteRequest.index(); - addFailureAndDiscardRequest(docWriteRequest, bulkItemRequest.id(), name, e); + var failureStoreStatus = isFailureStoreRequest(docWriteRequest) + ? IndexDocFailureStoreStatus.FAILED + : IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN; + addFailureAndDiscardRequest(docWriteRequest, bulkItemRequest.id(), name, e, failureStoreStatus); } } return requestsByShard; @@ -402,7 +420,12 @@ private void redirectFailuresOrCompleteBulkOperation() { private void completeBulkOperation() { listener.onResponse( - new BulkResponse(responses.toArray(new BulkItemResponse[responses.length()]), buildTookInMillis(startTimeNanos)) + new BulkResponse( + responses.toArray(new BulkItemResponse[responses.length()]), + buildTookInMillis(startTimeNanos), + BulkResponse.NO_INGEST_TOOK, + new BulkRequest.IncrementalState(shortCircuitShardFailures, bulkRequest.incrementalState().indexingPressureAccounted()) + ) ); // Allow memory for bulk shard request items to be reclaimed before all items have been completed bulkRequest = null; @@ -422,96 +445,148 @@ private void discardRedirectsAndFinish(Exception exception) { BulkItemResponse originalFailure = responses.get(slot); if (originalFailure.isFailed()) { originalFailure.getFailure().getCause().addSuppressed(exception); + originalFailure.getFailure().setFailureStoreStatus(IndexDocFailureStoreStatus.FAILED); } + // Always replace the item in the responses for thread visibility of any mutations + responses.set(slot, originalFailure); } completeBulkOperation(); } private void executeBulkShardRequest(BulkShardRequest bulkShardRequest, Releasable releaseOnFinish) { - client.executeLocally(TransportShardBulkAction.TYPE, bulkShardRequest, new ActionListener<>() { + ShardId shardId = bulkShardRequest.shardId(); + + // Short circuit the shard level request with the existing shard failure. + if (shortCircuitShardFailures.containsKey(shardId)) { + handleShardFailure(bulkShardRequest, clusterService.state(), shortCircuitShardFailures.get(shardId)); + releaseOnFinish.close(); + } else { + client.executeLocally(TransportShardBulkAction.TYPE, bulkShardRequest, new ActionListener<>() { - // Lazily get the cluster state to avoid keeping it around longer than it is needed - private ClusterState clusterState = null; + // Lazily get the cluster state to avoid keeping it around longer than it is needed + private ClusterState clusterState = null; - private ClusterState getClusterState() { - if (clusterState == null) { - clusterState = clusterService.state(); + private ClusterState getClusterState() { + if (clusterState == null) { + clusterState = clusterService.state(); + } + return clusterState; } - return clusterState; - } - @Override - public void onResponse(BulkShardResponse bulkShardResponse) { - for (int idx = 0; idx < bulkShardResponse.getResponses().length; idx++) { - // We zip the requests and responses together so that we can identify failed documents and potentially store them - BulkItemResponse bulkItemResponse = bulkShardResponse.getResponses()[idx]; - BulkItemRequest bulkItemRequest = bulkShardRequest.items()[idx]; - - if (bulkItemResponse.isFailed()) { - assert bulkItemRequest.id() == bulkItemResponse.getItemId() : "Bulk items were returned out of order"; - processFailure(bulkItemRequest, bulkItemResponse.getFailure().getCause()); - addFailure(bulkItemResponse); - } else { - bulkItemResponse.getResponse().setShardInfo(bulkShardResponse.getShardInfo()); - responses.set(bulkItemResponse.getItemId(), bulkItemResponse); + @Override + public void onResponse(BulkShardResponse bulkShardResponse) { + for (int idx = 0; idx < bulkShardResponse.getResponses().length; idx++) { + // We zip the requests and responses together so that we can identify failed documents and potentially store them + BulkItemResponse bulkItemResponse = bulkShardResponse.getResponses()[idx]; + BulkItemRequest bulkItemRequest = bulkShardRequest.items()[idx]; + + if (bulkItemResponse.isFailed()) { + assert bulkItemRequest.id() == bulkItemResponse.getItemId() : "Bulk items were returned out of order"; + IndexDocFailureStoreStatus failureStoreStatus = processFailure( + bulkItemRequest, + getClusterState(), + bulkItemResponse.getFailure().getCause() + ); + bulkItemResponse.getFailure().setFailureStoreStatus(failureStoreStatus); + addFailure(bulkItemResponse); + } else { + bulkItemResponse.getResponse().setShardInfo(bulkShardResponse.getShardInfo()); + if (isFailureStoreRequest(bulkItemRequest.request()) + && bulkItemResponse.getResponse() instanceof IndexResponse ir) { + ir.setFailureStoreStatus(IndexDocFailureStoreStatus.USED); + } + responses.set(bulkItemResponse.getItemId(), bulkItemResponse); + } } + completeShardOperation(); } - completeShardOperation(); - } - @Override - public void onFailure(Exception e) { - // create failures for all relevant requests - for (BulkItemRequest request : bulkShardRequest.items()) { - final String indexName = request.index(); - DocWriteRequest docWriteRequest = request.request(); - - processFailure(request, e); - addFailure(docWriteRequest, request.id(), indexName, e); + @Override + public void onFailure(Exception e) { + assert shortCircuitShardFailures.containsKey(shardId) == false; + shortCircuitShardFailures.put(shardId, e); + + // create failures for all relevant requests + handleShardFailure(bulkShardRequest, getClusterState(), e); + completeShardOperation(); } - completeShardOperation(); - } - private void completeShardOperation() { - // Clear our handle on the cluster state to allow it to be cleaned up - clusterState = null; - releaseOnFinish.close(); - } + private void completeShardOperation() { + // Clear our handle on the cluster state to allow it to be cleaned up + clusterState = null; + releaseOnFinish.close(); + } + }); + } + } - private void processFailure(BulkItemRequest bulkItemRequest, Exception cause) { - var error = ExceptionsHelper.unwrapCause(cause); - var errorType = ElasticsearchException.getExceptionName(error); - DocWriteRequest docWriteRequest = bulkItemRequest.request(); - DataStream failureStoreCandidate = getRedirectTargetCandidate(docWriteRequest, getClusterState().metadata()); - // If the candidate is not null, the BulkItemRequest targets a data stream, but we'll still have to check if - // it has the failure store enabled. - if (failureStoreCandidate != null) { - // Do not redirect documents to a failure store that were already headed to one. - var isFailureStoreDoc = docWriteRequest instanceof IndexRequest indexRequest && indexRequest.isWriteToFailureStore(); - if (isFailureStoreDoc == false - && failureStoreCandidate.isFailureStoreEnabled() - && error instanceof VersionConflictEngineException == false) { - // Redirect to failure store. - maybeMarkFailureStoreForRollover(failureStoreCandidate); - addDocumentToRedirectRequests(bulkItemRequest, cause, failureStoreCandidate.getName()); - failureStoreMetrics.incrementFailureStore( - bulkItemRequest.index(), - errorType, - FailureStoreMetrics.ErrorLocation.SHARD - ); - } else { - // If we can't redirect to a failure store (because either the data stream doesn't have the failure store enabled - // or this request was already targeting a failure store), we increment the rejected counter. - failureStoreMetrics.incrementRejected( - bulkItemRequest.index(), - errorType, - FailureStoreMetrics.ErrorLocation.SHARD, - isFailureStoreDoc - ); - } + private void handleShardFailure(BulkShardRequest bulkShardRequest, ClusterState clusterState, Exception e) { + // create failures for all relevant requests + for (BulkItemRequest request : bulkShardRequest.items()) { + final String indexName = request.index(); + DocWriteRequest docWriteRequest = request.request(); + IndexDocFailureStoreStatus failureStoreStatus = processFailure(request, clusterState, e); + addFailure(docWriteRequest, request.id(), indexName, e, failureStoreStatus); + } + } + + /** + * This method checks the eligibility of the failed document to be redirected to the failure store and updates the metrics. + * If the document is eligible it will call {@link BulkOperation#addDocumentToRedirectRequests}. Otherwise, it will return + * the appropriate failure store status and count this document as rejected. + * @return the status: + * - NOT_ENABLED, if the data stream didn't have the data store enabled and + * - FAILED if something went wrong in the preparation of the failure store request. + */ + private IndexDocFailureStoreStatus processFailure(BulkItemRequest bulkItemRequest, ClusterState clusterState, Exception cause) { + var error = ExceptionsHelper.unwrapCause(cause); + var errorType = ElasticsearchException.getExceptionName(error); + DocWriteRequest docWriteRequest = bulkItemRequest.request(); + DataStream failureStoreCandidate = getRedirectTargetCandidate(docWriteRequest, clusterState.metadata()); + // If the candidate is not null, the BulkItemRequest targets a data stream, but we'll still have to check if + // it has the failure store enabled. + if (failureStoreCandidate != null) { + // Do not redirect documents to a failure store that were already headed to one. + var isFailureStoreRequest = isFailureStoreRequest(docWriteRequest); + if (isFailureStoreRequest == false + && failureStoreCandidate.isFailureStoreEffectivelyEnabled(dataStreamFailureStoreSettings) + && error instanceof VersionConflictEngineException == false) { + // Prepare the data stream failure store if necessary + maybeMarkFailureStoreForRollover(failureStoreCandidate); + + // Enqueue the redirect to failure store. + boolean added = addDocumentToRedirectRequests(bulkItemRequest, cause, failureStoreCandidate.getName()); + if (added) { + failureStoreMetrics.incrementFailureStore(bulkItemRequest.index(), errorType, FailureStoreMetrics.ErrorLocation.SHARD); + return IndexDocFailureStoreStatus.USED; + } else { + failureStoreMetrics.incrementRejected( + bulkItemRequest.index(), + errorType, + FailureStoreMetrics.ErrorLocation.SHARD, + isFailureStoreRequest + ); + return IndexDocFailureStoreStatus.FAILED; + } + } else { + // If we can't redirect to a failure store (because either the data stream doesn't have the failure store enabled + // or this request was already targeting a failure store), or this was a version conflict we increment the + // rejected counter. + failureStoreMetrics.incrementRejected( + bulkItemRequest.index(), + errorType, + FailureStoreMetrics.ErrorLocation.SHARD, + isFailureStoreRequest + ); + if (isFailureStoreRequest) { + return IndexDocFailureStoreStatus.FAILED; + } + if (failureStoreCandidate.isFailureStoreEffectivelyEnabled(dataStreamFailureStoreSettings) == false) { + return IndexDocFailureStoreStatus.NOT_ENABLED; } } - }); + } + return IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN; } /** @@ -539,8 +614,9 @@ private static DataStream getRedirectTargetCandidate(DocWriteRequest docWrite * @param request The bulk item request that failed * @param cause The exception for the experienced the failure * @param failureStoreReference The data stream that contains the failure store for this item + * @return true, if adding the request to the queue was successful, false otherwise. */ - private void addDocumentToRedirectRequests(BulkItemRequest request, Exception cause, String failureStoreReference) { + private boolean addDocumentToRedirectRequests(BulkItemRequest request, Exception cause, String failureStoreReference) { // Convert the document into a failure document IndexRequest failureStoreRequest; try { @@ -565,12 +641,12 @@ private void addDocumentToRedirectRequests(BulkItemRequest request, Exception ca ); // Suppress and do not redirect cause.addSuppressed(ioException); - return; + return false; } // Store for second phase BulkItemRequest redirected = new BulkItemRequest(request.id(), failureStoreRequest); - failureStoreRedirects.add(redirected); + return failureStoreRedirects.add(redirected); } /** @@ -659,7 +735,7 @@ private boolean addFailureIfRequiresAliasAndAliasIsMissing(DocWriteRequest re "[" + DocWriteRequest.REQUIRE_ALIAS + "] request flag is [true] and [" + request.index() + "] is not an alias", request.index() ); - addFailureAndDiscardRequest(request, idx, request.index(), exception); + addFailureAndDiscardRequest(request, idx, request.index(), exception, IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN); return true; } return false; @@ -671,7 +747,7 @@ private boolean addFailureIfRequiresDataStreamAndNoParentDataStream(DocWriteRequ "[" + DocWriteRequest.REQUIRE_DATA_STREAM + "] request flag is [true] and [" + request.index() + "] is not a data stream", request.index() ); - addFailureAndDiscardRequest(request, idx, request.index(), exception); + addFailureAndDiscardRequest(request, idx, request.index(), exception, IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN); return true; } return false; @@ -680,27 +756,31 @@ private boolean addFailureIfRequiresDataStreamAndNoParentDataStream(DocWriteRequ private boolean addFailureIfIndexIsClosed(DocWriteRequest request, Index concreteIndex, int idx, final Metadata metadata) { IndexMetadata indexMetadata = metadata.getIndexSafe(concreteIndex); if (indexMetadata.getState() == IndexMetadata.State.CLOSE) { - addFailureAndDiscardRequest(request, idx, request.index(), new IndexClosedException(concreteIndex)); + var failureStoreStatus = isFailureStoreRequest(request) + ? IndexDocFailureStoreStatus.FAILED + : IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN; + addFailureAndDiscardRequest(request, idx, request.index(), new IndexClosedException(concreteIndex), failureStoreStatus); return true; } return false; } - private boolean addFailureIfIndexCannotBeCreated(DocWriteRequest request, int idx) { - IndexNotFoundException cannotCreate = indicesThatCannotBeCreated.get(request.index()); - if (cannotCreate != null) { - addFailureAndDiscardRequest(request, idx, request.index(), cannotCreate); - return true; - } - return false; + private static boolean isFailureStoreRequest(DocWriteRequest request) { + return request instanceof IndexRequest ir && ir.isWriteToFailureStore(); } /** - * Like {@link BulkOperation#addFailure(DocWriteRequest, int, String, Exception)} but this method will remove the corresponding entry - * from the working bulk request so that it never gets processed again during this operation. + * Like {@link BulkOperation#addFailure(DocWriteRequest, int, String, Exception, IndexDocFailureStoreStatus)} but this method will + * remove the corresponding entry from the working bulk request so that it never gets processed again during this operation. */ - private void addFailureAndDiscardRequest(DocWriteRequest request, int idx, String index, Exception exception) { - addFailure(request, idx, index, exception); + private void addFailureAndDiscardRequest( + DocWriteRequest request, + int idx, + String index, + Exception exception, + IndexDocFailureStoreStatus failureStoreStatus + ) { + addFailure(request, idx, index, exception, failureStoreStatus); // make sure the request gets never processed again bulkRequest.requests.set(idx, null); } @@ -714,18 +794,26 @@ private void addFailureAndDiscardRequest(DocWriteRequest request, int idx, St * @param idx The slot of the bulk entry this request corresponds to * @param index The resource that this entry was being written to when it failed * @param exception The exception encountered for this entry + * @param failureStoreStatus The failure status as it was identified by this entry * @see BulkOperation#addFailure(BulkItemResponse) BulkOperation.addFailure if you have a bulk item response object already */ - private void addFailure(DocWriteRequest request, int idx, String index, Exception exception) { + private void addFailure( + DocWriteRequest request, + int idx, + String index, + Exception exception, + IndexDocFailureStoreStatus failureStoreStatus + ) { BulkItemResponse bulkItemResponse = responses.get(idx); if (bulkItemResponse == null) { - BulkItemResponse.Failure failure = new BulkItemResponse.Failure(index, request.id(), exception); + BulkItemResponse.Failure failure = new BulkItemResponse.Failure(index, request.id(), exception, failureStoreStatus); bulkItemResponse = BulkItemResponse.failure(idx, request.opType(), failure); } else { // Response already recorded. We should only be here if the existing response is a failure and // we are encountering a new failure while redirecting. assert bulkItemResponse.isFailed() : "Attempting to overwrite successful bulk item result with a failure"; bulkItemResponse.getFailure().getCause().addSuppressed(exception); + bulkItemResponse.getFailure().setFailureStoreStatus(failureStoreStatus); } // Always replace the item in the responses for thread visibility of any mutations responses.set(idx, bulkItemResponse); @@ -736,8 +824,8 @@ private void addFailure(DocWriteRequest request, int idx, String index, Excep * already, the failure information provided to this call will be added to the existing failure as a suppressed exception. * * @param bulkItemResponse the item response to add to the overall result array - * @see BulkOperation#addFailure(DocWriteRequest, int, String, Exception) BulkOperation.addFailure which conditionally creates the - * failure response only when one does not exist already + * @see BulkOperation#addFailure(DocWriteRequest, int, String, Exception, IndexDocFailureStoreStatus) BulkOperation.addFailure which + * conditionally creates the failure response only when one does not exist already */ private void addFailure(BulkItemResponse bulkItemResponse) { assert bulkItemResponse.isFailed() : "Attempting to add a successful bulk item response via the addFailure method"; @@ -747,6 +835,7 @@ private void addFailure(BulkItemResponse bulkItemResponse) { // we are encountering a new failure while redirecting. assert existingBulkItemResponse.isFailed() : "Attempting to overwrite successful bulk item result with a failure"; existingBulkItemResponse.getFailure().getCause().addSuppressed(bulkItemResponse.getFailure().getCause()); + existingBulkItemResponse.getFailure().setFailureStoreStatus(bulkItemResponse.getFailure().getFailureStoreStatus()); bulkItemResponse = existingBulkItemResponse; } // Always replace the item in the responses for thread visibility of any mutations diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContext.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContext.java index 28eef30f9185d..4d6edd136de70 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContext.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkProcessor.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkProcessor.java index 7a2bd0e68608a..967ec1529ae26 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkProcessor.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkProcessor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkProcessor2.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkProcessor2.java index d1235ba317784..916c37f93e356 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkProcessor2.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkProcessor2.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java index 1a8bdb1c885c6..91caebc420ffb 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java @@ -1,15 +1,17 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.CompositeIndicesRequest; @@ -20,14 +22,18 @@ import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.replication.ReplicationRequest; import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.transport.RawIndexingDataTransportRequest; import org.elasticsearch.xcontent.XContentType; @@ -37,6 +43,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; @@ -69,6 +76,7 @@ public class BulkRequest extends ActionRequest private final Set indices = new HashSet<>(); protected TimeValue timeout = BulkShardRequest.DEFAULT_TIMEOUT; + private IncrementalState incrementalState = IncrementalState.EMPTY; private ActiveShardCount waitForActiveShards = ActiveShardCount.DEFAULT; private RefreshPolicy refreshPolicy = RefreshPolicy.NONE; private String globalPipeline; @@ -90,6 +98,11 @@ public BulkRequest(StreamInput in) throws IOException { for (DocWriteRequest request : requests) { indices.add(Objects.requireNonNull(request.index(), "request index must not be null")); } + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { + incrementalState = new BulkRequest.IncrementalState(in); + } else { + incrementalState = BulkRequest.IncrementalState.EMPTY; + } } public BulkRequest(@Nullable String globalIndex) { @@ -324,6 +337,10 @@ public final BulkRequest timeout(TimeValue timeout) { return this; } + public void incrementalState(IncrementalState incrementalState) { + this.incrementalState = incrementalState; + } + /** * Note for internal callers (NOT high level rest client), * the global parameter setting is ignored when used with: @@ -362,6 +379,10 @@ public TimeValue timeout() { return timeout; } + public IncrementalState incrementalState() { + return incrementalState; + } + public String pipeline() { return globalPipeline; } @@ -433,6 +454,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeCollection(requests, DocWriteRequest::writeDocumentRequest); refreshPolicy.writeTo(out); out.writeTimeValue(timeout); + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { + incrementalState.writeTo(out); + } } @Override @@ -474,4 +498,45 @@ public Set getIndices() { public boolean isSimulated() { return false; // Always false, but may be overridden by a subclass } + + /* + * Returns any component template substitutions that are to be used as part of this bulk request. We would likely only have + * substitutions in the event of a simulated request. + */ + public Map getComponentTemplateSubstitutions() throws IOException { + return Map.of(); + } + + public Map getIndexTemplateSubstitutions() throws IOException { + return Map.of(); + } + + record IncrementalState(Map shardLevelFailures, boolean indexingPressureAccounted) implements Writeable { + + static final IncrementalState EMPTY = new IncrementalState(Collections.emptyMap(), false); + + IncrementalState(StreamInput in) throws IOException { + this(in.readMap(ShardId::new, input -> input.readException()), false); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeMap(shardLevelFailures, (o, s) -> s.writeTo(o), StreamOutput::writeException); + } + } + + /* + * This copies this bulk request, but without all of its inner requests or the set of indices found in those requests + */ + public BulkRequest shallowClone() { + BulkRequest bulkRequest = new BulkRequest(globalIndex); + bulkRequest.setRefreshPolicy(getRefreshPolicy()); + bulkRequest.waitForActiveShards(waitForActiveShards()); + bulkRequest.timeout(timeout()); + bulkRequest.pipeline(pipeline()); + bulkRequest.routing(routing()); + bulkRequest.requireAlias(requireAlias()); + bulkRequest.requireDataStream(requireDataStream()); + return bulkRequest; + } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java index 31e54cdb872ca..fb65e487fa21a 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestHandler.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestHandler.java index 402b41761e3d1..9da9cac0712d1 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestHandler.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestHandler.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestModifier.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestModifier.java index d0a75bdf109c5..282e4d33fb83b 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestModifier.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestModifier.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; @@ -86,10 +87,7 @@ BulkRequest getBulkRequest() { if (itemResponses.isEmpty()) { return bulkRequest; } else { - BulkRequest modifiedBulkRequest = new BulkRequest(); - modifiedBulkRequest.setRefreshPolicy(bulkRequest.getRefreshPolicy()); - modifiedBulkRequest.waitForActiveShards(bulkRequest.waitForActiveShards()); - modifiedBulkRequest.timeout(bulkRequest.timeout()); + BulkRequest modifiedBulkRequest = bulkRequest.shallowClone(); int slot = 0; List> requests = bulkRequest.requests(); @@ -116,7 +114,12 @@ BulkRequest getBulkRequest() { ActionListener wrapActionListenerIfNeeded(long ingestTookInMillis, ActionListener actionListener) { if (itemResponses.isEmpty()) { return actionListener.map( - response -> new BulkResponse(response.getItems(), response.getTook().getMillis(), ingestTookInMillis) + response -> new BulkResponse( + response.getItems(), + response.getTook().getMillis(), + ingestTookInMillis, + response.getIncrementalState() + ) ); } else { return actionListener.map(response -> { @@ -141,7 +144,7 @@ ActionListener wrapActionListenerIfNeeded(long ingestTookInMillis, assertResponsesAreCorrect(bulkResponses, allResponses); } - return new BulkResponse(allResponses, response.getTook().getMillis(), ingestTookInMillis); + return new BulkResponse(allResponses, response.getTook().getMillis(), ingestTookInMillis, response.getIncrementalState()); }); } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java index 898bfd0e1652c..3a61c548ebecb 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; @@ -85,13 +86,13 @@ public BulkRequestParser(boolean deprecateOrErrorOnType, RestApiVersion restApiV .withRestApiVersion(restApiVersion); } - private static int findNextMarker(byte marker, int from, BytesReference data) { + private static int findNextMarker(byte marker, int from, BytesReference data, boolean lastData) { final int res = data.indexOf(marker, from); if (res != -1) { assert res >= 0; return res; } - if (from != data.length()) { + if (from != data.length() && lastData) { throw new IllegalArgumentException("The bulk request must be terminated by a newline [\\n]"); } return res; @@ -136,32 +137,176 @@ public void parse( Consumer updateRequestConsumer, Consumer deleteRequestConsumer ) throws IOException { - XContent xContent = xContentType.xContent(); - int line = 0; - int from = 0; - byte marker = xContent.bulkSeparator(); + IncrementalParser incrementalParser = new IncrementalParser( + defaultIndex, + defaultRouting, + defaultFetchSourceContext, + defaultPipeline, + defaultRequireAlias, + defaultRequireDataStream, + defaultListExecutedPipelines, + allowExplicitIndex, + xContentType, + indexRequestConsumer, + updateRequestConsumer, + deleteRequestConsumer + ); + + incrementalParser.parse(data, true); + } + + public IncrementalParser incrementalParser( + @Nullable String defaultIndex, + @Nullable String defaultRouting, + @Nullable FetchSourceContext defaultFetchSourceContext, + @Nullable String defaultPipeline, + @Nullable Boolean defaultRequireAlias, + @Nullable Boolean defaultRequireDataStream, + @Nullable Boolean defaultListExecutedPipelines, + boolean allowExplicitIndex, + XContentType xContentType, + BiConsumer indexRequestConsumer, + Consumer updateRequestConsumer, + Consumer deleteRequestConsumer + ) { + return new IncrementalParser( + defaultIndex, + defaultRouting, + defaultFetchSourceContext, + defaultPipeline, + defaultRequireAlias, + defaultRequireDataStream, + defaultListExecutedPipelines, + allowExplicitIndex, + xContentType, + indexRequestConsumer, + updateRequestConsumer, + deleteRequestConsumer + ); + } + + public class IncrementalParser { + // Bulk requests can contain a lot of repeated strings for the index, pipeline and routing parameters. This map is used to // deduplicate duplicate strings parsed for these parameters. While it does not prevent instantiating the duplicate strings, it // reduces their lifetime to the lifetime of this parse call instead of the lifetime of the full bulk request. - final Map stringDeduplicator = new HashMap<>(); + private final Map stringDeduplicator = new HashMap<>(); + + private final String defaultIndex; + private final String defaultRouting; + private final FetchSourceContext defaultFetchSourceContext; + private final String defaultPipeline; + private final Boolean defaultRequireAlias; + private final Boolean defaultRequireDataStream; + private final Boolean defaultListExecutedPipelines; + private final boolean allowExplicitIndex; + + private final XContentType xContentType; + private final byte marker; + private final BiConsumer indexRequestConsumer; + private final Consumer updateRequestConsumer; + private final Consumer deleteRequestConsumer; + + private Exception failure = null; + private int incrementalFromOffset = 0; + private int line = 0; boolean typesDeprecationLogged = false; - while (true) { - int nextMarker = findNextMarker(marker, from, data); - if (nextMarker == -1) { - break; + private DocWriteRequest currentRequest = null; + private String currentType = null; + private String currentPipeline = null; + private boolean currentListExecutedPipelines = false; + private FetchSourceContext currentFetchSourceContext = null; + + private IncrementalParser( + @Nullable String defaultIndex, + @Nullable String defaultRouting, + @Nullable FetchSourceContext defaultFetchSourceContext, + @Nullable String defaultPipeline, + @Nullable Boolean defaultRequireAlias, + @Nullable Boolean defaultRequireDataStream, + @Nullable Boolean defaultListExecutedPipelines, + boolean allowExplicitIndex, + XContentType xContentType, + BiConsumer indexRequestConsumer, + Consumer updateRequestConsumer, + Consumer deleteRequestConsumer + ) { + this.defaultIndex = defaultIndex; + this.defaultRouting = defaultRouting; + this.defaultFetchSourceContext = defaultFetchSourceContext; + this.defaultPipeline = defaultPipeline; + this.defaultRequireAlias = defaultRequireAlias; + this.defaultRequireDataStream = defaultRequireDataStream; + this.defaultListExecutedPipelines = defaultListExecutedPipelines; + this.allowExplicitIndex = allowExplicitIndex; + this.xContentType = xContentType; + this.marker = xContentType.xContent().bulkSeparator(); + this.indexRequestConsumer = indexRequestConsumer; + this.updateRequestConsumer = updateRequestConsumer; + this.deleteRequestConsumer = deleteRequestConsumer; + } + + public int parse(BytesReference data, boolean lastData) throws IOException { + if (failure != null) { + assert false : failure.getMessage(); + throw new IllegalStateException("Parser has already encountered exception", failure); } - line++; + try { + return tryParse(data, lastData); + } catch (Exception e) { + failure = e; + throw e; + } + } + + private int tryParse(BytesReference data, boolean lastData) throws IOException { + int from = 0; + int consumed = 0; + + while (true) { + int nextMarker = findNextMarker(marker, incrementalFromOffset, data, lastData); + if (nextMarker == -1) { + incrementalFromOffset = data.length() - consumed; + break; + } + incrementalFromOffset = nextMarker + 1; + line++; + + if (currentRequest == null) { + if (parseActionLine(data, from, nextMarker)) { + if (currentRequest instanceof DeleteRequest deleteRequest) { + deleteRequestConsumer.accept(deleteRequest); + currentRequest = null; + } + } + } else { + parseAndConsumeDocumentLine(data, from, nextMarker); + currentRequest = null; + } - // now parse the action - try (XContentParser parser = createParser(xContent, data, from, nextMarker)) { - // move pointers from = nextMarker + 1; + consumed = from; + } + + return lastData ? from : consumed; + } + + private boolean parseActionLine(BytesReference data, int from, int to) throws IOException { + assert currentRequest == null; + + // Reset the fields which are accessed during document line parsing + currentType = null; + currentPipeline = defaultPipeline; + currentListExecutedPipelines = defaultListExecutedPipelines != null && defaultListExecutedPipelines; + currentFetchSourceContext = defaultFetchSourceContext; + + try (XContentParser parser = createParser(xContentType.xContent(), data, from, to)) { // Move to START_OBJECT XContentParser.Token token = parser.nextToken(); if (token == null) { - continue; + return false; } if (token != XContentParser.Token.START_OBJECT) { throw new IllegalArgumentException( @@ -199,20 +344,16 @@ public void parse( } String index = defaultIndex; - String type = null; String id = null; String routing = defaultRouting; - FetchSourceContext fetchSourceContext = defaultFetchSourceContext; String opType = null; long version = Versions.MATCH_ANY; VersionType versionType = VersionType.INTERNAL; long ifSeqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; long ifPrimaryTerm = UNASSIGNED_PRIMARY_TERM; int retryOnConflict = 0; - String pipeline = defaultPipeline; boolean requireAlias = defaultRequireAlias != null && defaultRequireAlias; boolean requireDataStream = defaultRequireDataStream != null && defaultRequireDataStream; - boolean listExecutedPipelines = defaultListExecutedPipelines != null && defaultListExecutedPipelines; Map dynamicTemplates = Map.of(); // at this stage, next token can either be END_OBJECT (and use default index and type, with auto generated id) @@ -243,7 +384,7 @@ public void parse( "Action/metadata line [" + line + "] contains an unknown parameter [" + currentFieldName + "]" ); } - type = stringDeduplicator.computeIfAbsent(parser.text(), Function.identity()); + currentType = stringDeduplicator.computeIfAbsent(parser.text(), Function.identity()); } else if (ID.match(currentFieldName, parser.getDeprecationHandler())) { id = parser.text(); } else if (ROUTING.match(currentFieldName, parser.getDeprecationHandler())) { @@ -261,15 +402,15 @@ public void parse( } else if (RETRY_ON_CONFLICT.match(currentFieldName, parser.getDeprecationHandler())) { retryOnConflict = parser.intValue(); } else if (PIPELINE.match(currentFieldName, parser.getDeprecationHandler())) { - pipeline = stringDeduplicator.computeIfAbsent(parser.text(), Function.identity()); + currentPipeline = stringDeduplicator.computeIfAbsent(parser.text(), Function.identity()); } else if (SOURCE.match(currentFieldName, parser.getDeprecationHandler())) { - fetchSourceContext = FetchSourceContext.fromXContent(parser); + currentFetchSourceContext = FetchSourceContext.fromXContent(parser); } else if (REQUIRE_ALIAS.match(currentFieldName, parser.getDeprecationHandler())) { requireAlias = parser.booleanValue(); } else if (REQUIRE_DATA_STREAM.match(currentFieldName, parser.getDeprecationHandler())) { requireDataStream = parser.booleanValue(); } else if (LIST_EXECUTED_PIPELINES.match(currentFieldName, parser.getDeprecationHandler())) { - listExecutedPipelines = parser.booleanValue(); + currentListExecutedPipelines = parser.booleanValue(); } else { throw new IllegalArgumentException( "Action/metadata line [" + line + "] contains an unknown parameter [" + currentFieldName + "]" @@ -290,7 +431,7 @@ public void parse( dynamicTemplates = parser.mapStrings(); } else if (token == XContentParser.Token.START_OBJECT && SOURCE.match(currentFieldName, parser.getDeprecationHandler())) { - fetchSourceContext = FetchSourceContext.fromXContent(parser); + currentFetchSourceContext = FetchSourceContext.fromXContent(parser); } else if (token != XContentParser.Token.VALUE_NULL) { throw new IllegalArgumentException( "Malformed action/metadata line [" @@ -324,21 +465,13 @@ public void parse( "Delete request in line [" + line + "] does not accept " + DYNAMIC_TEMPLATES.getPreferredName() ); } - deleteRequestConsumer.accept( - new DeleteRequest(index).id(id) - .routing(routing) - .version(version) - .versionType(versionType) - .setIfSeqNo(ifSeqNo) - .setIfPrimaryTerm(ifPrimaryTerm) - ); + currentRequest = new DeleteRequest(index).id(id) + .routing(routing) + .version(version) + .versionType(versionType) + .setIfSeqNo(ifSeqNo) + .setIfPrimaryTerm(ifPrimaryTerm); } else { - nextMarker = findNextMarker(marker, from, data); - if (nextMarker == -1) { - break; - } - line++; - // we use internalAdd so we don't fork here, this allows us not to copy over the big byte array to small chunks // of index request. if ("index".equals(action) || "create".equals(action)) { @@ -346,20 +479,19 @@ public void parse( .routing(routing) .version(version) .versionType(versionType) - .setPipeline(pipeline) + .setPipeline(currentPipeline) .setIfSeqNo(ifSeqNo) .setIfPrimaryTerm(ifPrimaryTerm) - .source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType) .setDynamicTemplates(dynamicTemplates) .setRequireAlias(requireAlias) .setRequireDataStream(requireDataStream) - .setListExecutedPipelines(listExecutedPipelines); + .setListExecutedPipelines(currentListExecutedPipelines); if ("create".equals(action)) { indexRequest = indexRequest.create(true); } else if (opType != null) { indexRequest = indexRequest.create("create".equals(opType)); } - indexRequestConsumer.accept(indexRequest, type); + currentRequest = indexRequest; } else if ("update".equals(action)) { if (version != Versions.MATCH_ANY || versionType != VersionType.INTERNAL) { throw new IllegalArgumentException( @@ -386,29 +518,38 @@ public void parse( .setIfPrimaryTerm(ifPrimaryTerm) .setRequireAlias(requireAlias) .routing(routing); - try ( - XContentParser sliceParser = createParser( - xContent, - sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType) - ) - ) { - updateRequest.fromXContent(sliceParser); - } - if (fetchSourceContext != null) { - updateRequest.fetchSource(fetchSourceContext); - } - IndexRequest upsertRequest = updateRequest.upsertRequest(); - if (upsertRequest != null) { - upsertRequest.setPipeline(pipeline).setListExecutedPipelines(listExecutedPipelines); - } - - updateRequestConsumer.accept(updateRequest); + currentRequest = updateRequest; } - // move pointers - from = nextMarker + 1; } } + return true; } + + private void parseAndConsumeDocumentLine(BytesReference data, int from, int to) throws IOException { + assert currentRequest != null && currentRequest instanceof DeleteRequest == false; + if (currentRequest instanceof IndexRequest indexRequest) { + indexRequest.source(sliceTrimmingCarriageReturn(data, from, to, xContentType), xContentType); + indexRequestConsumer.accept(indexRequest, currentType); + } else if (currentRequest instanceof UpdateRequest updateRequest) { + try ( + XContentParser sliceParser = createParser( + xContentType.xContent(), + sliceTrimmingCarriageReturn(data, from, to, xContentType) + ) + ) { + updateRequest.fromXContent(sliceParser); + } + if (currentFetchSourceContext != null) { + updateRequest.fetchSource(currentFetchSourceContext); + } + IndexRequest upsertRequest = updateRequest.upsertRequest(); + if (upsertRequest != null) { + upsertRequest.setPipeline(currentPipeline).setListExecutedPipelines(currentListExecutedPipelines); + } + updateRequestConsumer.accept(updateRequest); + } + } + } @UpdateForV9 diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java index 111dbfb0f7af6..12d3aa67ca9bb 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java @@ -1,17 +1,20 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; +import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.common.xcontent.ChunkedToXContentObject; import org.elasticsearch.core.TimeValue; import org.elasticsearch.xcontent.ToXContent; @@ -36,12 +39,18 @@ public class BulkResponse extends ActionResponse implements Iterable toXContentChunked(ToXContent.Params params) { - return Iterators.concat(Iterators.single((builder, p) -> { - builder.startObject(); - builder.field(ERRORS, hasFailures()); - builder.field(TOOK, tookInMillis); + return ChunkedToXContent.builder(params).object(ob -> ob.append((b, p) -> { + b.field(ERRORS, hasFailures()); + b.field(TOOK, tookInMillis); if (ingestTookInMillis != BulkResponse.NO_INGEST_TOOK) { - builder.field(INGEST_TOOK, ingestTookInMillis); + b.field(INGEST_TOOK, ingestTookInMillis); } - return builder.startArray(ITEMS); - }), Iterators.forArray(responses), Iterators.single((builder, p) -> builder.endArray().endObject())); + return b; + }).array(ITEMS, Iterators.forArray(responses))); } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java index f7860c47d8b73..406ba9723b427 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; @@ -42,7 +43,7 @@ public final class BulkShardRequest extends ReplicatedWriteRequest i.readOptionalWriteable(inpt -> new BulkItemRequest(shardId, inpt)), BulkItemRequest[]::new); - if (in.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_VALIDATES_MAPPINGS)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { isSimulated = in.readBoolean(); } else { isSimulated = false; @@ -131,7 +132,7 @@ public void writeTo(StreamOutput out) throws IOException { } super.writeTo(out); out.writeArray((o, item) -> o.writeOptional(BulkItemRequest.THIN_WRITER, item), items); - if (out.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_VALIDATES_MAPPINGS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { out.writeBoolean(isSimulated); } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkShardResponse.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkShardResponse.java index eb1bb0468c9bb..f124fab3ee4e1 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkShardResponse.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkShardResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; diff --git a/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverter.java b/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverter.java index 527a886905aaf..a5a38a288d342 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverter.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; @@ -17,12 +18,14 @@ import org.elasticsearch.xcontent.json.JsonXContent; import java.io.IOException; +import java.time.Instant; import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Set; import java.util.function.Supplier; +import static org.elasticsearch.common.xcontent.XContentElasticsearchExtension.DEFAULT_FORMATTER; import static org.elasticsearch.ingest.CompoundProcessor.PIPELINE_ORIGIN_EXCEPTION_HEADER; import static org.elasticsearch.ingest.CompoundProcessor.PROCESSOR_TAG_EXCEPTION_HEADER; import static org.elasticsearch.ingest.CompoundProcessor.PROCESSOR_TYPE_EXCEPTION_HEADER; @@ -83,7 +86,7 @@ private static XContentBuilder createSource(IndexRequest source, Exception excep XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); { - builder.timeField("@timestamp", timeSupplier.get()); + builder.field("@timestamp", DEFAULT_FORMATTER.format(Instant.ofEpochMilli(timeSupplier.get()))); builder.startObject("document"); { if (source.id() != null) { diff --git a/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreMetrics.java b/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreMetrics.java index 5a36f10785790..10152d089e50c 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreMetrics.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreMetrics.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; diff --git a/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java b/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java new file mode 100644 index 0000000000000..a7ee0a182762e --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java @@ -0,0 +1,308 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.bulk; + +import org.apache.lucene.util.Accountable; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.support.ActiveShardCount; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.IndexingPressure; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; + +import static org.elasticsearch.common.settings.Setting.boolSetting; + +public class IncrementalBulkService { + + public static final Setting INCREMENTAL_BULK = boolSetting( + "rest.incremental_bulk", + false, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + private final Client client; + private final AtomicBoolean enabledForTests = new AtomicBoolean(true); + private final IndexingPressure indexingPressure; + + public IncrementalBulkService(Client client, IndexingPressure indexingPressure) { + this.client = client; + this.indexingPressure = indexingPressure; + } + + public Handler newBulkRequest() { + ensureEnabled(); + return newBulkRequest(null, null, null); + } + + public Handler newBulkRequest(@Nullable String waitForActiveShards, @Nullable TimeValue timeout, @Nullable String refresh) { + ensureEnabled(); + return new Handler(client, indexingPressure, waitForActiveShards, timeout, refresh); + } + + private void ensureEnabled() { + if (enabledForTests.get() == false) { + throw new AssertionError("Unexpected incremental bulk request"); + } + } + + // This method only exists to tests that the feature flag works. Remove once we no longer need the flag. + public void setForTests(boolean value) { + enabledForTests.set(value); + } + + public static class Enabled implements Supplier { + + private final AtomicBoolean incrementalBulksEnabled = new AtomicBoolean(true); + + public Enabled() {} + + public Enabled(ClusterSettings clusterSettings) { + incrementalBulksEnabled.set(clusterSettings.get(INCREMENTAL_BULK)); + clusterSettings.addSettingsUpdateConsumer(INCREMENTAL_BULK, incrementalBulksEnabled::set); + } + + @Override + public Boolean get() { + return incrementalBulksEnabled.get(); + } + } + + public static class Handler implements Releasable { + + public static final BulkRequest.IncrementalState EMPTY_STATE = new BulkRequest.IncrementalState(Collections.emptyMap(), true); + + private final Client client; + private final IndexingPressure indexingPressure; + private final ActiveShardCount waitForActiveShards; + private final TimeValue timeout; + private final String refresh; + + private final ArrayList releasables = new ArrayList<>(4); + private final ArrayList responses = new ArrayList<>(2); + private boolean closed = false; + private boolean globalFailure = false; + private boolean incrementalRequestSubmitted = false; + private boolean bulkInProgress = false; + private Exception bulkActionLevelFailure = null; + private long currentBulkSize = 0L; + private BulkRequest bulkRequest = null; + + protected Handler( + Client client, + IndexingPressure indexingPressure, + @Nullable String waitForActiveShards, + @Nullable TimeValue timeout, + @Nullable String refresh + ) { + this.client = client; + this.indexingPressure = indexingPressure; + this.waitForActiveShards = waitForActiveShards != null ? ActiveShardCount.parseString(waitForActiveShards) : null; + this.timeout = timeout; + this.refresh = refresh; + createNewBulkRequest(EMPTY_STATE); + } + + public void addItems(List> items, Releasable releasable, Runnable nextItems) { + assert closed == false; + assert bulkInProgress == false; + if (bulkActionLevelFailure != null) { + shortCircuitDueToTopLevelFailure(items, releasable); + nextItems.run(); + } else { + assert bulkRequest != null; + if (internalAddItems(items, releasable)) { + if (shouldBackOff()) { + final boolean isFirstRequest = incrementalRequestSubmitted == false; + incrementalRequestSubmitted = true; + final ArrayList toRelease = new ArrayList<>(releasables); + releasables.clear(); + bulkInProgress = true; + client.bulk(bulkRequest, ActionListener.runAfter(new ActionListener<>() { + + @Override + public void onResponse(BulkResponse bulkResponse) { + handleBulkSuccess(bulkResponse); + createNewBulkRequest( + new BulkRequest.IncrementalState(bulkResponse.getIncrementalState().shardLevelFailures(), true) + ); + } + + @Override + public void onFailure(Exception e) { + handleBulkFailure(isFirstRequest, e); + } + }, () -> { + bulkInProgress = false; + toRelease.forEach(Releasable::close); + nextItems.run(); + })); + } else { + nextItems.run(); + } + } else { + nextItems.run(); + } + } + } + + private boolean shouldBackOff() { + return indexingPressure.shouldSplitBulk(currentBulkSize); + } + + public void lastItems(List> items, Releasable releasable, ActionListener listener) { + assert bulkInProgress == false; + if (bulkActionLevelFailure != null) { + shortCircuitDueToTopLevelFailure(items, releasable); + errorResponse(listener); + } else { + assert bulkRequest != null; + if (internalAddItems(items, releasable)) { + final ArrayList toRelease = new ArrayList<>(releasables); + releasables.clear(); + // We do not need to set this back to false as this will be the last request. + bulkInProgress = true; + client.bulk(bulkRequest, ActionListener.runBefore(new ActionListener<>() { + + private final boolean isFirstRequest = incrementalRequestSubmitted == false; + + @Override + public void onResponse(BulkResponse bulkResponse) { + handleBulkSuccess(bulkResponse); + listener.onResponse(combineResponses()); + } + + @Override + public void onFailure(Exception e) { + handleBulkFailure(isFirstRequest, e); + errorResponse(listener); + } + }, () -> toRelease.forEach(Releasable::close))); + } else { + errorResponse(listener); + } + } + } + + @Override + public void close() { + closed = true; + releasables.forEach(Releasable::close); + releasables.clear(); + } + + private void shortCircuitDueToTopLevelFailure(List> items, Releasable releasable) { + assert releasables.isEmpty(); + assert bulkRequest == null; + if (globalFailure == false) { + addItemLevelFailures(items); + } + Releasables.close(releasable); + } + + private void errorResponse(ActionListener listener) { + if (globalFailure) { + listener.onFailure(bulkActionLevelFailure); + } else { + listener.onResponse(combineResponses()); + } + } + + private void handleBulkSuccess(BulkResponse bulkResponse) { + responses.add(bulkResponse); + currentBulkSize = 0L; + bulkRequest = null; + } + + private void handleBulkFailure(boolean isFirstRequest, Exception e) { + assert bulkActionLevelFailure == null; + globalFailure = isFirstRequest; + bulkActionLevelFailure = e; + addItemLevelFailures(bulkRequest.requests()); + currentBulkSize = 0; + bulkRequest = null; + } + + private void addItemLevelFailures(List> items) { + BulkItemResponse[] bulkItemResponses = new BulkItemResponse[items.size()]; + int idx = 0; + for (DocWriteRequest item : items) { + BulkItemResponse.Failure failure = new BulkItemResponse.Failure(item.index(), item.id(), bulkActionLevelFailure); + bulkItemResponses[idx++] = BulkItemResponse.failure(idx, item.opType(), failure); + } + + responses.add(new BulkResponse(bulkItemResponses, 0, 0)); + } + + private boolean internalAddItems(List> items, Releasable releasable) { + try { + bulkRequest.add(items); + releasables.add(releasable); + long size = items.stream().mapToLong(Accountable::ramBytesUsed).sum(); + releasables.add(indexingPressure.markCoordinatingOperationStarted(items.size(), size, false)); + currentBulkSize += size; + return true; + } catch (EsRejectedExecutionException e) { + handleBulkFailure(incrementalRequestSubmitted == false, e); + releasables.forEach(Releasable::close); + releasables.clear(); + return false; + } + } + + private void createNewBulkRequest(BulkRequest.IncrementalState incrementalState) { + assert currentBulkSize == 0L; + assert bulkRequest == null; + bulkRequest = new BulkRequest(); + bulkRequest.incrementalState(incrementalState); + + if (waitForActiveShards != null) { + bulkRequest.waitForActiveShards(waitForActiveShards); + } + if (timeout != null) { + bulkRequest.timeout(timeout); + } + if (refresh != null) { + bulkRequest.setRefreshPolicy(refresh); + } + } + + private BulkResponse combineResponses() { + long tookInMillis = 0; + long ingestTookInMillis = 0; + int itemResponseCount = 0; + for (BulkResponse response : responses) { + tookInMillis += response.getTookInMillis(); + ingestTookInMillis += response.getIngestTookInMillis(); + itemResponseCount += response.getItems().length; + } + BulkItemResponse[] bulkItemResponses = new BulkItemResponse[itemResponseCount]; + int i = 0; + for (BulkResponse response : responses) { + for (BulkItemResponse itemResponse : response.getItems()) { + bulkItemResponses[i++] = itemResponse; + } + } + + return new BulkResponse(bulkItemResponses, tookInMillis, ingestTookInMillis); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/action/bulk/IndexDocFailureStoreStatus.java b/server/src/main/java/org/elasticsearch/action/bulk/IndexDocFailureStoreStatus.java new file mode 100644 index 0000000000000..7367dfa1d53fd --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/bulk/IndexDocFailureStoreStatus.java @@ -0,0 +1,154 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.bulk; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xcontent.ToXContentFragment; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Locale; + +/** + * Captures the role of the failure store in this document response. For example, + * - USED, means that this document was stored in the failure store + * - NOT_ENABLED, means that this document was rejected by elasticsearch, but it could have been stored in + * the failure store has it been enabled. + * - FAILED, means that this failed document was eligible to be stored in the failure store and the failure store + * was enabled but something went wrong. + */ +public enum IndexDocFailureStoreStatus implements ToXContentFragment, Writeable { + /** + * This status represents that we have no information about this response or that the failure store is not applicable. + * For example: + * - when the doc was successfully indexed in a backing index of a data stream, + * - when we are running in a mixed version cluster and the information is not available, + * - when the doc was rejected by elasticsearch but failure store was not applicable (i.e. the target was an index). + */ + NOT_APPLICABLE_OR_UNKNOWN(0), + /** + * This status represents that this document was stored in the failure store successfully. + */ + USED(1), + /** + * This status represents that this document was rejected, but it could have ended up in the failure store if it was enabled. + */ + NOT_ENABLED(2), + /** + * This status represents that this document was rejected from the failure store. + */ + FAILED(3); + + private final byte id; + private final String label; + + IndexDocFailureStoreStatus(int id) { + this.id = (byte) id; + this.label = this.toString().toLowerCase(Locale.ROOT); + } + + public static IndexDocFailureStoreStatus read(StreamInput in) throws IOException { + return fromId(in.readByte()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeByte(id); + } + + /** + * @return id of the status, mainly used for wire serialisation purposes + */ + public byte getId() { + return id; + } + + /** + * @return the label of this status for display, just lowercase of the enum + */ + public String getLabel() { + return label; + } + + /** + * @param id a candidate id that (hopefully) can be converted to a FailureStoreStatus, used in wire serialisation + * @return the failure store status that corresponds to the id. + * @throws IllegalArgumentException when the id cannot produce a failure store status + */ + public static IndexDocFailureStoreStatus fromId(byte id) { + return switch (id) { + case 0 -> NOT_APPLICABLE_OR_UNKNOWN; + case 1 -> USED; + case 2 -> NOT_ENABLED; + case 3 -> FAILED; + default -> throw new IllegalArgumentException("Unknown failure store status: [" + id + "]"); + }; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + // We avoid adding the not_applicable status in the response to not increase the size of bulk responses. + if (DataStream.isFailureStoreFeatureFlagEnabled() && this.equals(NOT_APPLICABLE_OR_UNKNOWN) == false) { + builder.field("failure_store", label); + } + return builder; + } + + /** + * Exception wrapper class that adds the failure store status in the XContent response. + * Note: We are not using {@link ExceptionWithFailureStoreStatus} because then it unwraps it directly + * in the {@link ElasticsearchException} and we cannot add the field. + */ + public static class ExceptionWithFailureStoreStatus extends ElasticsearchException { + + private final IndexDocFailureStoreStatus failureStoreStatus; + + public ExceptionWithFailureStoreStatus(BulkItemResponse.Failure failure) { + super(failure.getCause()); + this.failureStoreStatus = failure.getFailureStoreStatus(); + } + + public ExceptionWithFailureStoreStatus(StreamInput in) throws IOException { + super(in); + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { + failureStoreStatus = IndexDocFailureStoreStatus.fromId(in.readByte()); + } else { + failureStoreStatus = NOT_APPLICABLE_OR_UNKNOWN; + } + } + + @Override + protected void writeTo(StreamOutput out, Writer nestedExceptionsWriter) throws IOException { + super.writeTo(out, nestedExceptionsWriter); + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { + out.writeByte(failureStoreStatus.getId()); + } + } + + @Override + protected XContentBuilder toXContent(XContentBuilder builder, Params params, int nestedLevel) throws IOException { + generateThrowableXContent(builder, params, this.getCause(), nestedLevel); + failureStoreStatus.toXContent(builder, params); + return builder; + } + + @Override + public RestStatus status() { + return ExceptionsHelper.status(getCause()); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/action/bulk/MappingUpdatePerformer.java b/server/src/main/java/org/elasticsearch/action/bulk/MappingUpdatePerformer.java index d37cd88d8d6e3..c726a5aaa9aac 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/MappingUpdatePerformer.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/MappingUpdatePerformer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; diff --git a/server/src/main/java/org/elasticsearch/action/bulk/Retry.java b/server/src/main/java/org/elasticsearch/action/bulk/Retry.java index 62ef9a08f0070..574799881369e 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/Retry.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/Retry.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; diff --git a/server/src/main/java/org/elasticsearch/action/bulk/Retry2.java b/server/src/main/java/org/elasticsearch/action/bulk/Retry2.java index 999bd6af925a6..2dfe4e5bd34ec 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/Retry2.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/Retry2.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; diff --git a/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkAction.java index 089869395c997..aaa83d7007214 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; diff --git a/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java index 1987d758eb09a..290d342e9dc12 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java @@ -1,23 +1,30 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.core.Nullable; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.xcontent.XContentParserConfiguration; import java.io.IOException; import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; /** - * This extends BulkRequest with support for providing substitute pipeline definitions. In a user request, the pipeline substitutions - * will look something like this: + * This extends BulkRequest with support for providing substitute pipeline definitions, component template definitions, and index template + * substitutions. In a user request, the substitutions will look something like this: * * "pipeline_substitutions": { * "my-pipeline-1": { @@ -44,6 +51,47 @@ * } * ] * } + * }, + * "component_template_substitutions": { + * "my-template-1": { + * "template": { + * "settings": { + * "number_of_shards": 1 + * }, + * "mappings": { + * "_source": { + * "enabled": false + * }, + * "properties": { + * "host_name": { + * "type": "keyword" + * }, + * "created_at": { + * "type": "date", + * "format": "EEE MMM dd HH:mm:ss Z yyyy" + * } + * } + * } + * } + * } + * }, + * "index_template_substitutions": { + * "my-index-template-1": { + * "template": { + * "index_patterns": ["foo*", "bar*"] + * "composed_of": [ + * "component-template-1", + * "component-template-2" + * ] + * } + * } + * }, + * "mapping_addition": { + * "dynamic": "strict", + * "properties": { + * "foo": { + * "type": "keyword" + * } * } * * The pipelineSubstitutions Map held by this class is intended to be the result of XContentHelper.convertToMap(). The top-level keys @@ -52,27 +100,66 @@ */ public class SimulateBulkRequest extends BulkRequest { private final Map> pipelineSubstitutions; + private final Map> componentTemplateSubstitutions; + private final Map> indexTemplateSubstitutions; + private final Map mappingAddition; /** * @param pipelineSubstitutions The pipeline definitions that are to be used in place of any pre-existing pipeline definitions with * the same pipelineId. The key of the map is the pipelineId, and the value the pipeline definition as * parsed by XContentHelper.convertToMap(). + * @param componentTemplateSubstitutions The component template definitions that are to be used in place of any pre-existing + * component template definitions with the same name. + * @param indexTemplateSubstitutions The index template definitions that are to be used in place of any pre-existing + * index template definitions with the same name. + * @param mappingAddition A mapping that will be merged into the final index's mapping for mapping validation */ - public SimulateBulkRequest(@Nullable Map> pipelineSubstitutions) { + public SimulateBulkRequest( + Map> pipelineSubstitutions, + Map> componentTemplateSubstitutions, + Map> indexTemplateSubstitutions, + Map mappingAddition + ) { super(); + Objects.requireNonNull(pipelineSubstitutions); + Objects.requireNonNull(componentTemplateSubstitutions); + Objects.requireNonNull(indexTemplateSubstitutions); + Objects.requireNonNull(mappingAddition); this.pipelineSubstitutions = pipelineSubstitutions; + this.componentTemplateSubstitutions = componentTemplateSubstitutions; + this.indexTemplateSubstitutions = indexTemplateSubstitutions; + this.mappingAddition = mappingAddition; } @SuppressWarnings("unchecked") public SimulateBulkRequest(StreamInput in) throws IOException { super(in); this.pipelineSubstitutions = (Map>) in.readGenericValue(); + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { + this.componentTemplateSubstitutions = (Map>) in.readGenericValue(); + this.indexTemplateSubstitutions = (Map>) in.readGenericValue(); + } else { + componentTemplateSubstitutions = Map.of(); + indexTemplateSubstitutions = Map.of(); + } + if (in.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_MAPPING_ADDITION)) { + this.mappingAddition = (Map) in.readGenericValue(); + } else { + mappingAddition = Map.of(); + } } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeGenericValue(pipelineSubstitutions); + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { + out.writeGenericValue(componentTemplateSubstitutions); + out.writeGenericValue(indexTemplateSubstitutions); + } + if (out.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_MAPPING_ADDITION)) { + out.writeGenericValue(mappingAddition); + } } public Map> getPipelineSubstitutions() { @@ -83,4 +170,60 @@ public Map> getPipelineSubstitutions() { public boolean isSimulated() { return true; } + + @Override + public Map getComponentTemplateSubstitutions() { + return componentTemplateSubstitutions.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> convertRawTemplateToComponentTemplate(entry.getValue()))); + } + + @Override + public Map getIndexTemplateSubstitutions() { + return indexTemplateSubstitutions.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> convertRawTemplateToIndexTemplate(entry.getValue()))); + } + + public Map getMappingAddition() { + return mappingAddition; + } + + private static ComponentTemplate convertRawTemplateToComponentTemplate(Map rawTemplate) { + ComponentTemplate componentTemplate; + try (var parser = XContentHelper.mapToXContentParser(XContentParserConfiguration.EMPTY, rawTemplate)) { + componentTemplate = ComponentTemplate.parse(parser); + } catch (IOException e) { + throw new RuntimeException(e); + } + return componentTemplate; + } + + private static ComposableIndexTemplate convertRawTemplateToIndexTemplate(Map rawTemplate) { + ComposableIndexTemplate indexTemplate; + try (var parser = XContentHelper.mapToXContentParser(XContentParserConfiguration.EMPTY, rawTemplate)) { + indexTemplate = ComposableIndexTemplate.parse(parser); + } catch (IOException e) { + throw new RuntimeException(e); + } + return indexTemplate; + } + + @Override + public BulkRequest shallowClone() { + BulkRequest bulkRequest = new SimulateBulkRequest( + pipelineSubstitutions, + componentTemplateSubstitutions, + indexTemplateSubstitutions, + mappingAddition + ); + bulkRequest.setRefreshPolicy(getRefreshPolicy()); + bulkRequest.waitForActiveShards(waitForActiveShards()); + bulkRequest.timeout(timeout()); + bulkRequest.pipeline(pipeline()); + bulkRequest.routing(routing()); + bulkRequest.requireAlias(requireAlias()); + bulkRequest.requireDataStream(requireDataStream()); + return bulkRequest; + } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java index 74864abe3ec50..24534826f8e3e 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; @@ -23,6 +24,8 @@ import org.elasticsearch.cluster.ClusterStateObserver; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.Writeable; @@ -38,6 +41,9 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import java.util.Objects; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; @@ -111,7 +117,12 @@ protected void doExecute(Task task, BulkRequest bulkRequest, ActionListener {}; + } else { + releasable = indexingPressure.markCoordinatingOperationStarted(indexingOps, indexingBytes, isOnlySystem); + } final ActionListener releasingListener = ActionListener.runBefore(listener, releasable::close); final Executor executor = isOnlySystem ? systemWriteExecutor : writeExecutor; ensureClusterStateThenForkAndExecute(task, bulkRequest, executor, releasingListener); @@ -162,19 +173,73 @@ public void onTimeout(TimeValue timeout) { private void forkAndExecute(Task task, BulkRequest bulkRequest, Executor executor, ActionListener releasingListener) { executor.execute(new ActionRunnable<>(releasingListener) { @Override - protected void doRun() { + protected void doRun() throws IOException { applyPipelinesAndDoInternalExecute(task, bulkRequest, executor, releasingListener); } }); } - private boolean applyPipelines(Task task, BulkRequest bulkRequest, Executor executor, ActionListener listener) { + private boolean applyPipelines(Task task, BulkRequest bulkRequest, Executor executor, ActionListener listener) + throws IOException { boolean hasIndexRequestsWithPipelines = false; - final Metadata metadata = clusterService.state().getMetadata(); + final Metadata metadata; + Map componentTemplateSubstitutions = bulkRequest.getComponentTemplateSubstitutions(); + Map indexTemplateSubstitutions = bulkRequest.getIndexTemplateSubstitutions(); + if (bulkRequest.isSimulated() + && (componentTemplateSubstitutions.isEmpty() == false || indexTemplateSubstitutions.isEmpty() == false)) { + /* + * If this is a simulated request, and there are template substitutions, then we want to create and use a new metadata that has + * those templates. That is, we want to add the new templates (which will replace any that already existed with the same name), + * and remove the indices and data streams that are referred to from the bulkRequest so that we get settings from the templates + * rather than from the indices/data streams. + */ + Metadata.Builder simulatedMetadataBuilder = Metadata.builder(clusterService.state().getMetadata()); + if (componentTemplateSubstitutions.isEmpty() == false) { + Map updatedComponentTemplates = new HashMap<>(); + updatedComponentTemplates.putAll(clusterService.state().metadata().componentTemplates()); + updatedComponentTemplates.putAll(componentTemplateSubstitutions); + simulatedMetadataBuilder.componentTemplates(updatedComponentTemplates); + } + if (indexTemplateSubstitutions.isEmpty() == false) { + Map updatedIndexTemplates = new HashMap<>(); + updatedIndexTemplates.putAll(clusterService.state().metadata().templatesV2()); + updatedIndexTemplates.putAll(indexTemplateSubstitutions); + simulatedMetadataBuilder.indexTemplates(updatedIndexTemplates); + } + /* + * We now remove the index from the simulated metadata to force the templates to be used. Note that simulated requests are + * always index requests -- no other type of request is supported. + */ + for (DocWriteRequest actionRequest : bulkRequest.requests) { + assert actionRequest != null : "Requests cannot be null in simulate mode"; + assert actionRequest instanceof IndexRequest + : "Only IndexRequests are supported in simulate mode, but got " + actionRequest.getClass(); + if (actionRequest != null) { + IndexRequest indexRequest = (IndexRequest) actionRequest; + String indexName = indexRequest.index(); + if (indexName != null) { + simulatedMetadataBuilder.remove(indexName); + simulatedMetadataBuilder.removeDataStream(indexName); + } + } + } + metadata = simulatedMetadataBuilder.build(); + } else { + metadata = clusterService.state().getMetadata(); + } + + Map resolvedPipelineCache = new HashMap<>(); for (DocWriteRequest actionRequest : bulkRequest.requests) { IndexRequest indexRequest = getIndexWriteRequest(actionRequest); if (indexRequest != null) { - IngestService.resolvePipelinesAndUpdateIndexRequest(actionRequest, indexRequest, metadata); + if (indexRequest.isPipelineResolved() == false) { + var pipeline = resolvedPipelineCache.computeIfAbsent( + indexRequest.index(), + // TODO perhaps this should use `threadPool.absoluteTimeInMillis()`, but leaving as is for now. + (index) -> IngestService.resolvePipelines(actionRequest, indexRequest, metadata, System.currentTimeMillis()) + ); + IngestService.setPipelineOnRequest(indexRequest, pipeline); + } hasIndexRequestsWithPipelines |= IngestService.hasPipeline(indexRequest); } @@ -244,7 +309,7 @@ private void processBulkIndexIngestRequest( } else { ActionRunnable runnable = new ActionRunnable<>(actionListener) { @Override - protected void doRun() { + protected void doRun() throws IOException { applyPipelinesAndDoInternalExecute(task, bulkRequest, executor, actionListener); } @@ -278,9 +343,11 @@ public boolean isForceExecution() { * @param indexName The index name to check. * @param metadata Cluster state metadata. * @param epochMillis A timestamp to use when resolving date math in the index name. - * @return true if this is not a simulation, and the given index name corresponds to a data stream with a failure store - * or if it matches a template that has a data stream failure store enabled. Returns false if the index name corresponds to a - * data stream, but it doesn't have the failure store enabled. Returns null when it doesn't correspond to a data stream. + * @return true if this is not a simulation, and the given index name corresponds to a data stream with a failure store, or if it + * matches a template that has a data stream failure store enabled, or if it matches a data stream template with no failure store + * option specified and the name matches the cluster setting to enable the failure store. Returns false if the index name + * corresponds to a data stream, but it doesn't have the failure store enabled by one of those conditions. Returns null when it + * doesn't correspond to a data stream. */ protected abstract Boolean resolveFailureStore(String indexName, Metadata metadata, long epochMillis); @@ -322,7 +389,7 @@ private void applyPipelinesAndDoInternalExecute( BulkRequest bulkRequest, Executor executor, ActionListener listener - ) { + ) throws IOException { final long relativeStartTimeNanos = relativeTimeNanos(); if (applyPipelines(task, bulkRequest, executor, listener) == false) { doInternalExecute(task, bulkRequest, executor, listener, relativeStartTimeNanos); @@ -343,6 +410,6 @@ protected abstract void doInternalExecute( Executor executor, ActionListener listener, long relativeStartTimeNanos - ); + ) throws IOException; } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index bdda4ff487f6b..65264faf50129 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; @@ -16,7 +17,6 @@ import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.DocWriteRequest; -import org.elasticsearch.action.DocWriteRequest.OpType; import org.elasticsearch.action.admin.indices.create.AutoCreateAction; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; @@ -34,6 +34,8 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings; +import org.elasticsearch.cluster.metadata.DataStreamOptions; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; @@ -41,8 +43,8 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.concurrent.AtomicArray; +import org.elasticsearch.core.Nullable; import org.elasticsearch.features.FeatureService; -import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.index.VersionType; import org.elasticsearch.indices.SystemIndices; @@ -52,12 +54,14 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.io.IOException; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.SortedMap; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; import java.util.function.Function; import java.util.function.LongSupplier; @@ -81,6 +85,7 @@ public class TransportBulkAction extends TransportAbstractBulkAction { private final IndexNameExpressionResolver indexNameExpressionResolver; private final OriginSettingClient rolloverClient; private final FailureStoreMetrics failureStoreMetrics; + private final DataStreamFailureStoreSettings dataStreamFailureStoreSettings; @Inject public TransportBulkAction( @@ -94,7 +99,8 @@ public TransportBulkAction( IndexNameExpressionResolver indexNameExpressionResolver, IndexingPressure indexingPressure, SystemIndices systemIndices, - FailureStoreMetrics failureStoreMetrics + FailureStoreMetrics failureStoreMetrics, + DataStreamFailureStoreSettings dataStreamFailureStoreSettings ) { this( threadPool, @@ -108,7 +114,8 @@ public TransportBulkAction( indexingPressure, systemIndices, threadPool::relativeTimeInNanos, - failureStoreMetrics + failureStoreMetrics, + dataStreamFailureStoreSettings ); } @@ -124,7 +131,8 @@ public TransportBulkAction( IndexingPressure indexingPressure, SystemIndices systemIndices, LongSupplier relativeTimeProvider, - FailureStoreMetrics failureStoreMetrics + FailureStoreMetrics failureStoreMetrics, + DataStreamFailureStoreSettings dataStreamFailureStoreSettings ) { this( TYPE, @@ -140,7 +148,8 @@ public TransportBulkAction( indexingPressure, systemIndices, relativeTimeProvider, - failureStoreMetrics + failureStoreMetrics, + dataStreamFailureStoreSettings ); } @@ -158,7 +167,8 @@ public TransportBulkAction( IndexingPressure indexingPressure, SystemIndices systemIndices, LongSupplier relativeTimeProvider, - FailureStoreMetrics failureStoreMetrics + FailureStoreMetrics failureStoreMetrics, + DataStreamFailureStoreSettings dataStreamFailureStoreSettings ) { super( bulkAction, @@ -172,6 +182,7 @@ public TransportBulkAction( systemIndices, relativeTimeProvider ); + this.dataStreamFailureStoreSettings = dataStreamFailureStoreSettings; Objects.requireNonNull(relativeTimeProvider); this.featureService = featureService; this.client = client; @@ -191,7 +202,11 @@ public static ActionListe final Response response = (Response) bulkItemResponse.getResponse(); l.onResponse(response); } else { - l.onFailure(bulkItemResponse.getFailure().getCause()); + if (IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN.equals(bulkItemResponse.getFailure().getFailureStoreStatus())) { + l.onFailure(bulkItemResponse.getFailure().getCause()); + } else { + l.onFailure(new IndexDocFailureStoreStatus.ExceptionWithFailureStoreStatus(bulkItemResponse.getFailure())); + } } }); } @@ -203,9 +218,12 @@ protected void doInternalExecute( Executor executor, ActionListener listener, long relativeStartTimeNanos - ) { + ) throws IOException { + assert (bulkRequest instanceof SimulateBulkRequest) == false + : "TransportBulkAction should never be called with a SimulateBulkRequest"; + assert bulkRequest.getComponentTemplateSubstitutions().isEmpty() + : "Component template substitutions are not allowed in a non-simulated bulk"; trackIndexRequests(bulkRequest); - Map indicesToAutoCreate = new HashMap<>(); Set dataStreamsToBeRolledOver = new HashSet<>(); Set failureStoresToBeRolledOver = new HashSet<>(); @@ -271,7 +289,7 @@ private void populateMissingTargets( for (DocWriteRequest request : bulkRequest.requests) { // Delete requests should not attempt to create the index (if the index does not exist), unless an external versioning is used. - if (request.opType() == OpType.DELETE + if (request.opType() == DocWriteRequest.OpType.DELETE && request.versionType() != VersionType.EXTERNAL && request.versionType() != VersionType.EXTERNAL_GTE) { continue; @@ -342,29 +360,36 @@ protected void createMissingIndicesAndIndexData( final AtomicArray responses = new AtomicArray<>(bulkRequest.requests.size()); // Optimizing when there are no prerequisite actions if (indicesToAutoCreate.isEmpty() && dataStreamsToBeRolledOver.isEmpty() && failureStoresToBeRolledOver.isEmpty()) { - executeBulk(task, bulkRequest, startTimeNanos, listener, executor, responses, Map.of()); + executeBulk(task, bulkRequest, startTimeNanos, listener, executor, responses); return; } - final Map indicesThatCannotBeCreated = new HashMap<>(); + Map indicesExceptions = new ConcurrentHashMap<>(); + Map dataStreamExceptions = new ConcurrentHashMap<>(); + Map failureStoreExceptions = new ConcurrentHashMap<>(); Runnable executeBulkRunnable = () -> executor.execute(new ActionRunnable<>(listener) { @Override protected void doRun() { - executeBulk(task, bulkRequest, startTimeNanos, listener, executor, responses, indicesThatCannotBeCreated); + failRequestsWhenPrerequisiteActionFailed( + indicesExceptions, + dataStreamExceptions, + failureStoreExceptions, + bulkRequest, + responses + ); + executeBulk(task, bulkRequest, startTimeNanos, listener, executor, responses); } }); try (RefCountingRunnable refs = new RefCountingRunnable(executeBulkRunnable)) { - createIndices(bulkRequest, indicesToAutoCreate, indicesThatCannotBeCreated, responses, refs); - rollOverDataStreams(bulkRequest, dataStreamsToBeRolledOver, false, responses, refs); - rollOverDataStreams(bulkRequest, failureStoresToBeRolledOver, true, responses, refs); + createIndices(indicesToAutoCreate, refs, indicesExceptions); + rollOverDataStreams(bulkRequest, dataStreamsToBeRolledOver, false, refs, dataStreamExceptions); + rollOverDataStreams(bulkRequest, failureStoresToBeRolledOver, true, refs, failureStoreExceptions); } } private void createIndices( - BulkRequest bulkRequest, Map indicesToAutoCreate, - Map indicesThatCannotBeCreated, - AtomicArray responses, - RefCountingRunnable refs + RefCountingRunnable refs, + final Map indicesExceptions ) { for (Map.Entry indexEntry : indicesToAutoCreate.entrySet()) { final String index = indexEntry.getKey(); @@ -375,25 +400,26 @@ public void onResponse(CreateIndexResponse createIndexResponse) {} @Override public void onFailure(Exception e) { final Throwable cause = ExceptionsHelper.unwrapCause(e); - if (cause instanceof IndexNotFoundException indexNotFoundException) { - synchronized (indicesThatCannotBeCreated) { - indicesThatCannotBeCreated.put(index, indexNotFoundException); - } - } else if ((cause instanceof ResourceAlreadyExistsException) == false) { + if ((cause instanceof ResourceAlreadyExistsException) == false) { // fail all requests involving this index, if create didn't work - failRequestsWhenPrerequisiteActionFailed(index, bulkRequest, responses, e); + indicesExceptions.put(index, e); } } }, refs.acquire())); } } + // Separate method to allow for overriding in tests. + void createIndex(CreateIndexRequest createIndexRequest, ActionListener listener) { + client.execute(AutoCreateAction.INSTANCE, createIndexRequest, listener); + } + private void rollOverDataStreams( BulkRequest bulkRequest, Set dataStreamsToBeRolledOver, boolean targetFailureStore, - AtomicArray responses, - RefCountingRunnable refs + RefCountingRunnable refs, + Map dataStreamExceptions ) { for (String dataStream : dataStreamsToBeRolledOver) { RolloverRequest rolloverRequest = new RolloverRequest(dataStream, null); @@ -401,13 +427,13 @@ private void rollOverDataStreams( if (targetFailureStore) { rolloverRequest.setIndicesOptions( IndicesOptions.builder(rolloverRequest.indicesOptions()) - .failureStoreOptions(new IndicesOptions.FailureStoreOptions(false, true)) + .selectorOptions(IndicesOptions.SelectorOptions.FAILURES) .build() ); } // We are executing a lazy rollover because it is an action specialised for this situation, when we want an // unconditional and performant rollover. - rolloverClient.execute(LazyRolloverAction.INSTANCE, rolloverRequest, ActionListener.releaseAfter(new ActionListener<>() { + rollOver(rolloverRequest, ActionListener.releaseAfter(new ActionListener<>() { @Override public void onResponse(RolloverResponse result) { @@ -422,32 +448,58 @@ public void onResponse(RolloverResponse result) { @Override public void onFailure(Exception e) { - failRequestsWhenPrerequisiteActionFailed(dataStream, bulkRequest, responses, e); + dataStreamExceptions.put(dataStream, e); } }, refs.acquire())); } } + // Separate method to allow for overriding in tests. + void rollOver(RolloverRequest rolloverRequest, ActionListener listener) { + rolloverClient.execute(LazyRolloverAction.INSTANCE, rolloverRequest, listener); + } + /** - * Fails all requests involving this index or data stream because the prerequisite action failed too. + * Mark all the requests for which the prerequisite action failed (i.e. index creation or data stream/failure store rollover) as failed. */ - private static void failRequestsWhenPrerequisiteActionFailed( - String target, + private void failRequestsWhenPrerequisiteActionFailed( + Map indicesExceptions, + Map dataStreamExceptions, + Map failureStoreExceptions, BulkRequest bulkRequest, - AtomicArray responses, - Exception error + AtomicArray responses ) { + if (indicesExceptions.isEmpty() && dataStreamExceptions.isEmpty() && failureStoreExceptions.isEmpty()) { + return; + } for (int i = 0; i < bulkRequest.requests.size(); i++) { DocWriteRequest request = bulkRequest.requests.get(i); - if (request != null && setResponseFailureIfIndexMatches(responses, i, request, target, error)) { - bulkRequest.requests.set(i, null); + if (request == null) { + continue; } + var exception = indicesExceptions.get(request.index()); + if (exception == null) { + if (request instanceof IndexRequest indexRequest && indexRequest.isWriteToFailureStore()) { + exception = failureStoreExceptions.get(request.index()); + } else { + exception = dataStreamExceptions.get(request.index()); + } + } + if (exception == null) { + continue; + } + var failureStoreStatus = request instanceof IndexRequest ir && ir.isWriteToFailureStore() + ? IndexDocFailureStoreStatus.FAILED + : IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN; + var failure = new BulkItemResponse.Failure(request.index(), request.id(), exception, failureStoreStatus); + responses.set(i, BulkItemResponse.failure(i, request.opType(), failure)); + bulkRequest.requests.set(i, null); } } static void prohibitAppendWritesInBackingIndices(DocWriteRequest writeRequest, IndexAbstraction indexAbstraction) { DocWriteRequest.OpType opType = writeRequest.opType(); - if ((opType == OpType.CREATE || opType == OpType.INDEX) == false) { + if ((opType == DocWriteRequest.OpType.CREATE || opType == DocWriteRequest.OpType.INDEX) == false) { // op type not create or index, then bail early return; } @@ -523,33 +575,13 @@ private static boolean isSystemIndex(SortedMap indices } } - void createIndex(CreateIndexRequest createIndexRequest, ActionListener listener) { - client.execute(AutoCreateAction.INSTANCE, createIndexRequest, listener); - } - - private static boolean setResponseFailureIfIndexMatches( - AtomicArray responses, - int idx, - DocWriteRequest request, - String index, - Exception e - ) { - if (index.equals(request.index())) { - BulkItemResponse.Failure failure = new BulkItemResponse.Failure(request.index(), request.id(), e); - responses.set(idx, BulkItemResponse.failure(idx, request.opType(), failure)); - return true; - } - return false; - } - void executeBulk( Task task, BulkRequest bulkRequest, long startTimeNanos, ActionListener listener, Executor executor, - AtomicArray responses, - Map indicesThatCannotBeCreated + AtomicArray responses ) { new BulkOperation( task, @@ -559,12 +591,12 @@ void executeBulk( bulkRequest, client, responses, - indicesThatCannotBeCreated, indexNameExpressionResolver, relativeTimeNanosProvider, startTimeNanos, listener, - failureStoreMetrics + failureStoreMetrics, + dataStreamFailureStoreSettings ).run(); } @@ -572,7 +604,7 @@ void executeBulk( * See {@link #resolveFailureStore(String, Metadata, long)} */ // Visibility for testing - static Boolean resolveFailureInternal(String indexName, Metadata metadata, long epochMillis) { + Boolean resolveFailureInternal(String indexName, Metadata metadata, long epochMillis) { if (DataStream.isFailureStoreFeatureFlagEnabled() == false) { return null; } @@ -580,7 +612,7 @@ static Boolean resolveFailureInternal(String indexName, Metadata metadata, long if (resolution != null) { return resolution; } - return resolveFailureStoreFromTemplate(indexName, metadata); + return resolveFailureStoreFromTemplate(indexName, metadata, epochMillis); } @Override @@ -595,7 +627,7 @@ protected Boolean resolveFailureStore(String indexName, Metadata metadata, long * @param epochMillis A timestamp to use when resolving date math in the index name. * @return true if the given index name corresponds to an existing data stream with a failure store enabled. */ - private static Boolean resolveFailureStoreFromMetadata(String indexName, Metadata metadata, long epochMillis) { + private Boolean resolveFailureStoreFromMetadata(String indexName, Metadata metadata, long epochMillis) { if (indexName == null) { return null; } @@ -612,29 +644,44 @@ private static Boolean resolveFailureStoreFromMetadata(String indexName, Metadat DataStream targetDataStream = DataStream.resolveDataStream(indexAbstraction, metadata); // We will store the failure if the write target belongs to a data stream with a failure store. - return targetDataStream != null && targetDataStream.isFailureStoreEnabled(); + return targetDataStream != null && targetDataStream.isFailureStoreEffectivelyEnabled(dataStreamFailureStoreSettings); } /** - * Determines if an index name is associated with an index template that has a data stream failure store enabled. + * Determines if an index name is associated with an index template that has a data stream failure store enabled. Since failure store is + * a data stream feature, the method returns true/false only if it is a data stream template, otherwise null. * @param indexName The index name to check. * @param metadata Cluster state metadata. - * @return true if the given index name corresponds to an index template with a data stream failure store enabled. + * @param epochMillis A timestamp to use when resolving date math in the index name. + * @return true the associated index template has failure store enabled, false if the failure store is disabled, true or false according + * to the cluster setting if there is a data stream template with no failure store option specified, and null if no template is + * found or if the template is not a data stream template. */ - private static Boolean resolveFailureStoreFromTemplate(String indexName, Metadata metadata) { + @Nullable + private Boolean resolveFailureStoreFromTemplate(String indexName, Metadata metadata, long epochMillis) { if (indexName == null) { return null; } // Check to see if the index name matches any templates such that an index would have been attributed // We don't check v1 templates at all because failure stores can only exist on data streams via a v2 template + // N.B. This currently does date math resolution itself and does *not* use epochMillis (it gets the system time again) String template = MetadataIndexTemplateService.findV2Template(metadata, indexName, false); if (template != null) { // Check if this is a data stream template or if it is just a normal index. ComposableIndexTemplate composableIndexTemplate = metadata.templatesV2().get(template); if (composableIndexTemplate.getDataStreamTemplate() != null) { // Check if the data stream has the failure store enabled - return composableIndexTemplate.getDataStreamTemplate().hasFailureStore(); + DataStreamOptions dataStreamOptions = MetadataIndexTemplateService.resolveDataStreamOptions( + composableIndexTemplate, + metadata.componentTemplates() + ).mapAndGet(DataStreamOptions.Template::toDataStreamOptions); + return DataStream.isFailureStoreEffectivelyEnabled( + dataStreamOptions, + dataStreamFailureStoreSettings, + IndexNameExpressionResolver.resolveDateMathExpression(indexName, epochMillis), + systemIndices + ); } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index ac9ceb44e5b76..89cee714a9ff2 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; @@ -47,8 +48,11 @@ import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.get.GetResult; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper; import org.elasticsearch.index.mapper.MapperException; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MappingLookup; +import org.elasticsearch.index.mapper.RoutingFieldMapper; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.IndexShard; @@ -324,7 +328,8 @@ static boolean executeBulkItemRequest( if (opType == DocWriteRequest.OpType.UPDATE) { final UpdateRequest updateRequest = (UpdateRequest) context.getCurrent(); try { - updateResult = updateHelper.prepare(updateRequest, context.getPrimary(), nowInMillisSupplier); + var gFields = getStoredFieldsSpec(context.getPrimary()); + updateResult = updateHelper.prepare(updateRequest, context.getPrimary(), nowInMillisSupplier, gFields); } catch (Exception failure) { // we may fail translating a update to index or delete operation // we use index result to communicate failure while translating update request @@ -399,6 +404,16 @@ static boolean executeBulkItemRequest( return true; } + private static String[] getStoredFieldsSpec(IndexShard indexShard) { + if (InferenceMetadataFieldsMapper.isEnabled(indexShard.mapperService().mappingLookup())) { + if (indexShard.mapperService().mappingLookup().inferenceFields().size() > 0) { + // Retrieves the inference metadata field containing the inference results for all semantic fields defined in the mapping. + return new String[] { RoutingFieldMapper.NAME, InferenceMetadataFieldsMapper.NAME }; + } + } + return new String[] { RoutingFieldMapper.NAME }; + } + private static boolean handleMappingUpdateRequired( BulkPrimaryExecutionContext context, MappingUpdatePerformer mappingUpdater, @@ -479,7 +494,17 @@ && isConflictException(executionResult.getFailure().getCause()) } final BulkItemResponse response; if (isUpdate) { - response = processUpdateResponse((UpdateRequest) docWriteRequest, context.getConcreteIndex(), executionResult, updateResult); + assert context.getPrimary().mapperService() != null; + final MappingLookup mappingLookup = context.getPrimary().mapperService().mappingLookup(); + assert mappingLookup != null; + + response = processUpdateResponse( + (UpdateRequest) docWriteRequest, + context.getConcreteIndex(), + mappingLookup, + executionResult, + updateResult + ); } else { if (isFailed) { final Exception failure = executionResult.getFailure().getCause(); @@ -517,6 +542,7 @@ private static boolean isConflictException(final Exception e) { private static BulkItemResponse processUpdateResponse( final UpdateRequest updateRequest, final String concreteIndex, + final MappingLookup mappingLookup, BulkItemResponse operationResponse, final UpdateHelper.Result translate ) { @@ -554,6 +580,7 @@ private static BulkItemResponse processUpdateResponse( UpdateHelper.extractGetResult( updateRequest, concreteIndex, + mappingLookup, indexResponse.getSeqNo(), indexResponse.getPrimaryTerm(), indexResponse.getVersion(), @@ -578,6 +605,7 @@ private static BulkItemResponse processUpdateResponse( final GetResult getResult = UpdateHelper.extractGetResult( updateRequest, concreteIndex, + mappingLookup, deleteResponse.getSeqNo(), deleteResponse.getPrimaryTerm(), deleteResponse.getVersion(), diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index 8da6fb409cb90..cb43e07faa612 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -1,13 +1,16 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.IndexableField; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.admin.indices.template.post.TransportSimulateIndexTemplateAction; @@ -15,6 +18,8 @@ import org.elasticsearch.action.ingest.SimulateIndexResponse; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; @@ -23,10 +28,14 @@ import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AtomicArray; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.Tuple; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexSettingProvider; import org.elasticsearch.index.IndexSettingProviders; @@ -34,6 +43,9 @@ import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.mapper.IgnoredFieldMapper; +import org.elasticsearch.index.mapper.LuceneDocument; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.IndexShard; @@ -47,7 +59,14 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -66,6 +85,13 @@ public class TransportSimulateBulkAction extends TransportAbstractBulkAction { public static final NodeFeature SIMULATE_MAPPING_VALIDATION = new NodeFeature("simulate.mapping.validation"); public static final NodeFeature SIMULATE_MAPPING_VALIDATION_TEMPLATES = new NodeFeature("simulate.mapping.validation.templates"); + public static final NodeFeature SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS = new NodeFeature( + "simulate.component.template.substitutions" + ); + public static final NodeFeature SIMULATE_INDEX_TEMPLATE_SUBSTITUTIONS = new NodeFeature("simulate.index.template.substitutions"); + public static final NodeFeature SIMULATE_MAPPING_ADDITION = new NodeFeature("simulate.mapping.addition"); + public static final NodeFeature SIMULATE_SUPPORT_NON_TEMPLATE_MAPPING = new NodeFeature("simulate.support.non.template.mapping"); + public static final NodeFeature SIMULATE_IGNORED_FIELDS = new NodeFeature("simulate.ignored.fields"); private final IndicesService indicesService; private final NamedXContentRegistry xContentRegistry; private final Set indexSettingProviders; @@ -107,13 +133,24 @@ protected void doInternalExecute( Executor executor, ActionListener listener, long relativeStartTimeNanos - ) { + ) throws IOException { + assert bulkRequest instanceof SimulateBulkRequest + : "TransportSimulateBulkAction should only ever be called with a SimulateBulkRequest but got a " + bulkRequest.getClass(); final AtomicArray responses = new AtomicArray<>(bulkRequest.requests.size()); + Map componentTemplateSubstitutions = bulkRequest.getComponentTemplateSubstitutions(); + Map indexTemplateSubstitutions = bulkRequest.getIndexTemplateSubstitutions(); + Map mappingAddition = ((SimulateBulkRequest) bulkRequest).getMappingAddition(); for (int i = 0; i < bulkRequest.requests.size(); i++) { DocWriteRequest docRequest = bulkRequest.requests.get(i); assert docRequest instanceof IndexRequest : "TransportSimulateBulkAction should only ever be called with IndexRequests"; IndexRequest request = (IndexRequest) docRequest; - Exception mappingValidationException = validateMappings(request); + Tuple, Exception> validationResult = validateMappings( + componentTemplateSubstitutions, + indexTemplateSubstitutions, + mappingAddition, + request + ); + Exception mappingValidationException = validationResult.v2(); responses.set( i, BulkItemResponse.success( @@ -126,6 +163,7 @@ protected void doInternalExecute( request.source(), request.getContentType(), request.getExecutedPipelines(), + validationResult.v1(), mappingValidationException ) ) @@ -139,10 +177,17 @@ protected void doInternalExecute( /** * This creates a temporary index with the mappings of the index in the request, and then attempts to index the source from the request * into it. If there is a mapping exception, that exception is returned. On success the returned exception is null. + * @param componentTemplateSubstitutions The component template definitions to use in place of existing ones for validation * @param request The IndexRequest whose source will be validated against the mapping (if it exists) of its index - * @return a mapping exception if the source does not match the mappings, otherwise null + * @return a Tuple containing: (1) in v1 the names of any fields that would be ignored upon indexing and (2) in v2 the mapping + * exception if the source does not match the mappings, otherwise null */ - private Exception validateMappings(IndexRequest request) { + private Tuple, Exception> validateMappings( + Map componentTemplateSubstitutions, + Map indexTemplateSubstitutions, + Map mappingAddition, + IndexRequest request + ) { final SourceToParse sourceToParse = new SourceToParse( request.id(), request.source(), @@ -154,9 +199,17 @@ private Exception validateMappings(IndexRequest request) { ClusterState state = clusterService.state(); Exception mappingValidationException = null; + Collection ignoredFields = List.of(); IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(request.index()); try { - if (indexAbstraction != null) { + if (indexAbstraction != null + && componentTemplateSubstitutions.isEmpty() + && indexTemplateSubstitutions.isEmpty() + && mappingAddition.isEmpty()) { + /* + * In this case the index exists and we don't have any component template overrides. So we can just use withTempIndexService + * to do the mapping validation, using all the existing logic for validation. + */ IndexMetadata imd = state.metadata().getIndexSafe(indexAbstraction.getWriteIndex(request, state.metadata())); indicesService.withTempIndexService(imd, indexService -> { indexService.mapperService().updateMapping(null, imd); @@ -177,18 +230,54 @@ private Exception validateMappings(IndexRequest request) { }); } else { /* - * The index did not exist, so we put together the mappings from existing templates. - * This reproduces a lot of the mapping resolution logic in MetadataCreateIndexService.applyCreateIndexRequest(). However, - * it does not deal with aliases (since an alias cannot be created if an index does not exist, and this is the path for - * when the index does not exist). And it does not deal with system indices since we do not intend for users to simulate - * writing to system indices. + * The index did not exist, or we have component template substitutions, so we put together the mappings from existing + * templates This reproduces a lot of the mapping resolution logic in MetadataCreateIndexService.applyCreateIndexRequest(). + * However, it does not deal with aliases (since an alias cannot be created if an index does not exist, and this is the + * path for when the index does not exist). And it does not deal with system indices since we do not intend for users to + * simulate writing to system indices. */ - String matchingTemplate = findV2Template(state.metadata(), request.index(), false); + ClusterState.Builder simulatedClusterStateBuilder = new ClusterState.Builder(state); + Metadata.Builder simulatedMetadata = Metadata.builder(state.metadata()); + if (indexAbstraction != null) { + /* + * We remove the index or data stream from the cluster state so that we are forced to fall back to the templates to get + * mappings. + */ + String indexRequest = request.index(); + assert indexRequest != null : "Index requests cannot be null in a simulate bulk call"; + if (indexRequest != null) { + simulatedMetadata.remove(indexRequest); + simulatedMetadata.removeDataStream(indexRequest); + } + } + if (componentTemplateSubstitutions.isEmpty() == false) { + /* + * We put the template substitutions into the cluster state. If they have the same name as an existing one, the + * existing one is replaced. + */ + Map updatedComponentTemplates = new HashMap<>(); + updatedComponentTemplates.putAll(state.metadata().componentTemplates()); + updatedComponentTemplates.putAll(componentTemplateSubstitutions); + simulatedMetadata.componentTemplates(updatedComponentTemplates); + } + if (indexTemplateSubstitutions.isEmpty() == false) { + Map updatedIndexTemplates = new HashMap<>(); + updatedIndexTemplates.putAll(state.metadata().templatesV2()); + updatedIndexTemplates.putAll(indexTemplateSubstitutions); + simulatedMetadata.indexTemplates(updatedIndexTemplates); + } + ClusterState simulatedState = simulatedClusterStateBuilder.metadata(simulatedMetadata).build(); + + String matchingTemplate = findV2Template(simulatedState.metadata(), request.index(), false); if (matchingTemplate != null) { + /* + * The index matches a v2 template (including possibly one or more of the substitutions passed in). So we use this + * template, and then possibly apply the mapping addition if it is not null, and validate. + */ final Template template = TransportSimulateIndexTemplateAction.resolveTemplate( matchingTemplate, request.index(), - state, + simulatedState, isDataStreamsLifecycleOnlyMode(clusterService.getSettings()), xContentRegistry, indicesService, @@ -196,83 +285,125 @@ private Exception validateMappings(IndexRequest request) { indexSettingProviders ); CompressedXContent mappings = template.mappings(); - if (mappings != null) { - MappingMetadata mappingMetadata = new MappingMetadata(mappings); - Settings dummySettings = Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) - .build(); - final IndexMetadata imd = IndexMetadata.builder(request.index()) - .settings(dummySettings) - .putMapping(mappingMetadata) - .build(); - indicesService.withTempIndexService(imd, indexService -> { - indexService.mapperService().updateMapping(null, imd); - return IndexShard.prepareIndex( - indexService.mapperService(), - sourceToParse, - SequenceNumbers.UNASSIGNED_SEQ_NO, - -1, - -1, - VersionType.INTERNAL, - Engine.Operation.Origin.PRIMARY, - Long.MIN_VALUE, - false, - request.ifSeqNo(), - request.ifPrimaryTerm(), - 0 - ); - }); - } + CompressedXContent mergedMappings = mergeMappings(mappings, mappingAddition); + ignoredFields = validateUpdatedMappings(mappings, mergedMappings, request, sourceToParse); } else { - List matchingTemplates = findV1Templates(state.metadata(), request.index(), false); - final Map mappingsMap = MetadataCreateIndexService.parseV1Mappings( - "{}", - matchingTemplates.stream().map(IndexTemplateMetadata::getMappings).collect(toList()), - xContentRegistry - ); - final CompressedXContent combinedMappings; - if (mappingsMap.isEmpty()) { - combinedMappings = null; + List matchingTemplates = findV1Templates(simulatedState.metadata(), request.index(), false); + if (matchingTemplates.isEmpty() == false) { + /* + * The index matches v1 mappings. These are not compatible with component_template_substitutions or + * index_template_substitutions, but we can apply a mapping_addition. + */ + final Map mappingsMap = MetadataCreateIndexService.parseV1Mappings( + "{}", + matchingTemplates.stream().map(IndexTemplateMetadata::getMappings).collect(toList()), + xContentRegistry + ); + final CompressedXContent combinedMappings = mergeMappings(new CompressedXContent(mappingsMap), mappingAddition); + ignoredFields = validateUpdatedMappings(null, combinedMappings, request, sourceToParse); + } else if (indexAbstraction != null && mappingAddition.isEmpty() == false) { + /* + * The index matched no templates of any kind, including the substitutions. But it might have a mapping. So we + * merge in the mapping addition if it exists, and validate. + */ + MappingMetadata mappingFromIndex = clusterService.state().metadata().index(indexAbstraction.getName()).mapping(); + CompressedXContent currentIndexCompressedXContent = mappingFromIndex == null ? null : mappingFromIndex.source(); + CompressedXContent combinedMappings = mergeMappings(currentIndexCompressedXContent, mappingAddition); + ignoredFields = validateUpdatedMappings(null, combinedMappings, request, sourceToParse); } else { - combinedMappings = new CompressedXContent(mappingsMap); + /* + * The index matched no templates and had no mapping of its own. If there were component template substitutions + * or index template substitutions, they didn't match anything. So just apply the mapping addition if it exists, + * and validate. + */ + final CompressedXContent combinedMappings = mergeMappings(null, mappingAddition); + ignoredFields = validateUpdatedMappings(null, combinedMappings, request, sourceToParse); } - Settings dummySettings = Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) - .build(); - MappingMetadata mappingMetadata = combinedMappings == null ? null : new MappingMetadata(combinedMappings); - final IndexMetadata imd = IndexMetadata.builder(request.index()) - .putMapping(mappingMetadata) - .settings(dummySettings) - .build(); - indicesService.withTempIndexService(imd, indexService -> { - indexService.mapperService().updateMapping(null, imd); - return IndexShard.prepareIndex( - indexService.mapperService(), - sourceToParse, - SequenceNumbers.UNASSIGNED_SEQ_NO, - -1, - -1, - VersionType.INTERNAL, - Engine.Operation.Origin.PRIMARY, - Long.MIN_VALUE, - false, - request.ifSeqNo(), - request.ifPrimaryTerm(), - 0 - ); - }); } } } catch (Exception e) { mappingValidationException = e; } - return mappingValidationException; + return Tuple.tuple(ignoredFields, mappingValidationException); + } + + /* + * Validates that when updatedMappings are applied. If any fields would be ignored while indexing, then those field names are returned. + * Otherwise the returned Collection is empty. + */ + private Collection validateUpdatedMappings( + @Nullable CompressedXContent originalMappings, + @Nullable CompressedXContent updatedMappings, + IndexRequest request, + SourceToParse sourceToParse + ) throws IOException { + if (updatedMappings == null) { + return List.of(); // no validation to do + } + Settings dummySettings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) + .build(); + IndexMetadata.Builder originalIndexMetadataBuilder = IndexMetadata.builder(request.index()).settings(dummySettings); + if (originalMappings != null) { + originalIndexMetadataBuilder.putMapping(new MappingMetadata(originalMappings)); + } + final IndexMetadata originalIndexMetadata = originalIndexMetadataBuilder.build(); + final IndexMetadata updatedIndexMetadata = IndexMetadata.builder(request.index()) + .settings(dummySettings) + .putMapping(new MappingMetadata(updatedMappings)) + .build(); + Engine.Index result = indicesService.withTempIndexService(originalIndexMetadata, indexService -> { + indexService.mapperService().merge(updatedIndexMetadata, MapperService.MergeReason.MAPPING_UPDATE); + return IndexShard.prepareIndex( + indexService.mapperService(), + sourceToParse, + SequenceNumbers.UNASSIGNED_SEQ_NO, + -1, + -1, + VersionType.INTERNAL, + Engine.Operation.Origin.PRIMARY, + Long.MIN_VALUE, + false, + request.ifSeqNo(), + request.ifPrimaryTerm(), + 0 + ); + }); + final Collection ignoredFields; + if (result == null) { + ignoredFields = List.of(); + } else { + List luceneDocuments = result.parsedDoc().docs(); + assert luceneDocuments == null || luceneDocuments.size() == 1 : "Expected a single lucene document from index attempt"; + if (luceneDocuments != null && luceneDocuments.size() == 1) { + ignoredFields = luceneDocuments.get(0) + .getFields() + .stream() + .filter(field -> field.name().equals(IgnoredFieldMapper.NAME) && field instanceof StringField) + .map(IndexableField::stringValue) + .toList(); + } else { + ignoredFields = List.of(); + } + } + return ignoredFields; + } + + private static CompressedXContent mergeMappings(@Nullable CompressedXContent originalMapping, Map mappingAddition) + throws IOException { + Map combinedMappingMap = new HashMap<>(); + if (originalMapping != null) { + combinedMappingMap.putAll(XContentHelper.convertToMap(originalMapping.uncompressed(), true, XContentType.JSON).v2()); + } + XContentHelper.update(combinedMappingMap, mappingAddition, true); + if (combinedMappingMap.isEmpty()) { + return null; + } else { + return convertMappingMapToXContent(combinedMappingMap); + } } /* @@ -290,4 +421,25 @@ protected Boolean resolveFailureStore(String indexName, Metadata metadata, long // A simulate bulk request should not change any persistent state in the system, so we never write to the failure store return null; } + + private static CompressedXContent convertMappingMapToXContent(Map rawAdditionalMapping) throws IOException { + CompressedXContent compressedXContent; + if (rawAdditionalMapping == null || rawAdditionalMapping.isEmpty()) { + compressedXContent = null; + } else { + try (var parser = XContentHelper.mapToXContentParser(XContentParserConfiguration.EMPTY, rawAdditionalMapping)) { + compressedXContent = mappingFromXContent(parser); + } + } + return compressedXContent; + } + + private static CompressedXContent mappingFromXContent(XContentParser parser) throws IOException { + XContentParser.Token token = parser.nextToken(); + if (token == XContentParser.Token.START_OBJECT) { + return new CompressedXContent(Strings.toString(XContentFactory.jsonBuilder().map(parser.mapOrdered()))); + } else { + throw new IllegalArgumentException("Unexpected token: " + token); + } + } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSingleItemBulkWriteAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSingleItemBulkWriteAction.java index 8853e2da3abc4..9e8636609cb4e 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSingleItemBulkWriteAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSingleItemBulkWriteAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; diff --git a/server/src/main/java/org/elasticsearch/action/bulk/WriteAckDelay.java b/server/src/main/java/org/elasticsearch/action/bulk/WriteAckDelay.java index 9a4211618982a..f4c76a1fb871f 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/WriteAckDelay.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/WriteAckDelay.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.bulk; diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/CreateDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/CreateDataStreamAction.java index 30c6699ac902e..48b2407e89b97 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/CreateDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/CreateDataStreamAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.datastreams; diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsActionUtil.java b/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsActionUtil.java index e9ff74459c153..a0a05138406c5 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsActionUtil.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsActionUtil.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.datastreams; diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsStatsAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsStatsAction.java index 9a4eaf9a78e9b..9266bae439b73 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsStatsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.datastreams; @@ -39,7 +40,30 @@ public static class Request extends BroadcastRequest { public Request() { // this doesn't really matter since data stream name resolution isn't affected by IndicesOptions and // a data stream's backing indices are retrieved from its metadata - super(null, IndicesOptions.fromOptions(false, true, true, true, true, false, true, false)); + super( + null, + IndicesOptions.builder() + .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) + .wildcardOptions( + IndicesOptions.WildcardOptions.builder() + .matchOpen(true) + .matchClosed(true) + .includeHidden(false) + .resolveAliases(false) + .allowEmptyExpressions(true) + .build() + ) + .gatekeeperOptions( + IndicesOptions.GatekeeperOptions.builder() + .allowAliasToMultipleIndices(true) + .allowClosedIndices(true) + .ignoreThrottled(false) + .allowFailureIndices(true) + .build() + ) + .selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE) + .build() + ); } public Request(StreamInput in) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/DeleteDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/DeleteDataStreamAction.java index d95a8bbc4b461..4f647d4f02884 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/DeleteDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/DeleteDataStreamAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.datastreams; @@ -45,7 +46,25 @@ public static class Request extends MasterNodeRequest implements Indice // empty response can be returned in case wildcards were used or // 404 status code returned in case no wildcard were used. private final boolean wildcardExpressionsOriginallySpecified; - private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true, false); + private IndicesOptions indicesOptions = IndicesOptions.builder() + .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) + .wildcardOptions( + IndicesOptions.WildcardOptions.builder() + .matchOpen(true) + .matchClosed(true) + .resolveAliases(false) + .allowEmptyExpressions(true) + .build() + ) + .gatekeeperOptions( + IndicesOptions.GatekeeperOptions.builder() + .allowAliasToMultipleIndices(false) + .allowClosedIndices(true) + .ignoreThrottled(false) + .allowFailureIndices(true) + .build() + ) + .build(); public Request(TimeValue masterNodeTimeout, String... names) { super(masterNodeTimeout); diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java index 2f9ba9220fbce..883fc543749c2 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.datastreams; @@ -55,7 +56,26 @@ private GetDataStreamAction() { public static class Request extends MasterNodeReadRequest implements IndicesRequest.Replaceable { private String[] names; - private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true, false); + private IndicesOptions indicesOptions = IndicesOptions.builder() + .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) + .wildcardOptions( + IndicesOptions.WildcardOptions.builder() + .matchOpen(true) + .matchClosed(true) + .includeHidden(false) + .resolveAliases(false) + .allowEmptyExpressions(true) + .build() + ) + .gatekeeperOptions( + IndicesOptions.GatekeeperOptions.builder() + .allowAliasToMultipleIndices(false) + .allowClosedIndices(true) + .ignoreThrottled(false) + .allowFailureIndices(true) + .build() + ) + .build(); private boolean includeDefaults = false; private boolean verbose = false; @@ -92,7 +112,7 @@ public Request(StreamInput in) throws IOException { } else { this.includeDefaults = false; } - if (in.getTransportVersion().onOrAfter(TransportVersions.GET_DATA_STREAMS_VERBOSE)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { this.verbose = in.readBoolean(); } else { this.verbose = false; @@ -107,7 +127,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { out.writeBoolean(includeDefaults); } - if (out.getTransportVersion().onOrAfter(TransportVersions.GET_DATA_STREAMS_VERBOSE)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeBoolean(verbose); } } @@ -214,6 +234,7 @@ public static class DataStreamInfo implements SimpleDiffable, To private final DataStream dataStream; private final ClusterHealthStatus dataStreamStatus; + private final boolean failureStoreEffectivelyEnabled; // Must be serialized independently of dataStream as depends on settings @Nullable private final String indexTemplate; @Nullable @@ -227,6 +248,7 @@ public static class DataStreamInfo implements SimpleDiffable, To public DataStreamInfo( DataStream dataStream, + boolean failureStoreEffectivelyEnabled, ClusterHealthStatus dataStreamStatus, @Nullable String indexTemplate, @Nullable String ilmPolicyName, @@ -236,6 +258,7 @@ public DataStreamInfo( @Nullable Long maximumTimestamp ) { this.dataStream = dataStream; + this.failureStoreEffectivelyEnabled = failureStoreEffectivelyEnabled; this.dataStreamStatus = dataStreamStatus; this.indexTemplate = indexTemplate; this.ilmPolicyName = ilmPolicyName; @@ -247,22 +270,32 @@ public DataStreamInfo( @SuppressWarnings("unchecked") DataStreamInfo(StreamInput in) throws IOException { - this( - DataStream.read(in), - ClusterHealthStatus.readFrom(in), - in.readOptionalString(), - in.readOptionalString(), - in.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0) ? in.readOptionalWriteable(TimeSeries::new) : null, - in.getTransportVersion().onOrAfter(V_8_11_X) ? in.readMap(Index::new, IndexProperties::new) : Map.of(), - in.getTransportVersion().onOrAfter(V_8_11_X) ? in.readBoolean() : true, - in.getTransportVersion().onOrAfter(TransportVersions.GET_DATA_STREAMS_VERBOSE) ? in.readOptionalVLong() : null - ); + this.dataStream = DataStream.read(in); + this.failureStoreEffectivelyEnabled = in.getTransportVersion() + .onOrAfter(TransportVersions.FAILURE_STORE_ENABLED_BY_CLUSTER_SETTING) + ? in.readBoolean() + : dataStream.isFailureStoreExplicitlyEnabled(); // Revert to the behaviour before this field was added + this.dataStreamStatus = ClusterHealthStatus.readFrom(in); + this.indexTemplate = in.readOptionalString(); + this.ilmPolicyName = in.readOptionalString(); + this.timeSeries = in.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0) + ? in.readOptionalWriteable(TimeSeries::new) + : null; + this.indexSettingsValues = in.getTransportVersion().onOrAfter(V_8_11_X) + ? in.readMap(Index::new, IndexProperties::new) + : Map.of(); + this.templatePreferIlmValue = in.getTransportVersion().onOrAfter(V_8_11_X) ? in.readBoolean() : true; + this.maximumTimestamp = in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readOptionalVLong() : null; } public DataStream getDataStream() { return dataStream; } + public boolean isFailureStoreEffectivelyEnabled() { + return failureStoreEffectivelyEnabled; + } + public ClusterHealthStatus getDataStreamStatus() { return dataStreamStatus; } @@ -298,6 +331,9 @@ public Long getMaximumTimestamp() { @Override public void writeTo(StreamOutput out) throws IOException { dataStream.writeTo(out); + if (out.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_ENABLED_BY_CLUSTER_SETTING)) { + out.writeBoolean(failureStoreEffectivelyEnabled); + } dataStreamStatus.writeTo(out); out.writeOptionalString(indexTemplate); out.writeOptionalString(ilmPolicyName); @@ -308,7 +344,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeMap(indexSettingsValues); out.writeBoolean(templatePreferIlmValue); } - if (out.getTransportVersion().onOrAfter(TransportVersions.GET_DATA_STREAMS_VERBOSE)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeOptionalVLong(maximumTimestamp); } } @@ -378,7 +414,7 @@ public XContentBuilder toXContent( } if (DataStream.isFailureStoreFeatureFlagEnabled()) { builder.startObject(DataStream.FAILURE_STORE_FIELD.getPreferredName()); - builder.field(FAILURE_STORE_ENABLED.getPreferredName(), dataStream.isFailureStoreEnabled()); + builder.field(FAILURE_STORE_ENABLED.getPreferredName(), failureStoreEffectivelyEnabled); builder.field( DataStream.ROLLOVER_ON_WRITE_FIELD.getPreferredName(), dataStream.getFailureIndices().isRolloverOnWrite() @@ -457,6 +493,7 @@ public boolean equals(Object o) { DataStreamInfo that = (DataStreamInfo) o; return templatePreferIlmValue == that.templatePreferIlmValue && Objects.equals(dataStream, that.dataStream) + && failureStoreEffectivelyEnabled == that.failureStoreEffectivelyEnabled && dataStreamStatus == that.dataStreamStatus && Objects.equals(indexTemplate, that.indexTemplate) && Objects.equals(ilmPolicyName, that.ilmPolicyName) @@ -470,6 +507,7 @@ public int hashCode() { return Objects.hash( dataStream, dataStreamStatus, + failureStoreEffectivelyEnabled, indexTemplate, ilmPolicyName, timeSeries, diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/MigrateToDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/MigrateToDataStreamAction.java index 894708a282ea9..86cb43e18c131 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/MigrateToDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/MigrateToDataStreamAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.datastreams; diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/ModifyDataStreamsAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/ModifyDataStreamsAction.java index dcc1e1897bc3d..9c51d0dd209cd 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/ModifyDataStreamsAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/ModifyDataStreamsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.datastreams; diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/PromoteDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/PromoteDataStreamAction.java index f423969b617f7..4431c464ceb5a 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/PromoteDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/PromoteDataStreamAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.datastreams; diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/autosharding/AutoShardingResult.java b/server/src/main/java/org/elasticsearch/action/datastreams/autosharding/AutoShardingResult.java index 7bbd3291caf3a..e0d48b139acff 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/autosharding/AutoShardingResult.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/autosharding/AutoShardingResult.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.datastreams.autosharding; diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/autosharding/AutoShardingType.java b/server/src/main/java/org/elasticsearch/action/datastreams/autosharding/AutoShardingType.java index 50d3027abbc88..5f332e2194fac 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/autosharding/AutoShardingType.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/autosharding/AutoShardingType.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.datastreams.autosharding; diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/autosharding/DataStreamAutoShardingService.java b/server/src/main/java/org/elasticsearch/action/datastreams/autosharding/DataStreamAutoShardingService.java index da1c85834d2c8..580cf92d15e2c 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/autosharding/DataStreamAutoShardingService.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/autosharding/DataStreamAutoShardingService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.datastreams.autosharding; diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ErrorEntry.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ErrorEntry.java index 62406ccd4f853..47d60ecbfa5e1 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ErrorEntry.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ErrorEntry.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.datastreams.lifecycle; diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainDataStreamLifecycleAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainDataStreamLifecycleAction.java index d51f00681bb5e..b522225d94246 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainDataStreamLifecycleAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainDataStreamLifecycleAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.datastreams.lifecycle; diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java index 962c2975f5998..f9bd135968246 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.datastreams.lifecycle; @@ -81,7 +82,7 @@ public ExplainIndexDataStreamLifecycle( public ExplainIndexDataStreamLifecycle(StreamInput in) throws IOException { this.index = in.readString(); this.managedByLifecycle = in.readBoolean(); - if (in.getTransportVersion().onOrAfter(TransportVersions.NO_GLOBAL_RETENTION_FOR_SYSTEM_DATA_STREAMS)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { this.isInternalDataStream = in.readBoolean(); } else { this.isInternalDataStream = false; @@ -122,7 +123,7 @@ public XContentBuilder toXContent( builder.field(MANAGED_BY_LIFECYCLE_FIELD.getPreferredName(), managedByLifecycle); if (managedByLifecycle) { if (indexCreationDate != null) { - builder.timeField( + builder.timestampFieldsFromUnixEpochMillis( INDEX_CREATION_DATE_MILLIS_FIELD.getPreferredName(), INDEX_CREATION_DATE_FIELD.getPreferredName(), indexCreationDate @@ -133,7 +134,11 @@ public XContentBuilder toXContent( ); } if (rolloverDate != null) { - builder.timeField(ROLLOVER_DATE_MILLIS_FIELD.getPreferredName(), ROLLOVER_DATE_FIELD.getPreferredName(), rolloverDate); + builder.timestampFieldsFromUnixEpochMillis( + ROLLOVER_DATE_MILLIS_FIELD.getPreferredName(), + ROLLOVER_DATE_FIELD.getPreferredName(), + rolloverDate + ); builder.field(TIME_SINCE_ROLLOVER_FIELD.getPreferredName(), getTimeSinceRollover(nowSupplier).toHumanReadableString(2)); } if (generationDateMillis != null) { @@ -160,7 +165,7 @@ public XContentBuilder toXContent( public void writeTo(StreamOutput out) throws IOException { out.writeString(index); out.writeBoolean(managedByLifecycle); - if (out.getTransportVersion().onOrAfter(TransportVersions.NO_GLOBAL_RETENTION_FOR_SYSTEM_DATA_STREAMS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { out.writeBoolean(isInternalDataStream); } if (managedByLifecycle) { diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java index 2ae6e544b3f53..a43d29501a7ee 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.datastreams.lifecycle; @@ -46,7 +47,26 @@ private GetDataStreamLifecycleAction() {/* no instances */} public static class Request extends MasterNodeReadRequest implements IndicesRequest.Replaceable { private String[] names; - private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true, false); + private IndicesOptions indicesOptions = IndicesOptions.builder() + .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) + .wildcardOptions( + IndicesOptions.WildcardOptions.builder() + .matchOpen(true) + .matchClosed(true) + .includeHidden(false) + .resolveAliases(false) + .allowEmptyExpressions(true) + .build() + ) + .gatekeeperOptions( + IndicesOptions.GatekeeperOptions.builder() + .allowAliasToMultipleIndices(false) + .allowClosedIndices(true) + .ignoreThrottled(false) + .allowFailureIndices(true) + .build() + ) + .build(); private boolean includeDefaults = false; public Request(TimeValue masterNodeTimeout, String[] names) { @@ -153,7 +173,7 @@ public record DataStreamLifecycle( this( in.readString(), in.readOptionalWriteable(org.elasticsearch.cluster.metadata.DataStreamLifecycle::new), - in.getTransportVersion().onOrAfter(TransportVersions.NO_GLOBAL_RETENTION_FOR_SYSTEM_DATA_STREAMS) && in.readBoolean() + in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0) && in.readBoolean() ); } @@ -161,7 +181,7 @@ public record DataStreamLifecycle( public void writeTo(StreamOutput out) throws IOException { out.writeString(dataStreamName); out.writeOptionalWriteable(lifecycle); - if (out.getTransportVersion().onOrAfter(TransportVersions.NO_GLOBAL_RETENTION_FOR_SYSTEM_DATA_STREAMS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { out.writeBoolean(isInternalDataStream); } } diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java index c32fa849d9fd3..b054d12890366 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.datastreams.lifecycle; @@ -77,7 +78,26 @@ public static Request parseRequest(XContentParser parser, Factory factory) { } private String[] names; - private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true, false); + private IndicesOptions indicesOptions = IndicesOptions.builder() + .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) + .wildcardOptions( + IndicesOptions.WildcardOptions.builder() + .matchOpen(true) + .matchClosed(true) + .includeHidden(false) + .resolveAliases(false) + .allowEmptyExpressions(true) + .build() + ) + .gatekeeperOptions( + IndicesOptions.GatekeeperOptions.builder() + .allowAliasToMultipleIndices(false) + .allowClosedIndices(true) + .ignoreThrottled(false) + .allowFailureIndices(false) + .build() + ) + .build(); private final DataStreamLifecycle lifecycle; public Request(StreamInput in) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java b/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java index e4b8edea58114..1a5495412f605 100644 --- a/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java +++ b/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.delete; @@ -236,11 +237,6 @@ public boolean isRequireDataStream() { return false; } - @Override - public void process(IndexRouting indexRouting) { - // Nothing to do - } - @Override public int route(IndexRouting indexRouting) { return indexRouting.deleteShard(id, routing); diff --git a/server/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java index 40810f004b0de..b70271c0ce47a 100644 --- a/server/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.delete; diff --git a/server/src/main/java/org/elasticsearch/action/delete/DeleteResponse.java b/server/src/main/java/org/elasticsearch/action/delete/DeleteResponse.java index 47202998d3193..ddaf80887bc5f 100644 --- a/server/src/main/java/org/elasticsearch/action/delete/DeleteResponse.java +++ b/server/src/main/java/org/elasticsearch/action/delete/DeleteResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.delete; diff --git a/server/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java b/server/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java index 44f89ba01edb7..2f034893a9f00 100644 --- a/server/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java +++ b/server/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.delete; diff --git a/server/src/main/java/org/elasticsearch/action/delete/package-info.java b/server/src/main/java/org/elasticsearch/action/delete/package-info.java index 744fcb92afe81..bc406dc21b5b0 100644 --- a/server/src/main/java/org/elasticsearch/action/delete/package-info.java +++ b/server/src/main/java/org/elasticsearch/action/delete/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/server/src/main/java/org/elasticsearch/action/downsample/DownsampleAction.java b/server/src/main/java/org/elasticsearch/action/downsample/DownsampleAction.java index 8ccc190a0444b..62771230636c1 100644 --- a/server/src/main/java/org/elasticsearch/action/downsample/DownsampleAction.java +++ b/server/src/main/java/org/elasticsearch/action/downsample/DownsampleAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.downsample; diff --git a/server/src/main/java/org/elasticsearch/action/downsample/DownsampleConfig.java b/server/src/main/java/org/elasticsearch/action/downsample/DownsampleConfig.java index 432dc69caf634..43d722cdcae3b 100644 --- a/server/src/main/java/org/elasticsearch/action/downsample/DownsampleConfig.java +++ b/server/src/main/java/org/elasticsearch/action/downsample/DownsampleConfig.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.downsample; diff --git a/server/src/main/java/org/elasticsearch/action/downsample/DownsampleTask.java b/server/src/main/java/org/elasticsearch/action/downsample/DownsampleTask.java index aeb7cdda9103c..fb1c30d723408 100644 --- a/server/src/main/java/org/elasticsearch/action/downsample/DownsampleTask.java +++ b/server/src/main/java/org/elasticsearch/action/downsample/DownsampleTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.downsample; diff --git a/server/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java b/server/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java index d1e48574fe226..91dbe32feca39 100644 --- a/server/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java +++ b/server/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.explain; diff --git a/server/src/main/java/org/elasticsearch/action/explain/ExplainRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/explain/ExplainRequestBuilder.java index f8684cdb4bb0c..b28e4a4bca341 100644 --- a/server/src/main/java/org/elasticsearch/action/explain/ExplainRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/explain/ExplainRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.explain; diff --git a/server/src/main/java/org/elasticsearch/action/explain/ExplainResponse.java b/server/src/main/java/org/elasticsearch/action/explain/ExplainResponse.java index 9d8ba5f126fd5..caedec6a563d4 100644 --- a/server/src/main/java/org/elasticsearch/action/explain/ExplainResponse.java +++ b/server/src/main/java/org/elasticsearch/action/explain/ExplainResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.explain; diff --git a/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java b/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java index 47e9624d0ec12..84c6df7b8a66f 100644 --- a/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java +++ b/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.explain; @@ -17,6 +18,7 @@ import org.elasticsearch.action.support.single.shard.TransportSingleShardAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.Writeable; @@ -108,7 +110,7 @@ protected boolean resolveIndex(ExplainRequest request) { @Override protected void resolveRequest(ClusterState state, InternalRequest request) { - final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(state, request.request().index()); + final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(state, request.request().index()); final AliasFilter aliasFilter = searchService.buildAliasFilter(state, request.concreteIndex(), indicesAndAliases); request.request().filteringAlias(aliasFilter); } diff --git a/server/src/main/java/org/elasticsearch/action/explain/package-info.java b/server/src/main/java/org/elasticsearch/action/explain/package-info.java index d4e83a6569c56..a42570d41243f 100644 --- a/server/src/main/java/org/elasticsearch/action/explain/package-info.java +++ b/server/src/main/java/org/elasticsearch/action/explain/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java index 856571c305615..321cc394ea809 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.fieldcaps; diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFailure.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFailure.java index e47de4bebc1bb..cc2ea2a4ed57f 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFailure.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFailure.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.fieldcaps; diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java index e435655668882..e6d7af11d06cc 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.fieldcaps; diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java index 5a50ed4c9f573..6f510ad26f5ec 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.fieldcaps; @@ -61,7 +62,7 @@ public FieldCapabilitiesIndexResponse( } else { this.indexMappingHash = null; } - if (in.getTransportVersion().onOrAfter(TransportVersions.FIELD_CAPS_RESPONSE_INDEX_MODE)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { this.indexMode = IndexMode.readFrom(in); } else { this.indexMode = IndexMode.STANDARD; @@ -76,7 +77,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(MAPPING_HASH_VERSION)) { out.writeOptionalString(indexMappingHash); } - if (out.getTransportVersion().onOrAfter(TransportVersions.FIELD_CAPS_RESPONSE_INDEX_MODE)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { IndexMode.writeTo(indexMode, out); } } @@ -104,7 +105,7 @@ static List readList(StreamInput input) throws I private static void collectCompressedResponses(StreamInput input, int groups, ArrayList responses) throws IOException { final CompressedGroup[] compressedGroups = new CompressedGroup[groups]; - final boolean readIndexMode = input.getTransportVersion().onOrAfter(TransportVersions.FIELD_CAPS_RESPONSE_INDEX_MODE); + final boolean readIndexMode = input.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0); for (int i = 0; i < groups; i++) { final String[] indices = input.readStringArray(); final IndexMode indexMode = readIndexMode ? IndexMode.readFrom(input) : IndexMode.STANDARD; @@ -178,7 +179,7 @@ private static void writeCompressedResponses(StreamOutput output, Map { o.writeCollection(fieldCapabilitiesIndexResponses, (oo, r) -> oo.writeString(r.indexName)); var first = fieldCapabilitiesIndexResponses.get(0); - if (output.getTransportVersion().onOrAfter(TransportVersions.FIELD_CAPS_RESPONSE_INDEX_MODE)) { + if (output.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { IndexMode.writeTo(first.indexMode, o); } o.writeString(first.indexMappingHash); diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesNodeRequest.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesNodeRequest.java index 6c1734bde401f..df9e6aa8f58da 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesNodeRequest.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesNodeRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.fieldcaps; diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesNodeResponse.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesNodeResponse.java index d358e69c5b2ff..9f0f7eeb2e213 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesNodeResponse.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesNodeResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.fieldcaps; diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java index 6fab92219511d..80cdbf9d3d867 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.fieldcaps; @@ -34,7 +35,7 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements IndicesRequest.Replaceable, ToXContentObject { public static final String NAME = "field_caps_request"; - public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpen(); + public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpenAndForbidClosed(); private String[] indices = Strings.EMPTY_ARRAY; private IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS; diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java index 8015bd6b4542b..ae5ad2bff26be 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java @@ -1,14 +1,16 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.fieldcaps; import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.ElasticsearchClient; import org.elasticsearch.index.query.QueryBuilder; @@ -37,6 +39,11 @@ public FieldCapabilitiesRequestBuilder setincludeEmptyFields(boolean includeEmpt return this; } + public FieldCapabilitiesRequestBuilder setIndicesOptions(IndicesOptions indicesOptions) { + request().indicesOptions(indicesOptions); + return this; + } + public FieldCapabilitiesRequestBuilder setIndexFilter(QueryBuilder indexFilter) { request().indexFilter(indexFilter); return this; diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java index 4946f6ca7835d..e5c2678bbf38e 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.fieldcaps; diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/IndexFieldCapabilities.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/IndexFieldCapabilities.java index de2f6965e011d..349ae476dd724 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/IndexFieldCapabilities.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/IndexFieldCapabilities.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.fieldcaps; diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/RequestDispatcher.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/RequestDispatcher.java index 6eec2f56d52f1..fce925d868532 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/RequestDispatcher.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/RequestDispatcher.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.fieldcaps; @@ -94,7 +95,7 @@ final class RequestDispatcher { for (String index : indices) { final GroupShardsIterator shardIts; try { - shardIts = clusterService.operationRouting().searchShards(clusterState, new String[] { index }, null, null, null, null); + shardIts = clusterService.operationRouting().searchShards(clusterState, new String[] { index }, null, null); } catch (Exception e) { onIndexFailure.accept(index, e); continue; diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/ResponseRewriter.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/ResponseRewriter.java index c4e9b1bce6d81..42ec98209a21b 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/ResponseRewriter.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/ResponseRewriter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.fieldcaps; diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java index bb97b0dc48c42..37af070effc9f 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.fieldcaps; @@ -21,6 +22,7 @@ import org.elasticsearch.action.support.ChannelActionListener; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.action.support.RefCountingRunnable; +import org.elasticsearch.client.internal.RemoteClusterClient; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -112,23 +114,28 @@ public TransportFieldCapabilitiesAction( @Override protected void doExecute(Task task, FieldCapabilitiesRequest request, final ActionListener listener) { - executeRequest(task, request, REMOTE_TYPE, listener); + executeRequest( + task, + request, + (remoteClient, remoteRequest, remoteListener) -> remoteClient.execute(REMOTE_TYPE, remoteRequest, remoteListener), + listener + ); } public void executeRequest( Task task, FieldCapabilitiesRequest request, - RemoteClusterActionType remoteAction, + RemoteRequestExecutor remoteRequestExecutor, ActionListener listener ) { // workaround for https://github.com/elastic/elasticsearch/issues/97916 - TODO remove this when we can - searchCoordinationExecutor.execute(ActionRunnable.wrap(listener, l -> doExecuteForked(task, request, remoteAction, l))); + searchCoordinationExecutor.execute(ActionRunnable.wrap(listener, l -> doExecuteForked(task, request, remoteRequestExecutor, l))); } private void doExecuteForked( Task task, FieldCapabilitiesRequest request, - RemoteClusterActionType remoteAction, + RemoteRequestExecutor remoteRequestExecutor, ActionListener listener ) { if (ccsCheckCompatibility) { @@ -281,8 +288,8 @@ private void doExecuteForked( handleIndexFailure.accept(RemoteClusterAware.buildRemoteIndexName(clusterAlias, index), ex); } }); - remoteClusterClient.execute( - remoteAction, + remoteRequestExecutor.executeRemoteRequest( + remoteClusterClient, remoteRequest, // The underlying transport service may call onFailure with a thread pool other than search_coordinator. // This fork is a workaround to ensure that the merging of field-caps always occurs on the search_coordinator. @@ -297,6 +304,14 @@ private void doExecuteForked( } } + public interface RemoteRequestExecutor { + void executeRemoteRequest( + RemoteClusterClient remoteClient, + FieldCapabilitiesRequest remoteRequest, + ActionListener remoteListener + ); + } + private static void checkIndexBlocks(ClusterState clusterState, String[] concreteIndices) { var blocks = clusterState.blocks(); if (blocks.global().isEmpty() && blocks.indices().isEmpty()) { diff --git a/server/src/main/java/org/elasticsearch/action/get/GetRequest.java b/server/src/main/java/org/elasticsearch/action/get/GetRequest.java index 9f7ed8fd56474..9ff4e662270f9 100644 --- a/server/src/main/java/org/elasticsearch/action/get/GetRequest.java +++ b/server/src/main/java/org/elasticsearch/action/get/GetRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.get; diff --git a/server/src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java index d581bfa73a1f5..9d25c56f5c359 100644 --- a/server/src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.get; diff --git a/server/src/main/java/org/elasticsearch/action/get/GetResponse.java b/server/src/main/java/org/elasticsearch/action/get/GetResponse.java index 4c48cbc44e207..fcf4f24eec662 100644 --- a/server/src/main/java/org/elasticsearch/action/get/GetResponse.java +++ b/server/src/main/java/org/elasticsearch/action/get/GetResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.get; diff --git a/server/src/main/java/org/elasticsearch/action/get/MultiGetItemResponse.java b/server/src/main/java/org/elasticsearch/action/get/MultiGetItemResponse.java index b10471e75fcf8..f4d4dd50ff9ed 100644 --- a/server/src/main/java/org/elasticsearch/action/get/MultiGetItemResponse.java +++ b/server/src/main/java/org/elasticsearch/action/get/MultiGetItemResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.get; diff --git a/server/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java b/server/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java index e00edefc26289..cc1d0497bd51a 100644 --- a/server/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java +++ b/server/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.get; diff --git a/server/src/main/java/org/elasticsearch/action/get/MultiGetRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/get/MultiGetRequestBuilder.java index 8512ae959cb52..a54a67cc64b0a 100644 --- a/server/src/main/java/org/elasticsearch/action/get/MultiGetRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/get/MultiGetRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.get; diff --git a/server/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java b/server/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java index 3306ac874243c..fc126b29b3265 100644 --- a/server/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java +++ b/server/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.get; diff --git a/server/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java b/server/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java index fd0aa8ea2e232..77f7b5087c96e 100644 --- a/server/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java +++ b/server/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.get; diff --git a/server/src/main/java/org/elasticsearch/action/get/MultiGetShardResponse.java b/server/src/main/java/org/elasticsearch/action/get/MultiGetShardResponse.java index eb5f1964388b0..6bc536fe643ae 100644 --- a/server/src/main/java/org/elasticsearch/action/get/MultiGetShardResponse.java +++ b/server/src/main/java/org/elasticsearch/action/get/MultiGetShardResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.get; diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java index 330a97e27f278..e9b625bb49667 100644 --- a/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java +++ b/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.get; @@ -26,7 +27,6 @@ import org.elasticsearch.cluster.ClusterStateObserver; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.routing.OperationRouting; import org.elasticsearch.cluster.routing.PlainShardIterator; import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.cluster.service.ClusterService; @@ -107,10 +107,7 @@ protected ShardIterator shards(ClusterState state, InternalRequest request) { if (iterator == null) { return null; } - return new PlainShardIterator( - iterator.shardId(), - iterator.getShardRoutings().stream().filter(shardRouting -> OperationRouting.canSearchShard(shardRouting, state)).toList() - ); + return PlainShardIterator.allSearchableShards(iterator); } @Override @@ -124,13 +121,11 @@ protected void asyncShardOperation(GetRequest request, ShardId shardId, ActionLi IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); IndexShard indexShard = indexService.getShard(shardId.id()); if (indexShard.routingEntry().isPromotableToPrimary() == false) { - assert indexShard.indexSettings().isFastRefresh() == false - : "a search shard should not receive a TransportGetAction for an index with fast refresh"; handleGetOnUnpromotableShard(request, indexShard, listener); return; } - assert DiscoveryNode.isStateless(clusterService.getSettings()) == false || indexShard.indexSettings().isFastRefresh() - : "in Stateless a promotable to primary shard can receive a TransportGetAction only if an index has the fast refresh setting"; + assert DiscoveryNode.isStateless(clusterService.getSettings()) == false + : "in Stateless a promotable to primary shard should not receive a TransportGetAction"; if (request.realtime()) { // we are not tied to a refresh cycle here anyway asyncGet(request, shardId, listener); } else { diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportGetFromTranslogAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportGetFromTranslogAction.java index 0b63bdc0cf980..3cbd7497dcf39 100644 --- a/server/src/main/java/org/elasticsearch/action/get/TransportGetFromTranslogAction.java +++ b/server/src/main/java/org/elasticsearch/action/get/TransportGetFromTranslogAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.get; diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java index 671ba29db242c..e5087a790a292 100644 --- a/server/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java +++ b/server/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.get; @@ -18,6 +19,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.routing.OperationRouting; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.common.util.concurrent.EsExecutors; @@ -80,7 +82,7 @@ protected void doExecute(Task task, final MultiGetRequest request, final ActionL lastResolvedIndex = Tuple.tuple(item.index(), concreteSingleIndex); } item.routing(clusterState.metadata().resolveIndexRouting(item.routing(), item.index())); - shardId = clusterService.operationRouting().shardId(clusterState, concreteSingleIndex, item.id(), item.routing()); + shardId = OperationRouting.shardId(clusterState, concreteSingleIndex, item.id(), item.routing()); } catch (RoutingMissingException e) { responses.set(i, newItemFailure(e.getIndex().getName(), e.getId(), e)); continue; diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java index 07b586713f152..754ea77650180 100644 --- a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java +++ b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.get; @@ -26,7 +27,6 @@ import org.elasticsearch.cluster.ClusterStateObserver; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.routing.OperationRouting; import org.elasticsearch.cluster.routing.PlainShardIterator; import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.cluster.service.ClusterService; @@ -111,10 +111,7 @@ protected ShardIterator shards(ClusterState state, InternalRequest request) { if (iterator == null) { return null; } - return new PlainShardIterator( - iterator.shardId(), - iterator.getShardRoutings().stream().filter(shardRouting -> OperationRouting.canSearchShard(shardRouting, state)).toList() - ); + return PlainShardIterator.allSearchableShards(iterator); } @Override @@ -123,14 +120,11 @@ protected void asyncShardOperation(MultiGetShardRequest request, ShardId shardId IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); IndexShard indexShard = indexService.getShard(shardId.id()); if (indexShard.routingEntry().isPromotableToPrimary() == false) { - assert indexShard.indexSettings().isFastRefresh() == false - : "a search shard should not receive a TransportShardMultiGetAction for an index with fast refresh"; handleMultiGetOnUnpromotableShard(request, indexShard, listener); return; } - assert DiscoveryNode.isStateless(clusterService.getSettings()) == false || indexShard.indexSettings().isFastRefresh() - : "in Stateless a promotable to primary shard can receive a TransportShardMultiGetAction only if an index has " - + "the fast refresh setting"; + assert DiscoveryNode.isStateless(clusterService.getSettings()) == false + : "in Stateless a promotable to primary shard should not receive a TransportShardMultiGetAction"; if (request.realtime()) { // we are not tied to a refresh cycle here anyway asyncShardMultiGet(request, shardId, listener); } else { diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetFomTranslogAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetFomTranslogAction.java index ee354af979833..e953ff527f637 100644 --- a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetFomTranslogAction.java +++ b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetFomTranslogAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.get; diff --git a/server/src/main/java/org/elasticsearch/action/get/package-info.java b/server/src/main/java/org/elasticsearch/action/get/package-info.java index 75f058ead27d6..1d4746168f642 100644 --- a/server/src/main/java/org/elasticsearch/action/get/package-info.java +++ b/server/src/main/java/org/elasticsearch/action/get/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java index 61d610c9eda4e..4343451256920 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.index; @@ -50,6 +51,8 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.OptionalInt; +import java.util.function.Supplier; import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM; @@ -75,6 +78,8 @@ public class IndexRequest extends ReplicatedWriteRequest implement private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(IndexRequest.class); private static final TransportVersion PIPELINES_HAVE_RUN_FIELD_ADDED = TransportVersions.V_8_10_X; + private static final Supplier ID_GENERATOR = UUIDs::base64UUID; + /** * Max length of the source document to include into string() * @@ -145,9 +150,6 @@ public class IndexRequest extends ReplicatedWriteRequest implement * rawTimestamp field is used on the coordinate node, it doesn't need to be serialised. */ private Object rawTimestamp; - private long normalisedBytesParsed = -1; - private boolean originatesFromUpdateByScript; - private boolean originatesFromUpdateByDoc; public IndexRequest(StreamInput in) throws IOException { this(null, in); @@ -182,7 +184,7 @@ public IndexRequest(@Nullable ShardId shardId, StreamInput in) throws IOExceptio dynamicTemplates = in.readMap(StreamInput::readString); if (in.getTransportVersion().onOrAfter(PIPELINES_HAVE_RUN_FIELD_ADDED) && in.getTransportVersion().before(TransportVersions.V_8_13_0)) { - in.readBoolean(); + in.readBoolean(); // obsolete, prior to tracking normalisedBytesParsed } if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) { this.listExecutedPipelines = in.readBoolean(); @@ -195,21 +197,18 @@ public IndexRequest(@Nullable ShardId shardId, StreamInput in) throws IOExceptio } if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) { requireDataStream = in.readBoolean(); - normalisedBytesParsed = in.readZLong(); } else { requireDataStream = false; } - if (in.getTransportVersion().onOrAfter(TransportVersions.INDEX_REQUEST_UPDATE_BY_SCRIPT_ORIGIN)) { - originatesFromUpdateByScript = in.readBoolean(); - } else { - originatesFromUpdateByScript = false; - } - - if (in.getTransportVersion().onOrAfter(TransportVersions.INDEX_REQUEST_UPDATE_BY_DOC_ORIGIN)) { - originatesFromUpdateByDoc = in.readBoolean(); - } else { - originatesFromUpdateByDoc = false; + if (in.getTransportVersion().before(TransportVersions.INDEX_REQUEST_REMOVE_METERING)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) { + in.readZLong(); // obsolete normalisedBytesParsed + } + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { + in.readBoolean(); // obsolete originatesFromUpdateByScript + in.readBoolean(); // obsolete originatesFromUpdateByDoc + } } } @@ -686,8 +685,13 @@ public VersionType versionType() { } @Override - public void process(IndexRouting indexRouting) { - indexRouting.process(this); + public void preRoutingProcess(IndexRouting indexRouting) { + indexRouting.preProcess(this); + } + + @Override + public void postRoutingProcess(IndexRouting indexRouting) { + indexRouting.postProcess(this); } /** @@ -695,10 +699,27 @@ public void process(IndexRouting indexRouting) { * request compatible with the append-only optimization. */ public void autoGenerateId() { - assert id == null; - assert autoGeneratedTimestamp == UNSET_AUTO_GENERATED_TIMESTAMP : "timestamp has already been generated!"; - assert ifSeqNo == UNASSIGNED_SEQ_NO; - assert ifPrimaryTerm == UNASSIGNED_PRIMARY_TERM; + assertBeforeGeneratingId(); + autoGenerateTimestamp(); + id(ID_GENERATOR.get()); + } + + public void autoGenerateTimeBasedId() { + autoGenerateTimeBasedId(OptionalInt.empty()); + } + + /** + * Set the {@code #id()} to an automatically generated one, optimized for storage (compression) efficiency. + * If a routing hash is passed, it is included in the generated id starting at 9 bytes before the end. + * @param hash optional routing hash value, used to route requests by id to the right shard. + */ + public void autoGenerateTimeBasedId(OptionalInt hash) { + assertBeforeGeneratingId(); + autoGenerateTimestamp(); + id(UUIDs.base64TimeBasedKOrderedUUIDWithHash(hash)); + } + + private void autoGenerateTimestamp() { /* * Set the auto generated timestamp so the append only optimization * can quickly test if this request *must* be unique without reaching @@ -707,8 +728,13 @@ public void autoGenerateId() { * never work before 1970, but that's ok. It's after 1970. */ autoGeneratedTimestamp = Math.max(0, System.currentTimeMillis()); - String uid = UUIDs.base64UUID(); - id(uid); + } + + private void assertBeforeGeneratingId() { + assert id == null; + assert autoGeneratedTimestamp == UNSET_AUTO_GENERATED_TIMESTAMP : "timestamp has already been generated!"; + assert ifSeqNo == UNASSIGNED_SEQ_NO; + assert ifPrimaryTerm == UNASSIGNED_PRIMARY_TERM; } /** @@ -758,7 +784,7 @@ private void writeBody(StreamOutput out) throws IOException { out.writeMap(dynamicTemplates, StreamOutput::writeString); if (out.getTransportVersion().onOrAfter(PIPELINES_HAVE_RUN_FIELD_ADDED) && out.getTransportVersion().before(TransportVersions.V_8_13_0)) { - out.writeBoolean(normalisedBytesParsed != -1L); + out.writeBoolean(false); // obsolete, prior to tracking normalisedBytesParsed } if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) { out.writeBoolean(listExecutedPipelines); @@ -769,15 +795,16 @@ private void writeBody(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) { out.writeBoolean(requireDataStream); - out.writeZLong(normalisedBytesParsed); } - if (out.getTransportVersion().onOrAfter(TransportVersions.INDEX_REQUEST_UPDATE_BY_SCRIPT_ORIGIN)) { - out.writeBoolean(originatesFromUpdateByScript); - } - - if (out.getTransportVersion().onOrAfter(TransportVersions.INDEX_REQUEST_UPDATE_BY_DOC_ORIGIN)) { - out.writeBoolean(originatesFromUpdateByDoc); + if (out.getTransportVersion().before(TransportVersions.INDEX_REQUEST_REMOVE_METERING)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) { + out.writeZLong(-1); // obsolete normalisedBytesParsed + } + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { + out.writeBoolean(false); // obsolete originatesFromUpdateByScript + out.writeBoolean(false); // obsolete originatesFromUpdateByDoc + } } } @@ -872,7 +899,7 @@ public Index getConcreteWriteIndex(IndexAbstraction ia, Metadata metadata) { @Override public int route(IndexRouting indexRouting) { - return indexRouting.indexShard(id, routing, contentType, source, this::routing); + return indexRouting.indexShard(id, routing, contentType, source); } public IndexRequest setRequireAlias(boolean requireAlias) { @@ -880,10 +907,19 @@ public IndexRequest setRequireAlias(boolean requireAlias) { return this; } + /** + * Returns a transient flag denoting that the local request should be routed to a failure store. Not persisted across the wire. N.B. If + * true, the failure store will be used regardless of whether the metadata indicates that the failure store is enabled. + */ public boolean isWriteToFailureStore() { return writeToFailureStore; } + /** + * Sets a transient flag denoting that the local request should be routed to a failure store. Not persisted across the wire. N.B. If + * true, the failure store will be used regardless of whether the metadata indicates that the failure store is enabled. It is the + * caller's responsibility to ensure that this is correct. + */ public IndexRequest setWriteToFailureStore(boolean writeToFailureStore) { this.writeToFailureStore = writeToFailureStore; return this; @@ -924,24 +960,6 @@ public void setRawTimestamp(Object rawTimestamp) { this.rawTimestamp = rawTimestamp; } - /** - * Returns a number of bytes observed when parsing a document in earlier stages of ingestion (like update/ingest service) - * Defaults to -1 when a document size was not observed in earlier stages. - * @return a number of bytes observed - */ - public long getNormalisedBytesParsed() { - return normalisedBytesParsed; - } - - /** - * Sets number of bytes observed by a DocumentSizeObserver - * @return an index request - */ - public IndexRequest setNormalisedBytesParsed(long normalisedBytesParsed) { - this.normalisedBytesParsed = normalisedBytesParsed; - return this; - } - /** * Adds the pipeline to the list of executed pipelines, if listExecutedPipelines is true * @@ -972,22 +990,4 @@ public List getExecutedPipelines() { return Collections.unmodifiableList(executedPipelines); } } - - public IndexRequest setOriginatesFromUpdateByScript(boolean originatesFromUpdateByScript) { - this.originatesFromUpdateByScript = originatesFromUpdateByScript; - return this; - } - - public boolean originatesFromUpdateByScript() { - return originatesFromUpdateByScript; - } - - public boolean originatesFromUpdateByDoc() { - return originatesFromUpdateByDoc; - } - - public IndexRequest setOriginatesFromUpdateByDoc(boolean originatesFromUpdateByDoc) { - this.originatesFromUpdateByDoc = originatesFromUpdateByDoc; - return this; - } } diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java index 3f3d97e2115cb..3041e6bf5e274 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.index; diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexResponse.java b/server/src/main/java/org/elasticsearch/action/index/IndexResponse.java index c547eb7ba8f30..7c45de8905174 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexResponse.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexResponse.java @@ -1,15 +1,17 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.index; import org.elasticsearch.TransportVersions; import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.bulk.IndexDocFailureStoreStatus; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -35,6 +37,7 @@ public class IndexResponse extends DocWriteResponse { */ @Nullable protected final List executedPipelines; + private IndexDocFailureStoreStatus failureStoreStatus; public IndexResponse(ShardId shardId, StreamInput in) throws IOException { super(shardId, in); @@ -43,6 +46,11 @@ public IndexResponse(ShardId shardId, StreamInput in) throws IOException { } else { executedPipelines = null; } + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { + failureStoreStatus = IndexDocFailureStoreStatus.read(in); + } else { + failureStoreStatus = IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN; + } } public IndexResponse(StreamInput in) throws IOException { @@ -52,10 +60,15 @@ public IndexResponse(StreamInput in) throws IOException { } else { executedPipelines = null; } + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { + failureStoreStatus = IndexDocFailureStoreStatus.read(in); + } else { + failureStoreStatus = IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN; + } } public IndexResponse(ShardId shardId, String id, long seqNo, long primaryTerm, long version, boolean created) { - this(shardId, id, seqNo, primaryTerm, version, created, null); + this(shardId, id, seqNo, primaryTerm, version, created, null, IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN); } public IndexResponse( @@ -67,7 +80,29 @@ public IndexResponse( boolean created, @Nullable List executedPipelines ) { - this(shardId, id, seqNo, primaryTerm, version, created ? Result.CREATED : Result.UPDATED, executedPipelines); + this( + shardId, + id, + seqNo, + primaryTerm, + version, + created ? Result.CREATED : Result.UPDATED, + executedPipelines, + IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN + ); + } + + public IndexResponse( + ShardId shardId, + String id, + long seqNo, + long primaryTerm, + long version, + boolean created, + @Nullable List executedPipelines, + IndexDocFailureStoreStatus failureStoreStatus + ) { + this(shardId, id, seqNo, primaryTerm, version, created ? Result.CREATED : Result.UPDATED, executedPipelines, failureStoreStatus); } private IndexResponse( @@ -77,10 +112,12 @@ private IndexResponse( long primaryTerm, long version, Result result, - @Nullable List executedPipelines + @Nullable List executedPipelines, + IndexDocFailureStoreStatus failureStoreStatus ) { super(shardId, id, seqNo, primaryTerm, version, assertCreatedOrUpdated(result)); this.executedPipelines = executedPipelines; + this.failureStoreStatus = failureStoreStatus; } @Override @@ -89,6 +126,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) { out.writeOptionalCollection(executedPipelines, StreamOutput::writeString); } + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { + failureStoreStatus.writeTo(out); + } } @Override @@ -97,6 +137,9 @@ public void writeThin(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) { out.writeOptionalCollection(executedPipelines, StreamOutput::writeString); } + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { + failureStoreStatus.writeTo(out); + } } public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { @@ -104,6 +147,7 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t if (executedPipelines != null) { updatedBuilder = updatedBuilder.field("executed_pipelines", executedPipelines.toArray()); } + failureStoreStatus.toXContent(builder, params); return updatedBuilder; } @@ -117,6 +161,15 @@ public RestStatus status() { return result == Result.CREATED ? RestStatus.CREATED : super.status(); } + public void setFailureStoreStatus(IndexDocFailureStoreStatus failureStoreStatus) { + this.failureStoreStatus = failureStoreStatus; + } + + @Override + public IndexDocFailureStoreStatus getFailureStoreStatus() { + return failureStoreStatus; + } + @Override public String toString() { StringBuilder builder = new StringBuilder(); @@ -128,6 +181,7 @@ public String toString() { builder.append(",seqNo=").append(getSeqNo()); builder.append(",primaryTerm=").append(getPrimaryTerm()); builder.append(",shards=").append(Strings.toString(getShardInfo())); + builder.append(",failure_store=").append(failureStoreStatus.getLabel()); return builder.append("]").toString(); } @@ -139,7 +193,16 @@ public String toString() { public static class Builder extends DocWriteResponse.Builder { @Override public IndexResponse build() { - IndexResponse indexResponse = new IndexResponse(shardId, id, seqNo, primaryTerm, version, result, null); + IndexResponse indexResponse = new IndexResponse( + shardId, + id, + seqNo, + primaryTerm, + version, + result, + null, + IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN + ); indexResponse.setForcedRefresh(forcedRefresh); if (shardInfo != null) { indexResponse.setShardInfo(shardInfo); diff --git a/server/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java b/server/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java index 304f37401f5fd..90a7a2e9f3b04 100644 --- a/server/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.index; diff --git a/server/src/main/java/org/elasticsearch/action/index/package-info.java b/server/src/main/java/org/elasticsearch/action/index/package-info.java index 25d3afc505447..a486870762217 100644 --- a/server/src/main/java/org/elasticsearch/action/index/package-info.java +++ b/server/src/main/java/org/elasticsearch/action/index/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineRequest.java b/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineRequest.java index 4ac4d63ba5de0..a3be50b282b0b 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineRequest.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.ingest; diff --git a/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineRequestBuilder.java index ef08f64765f98..721ce5fec6927 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.ingest; diff --git a/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java b/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java index 99a8cf5a38f13..f721df2616e37 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.ingest; diff --git a/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineAction.java b/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineAction.java index 48cdf5dbd92e6..243038f7e7f1d 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineAction.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.ingest; diff --git a/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineRequest.java b/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineRequest.java index 6a23bc578336c..a6a648dd4fe4c 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineRequest.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.ingest; diff --git a/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineRequestBuilder.java index ca873c5aa3843..52cb19a86b62a 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.ingest; diff --git a/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineResponse.java b/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineResponse.java index c685a49cddf2f..760b87af49a78 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineResponse.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.ingest; @@ -79,7 +80,7 @@ public RestStatus status() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); for (PipelineConfiguration pipeline : pipelines) { - builder.field(pipeline.getId(), summary ? Map.of() : pipeline.getConfigAsMap()); + builder.field(pipeline.getId(), summary ? Map.of() : pipeline.getConfig()); } builder.endObject(); return builder; diff --git a/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineTransportAction.java b/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineTransportAction.java index 834e6e5991b26..2618ac934251d 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineTransportAction.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineTransportAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.ingest; diff --git a/server/src/main/java/org/elasticsearch/action/ingest/IngestActionForwarder.java b/server/src/main/java/org/elasticsearch/action/ingest/IngestActionForwarder.java index dcc097ea78389..849a878f151d3 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/IngestActionForwarder.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/IngestActionForwarder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.ingest; diff --git a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java index b55454eaa810f..1189012991aa1 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.ingest; diff --git a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java index 2fce285d83f06..7c6354cb898c1 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.ingest; diff --git a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java index 46d5ad0017321..879142a9db1bf 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.ingest; diff --git a/server/src/main/java/org/elasticsearch/action/ingest/ReservedPipelineAction.java b/server/src/main/java/org/elasticsearch/action/ingest/ReservedPipelineAction.java index aca9bb81fb53f..752220ce4de90 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/ReservedPipelineAction.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/ReservedPipelineAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.ingest; diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java index 9e4984d7497a3..6ca66da0ebcaf 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.ingest; diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentResult.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentResult.java index 3180cedf8e3a5..020e639fea00f 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentResult.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentResult.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.ingest; diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentVerboseResult.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentVerboseResult.java index 28aec1ee0ebb8..20b3a04ed6d1d 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentVerboseResult.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentVerboseResult.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.ingest; diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java index 2dde9e53f5e7f..06cefe35e6ac8 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.ingest; diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulateIndexResponse.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulateIndexResponse.java index 258cd5ceaa8e7..307996a4c72cb 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulateIndexResponse.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulateIndexResponse.java @@ -1,15 +1,17 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.ingest; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.TransportVersions; +import org.elasticsearch.action.bulk.IndexDocFailureStoreStatus; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; @@ -22,6 +24,7 @@ import org.elasticsearch.xcontent.XContentType; import java.io.IOException; +import java.util.Collection; import java.util.List; /** @@ -32,6 +35,7 @@ public class SimulateIndexResponse extends IndexResponse { private final BytesReference source; private final XContentType sourceXContentType; + private final Collection ignoredFields; private final Exception exception; @SuppressWarnings("this-escape") @@ -40,11 +44,16 @@ public SimulateIndexResponse(StreamInput in) throws IOException { this.source = in.readBytesReference(); this.sourceXContentType = XContentType.valueOf(in.readString()); setShardInfo(ShardInfo.EMPTY); - if (in.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_VALIDATES_MAPPINGS)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { this.exception = in.readException(); } else { this.exception = null; } + if (in.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_IGNORED_FIELDS)) { + this.ignoredFields = in.readStringCollectionAsList(); + } else { + this.ignoredFields = List.of(); + } } @SuppressWarnings("this-escape") @@ -55,13 +64,24 @@ public SimulateIndexResponse( BytesReference source, XContentType sourceXContentType, List pipelines, + Collection ignoredFields, @Nullable Exception exception ) { // We don't actually care about most of the IndexResponse fields: - super(new ShardId(index, "", 0), id == null ? "" : id, 0, 0, version, true, pipelines); + super( + new ShardId(index, "", 0), + id == null ? "" : id, + 0, + 0, + version, + true, + pipelines, + IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN + ); this.source = source; this.sourceXContentType = sourceXContentType; setShardInfo(ShardInfo.EMPTY); + this.ignoredFields = ignoredFields; this.exception = exception; } @@ -73,6 +93,16 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t builder.field("_source", XContentHelper.convertToMap(source, false, sourceXContentType).v2()); assert executedPipelines != null : "executedPipelines is null when it shouldn't be - we always list pipelines in simulate mode"; builder.array("executed_pipelines", executedPipelines.toArray()); + if (ignoredFields.isEmpty() == false) { + builder.startArray("ignored_fields"); + for (String ignoredField : ignoredFields) { + builder.startObject(); + builder.field("field", ignoredField); + builder.endObject(); + } + ; + builder.endArray(); + } if (exception != null) { builder.startObject("error"); ElasticsearchException.generateThrowableXContent(builder, params, exception); @@ -91,9 +121,12 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeBytesReference(source); out.writeString(sourceXContentType.name()); - if (out.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_VALIDATES_MAPPINGS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { out.writeException(exception); } + if (out.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_IGNORED_FIELDS)) { + out.writeStringCollection(ignoredFields); + } } public Exception getException() { diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineAction.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineAction.java index d62bbbacd13ce..70a92df912ff4 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineAction.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.ingest; diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java index 39c96e1dd9ae0..6adcd80cb9b19 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.ingest; diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequestBuilder.java index 0c9995c6edf3c..05e30685c6a9b 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.ingest; diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java index e7ad434e3ad7c..3e5b99a810b5e 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.ingest; diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java index 2fc59086b54a6..6b2b96fb76402 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.ingest; diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java index 1d5776896ea19..55852b8ad392e 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.ingest; diff --git a/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java b/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java index a8fb51b30713c..6570343452afe 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.ingest; diff --git a/server/src/main/java/org/elasticsearch/action/package-info.java b/server/src/main/java/org/elasticsearch/action/package-info.java index 9d9c1a17ba00e..1d08a4d4d9c5e 100644 --- a/server/src/main/java/org/elasticsearch/action/package-info.java +++ b/server/src/main/java/org/elasticsearch/action/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/server/src/main/java/org/elasticsearch/action/resync/ResyncReplicationRequest.java b/server/src/main/java/org/elasticsearch/action/resync/ResyncReplicationRequest.java index 1dc15a6015a98..f58cbc2676dbf 100644 --- a/server/src/main/java/org/elasticsearch/action/resync/ResyncReplicationRequest.java +++ b/server/src/main/java/org/elasticsearch/action/resync/ResyncReplicationRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.resync; diff --git a/server/src/main/java/org/elasticsearch/action/resync/ResyncReplicationResponse.java b/server/src/main/java/org/elasticsearch/action/resync/ResyncReplicationResponse.java index ce94a5fff2da0..9d838d0ab46d4 100644 --- a/server/src/main/java/org/elasticsearch/action/resync/ResyncReplicationResponse.java +++ b/server/src/main/java/org/elasticsearch/action/resync/ResyncReplicationResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.resync; diff --git a/server/src/main/java/org/elasticsearch/action/resync/TransportResyncReplicationAction.java b/server/src/main/java/org/elasticsearch/action/resync/TransportResyncReplicationAction.java index e3f0f5842cfb3..8151155864a5f 100644 --- a/server/src/main/java/org/elasticsearch/action/resync/TransportResyncReplicationAction.java +++ b/server/src/main/java/org/elasticsearch/action/resync/TransportResyncReplicationAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.resync; diff --git a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java index 1e5b5ebbefe48..219e4a78ace46 100644 --- a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; @@ -20,14 +21,13 @@ import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.search.TransportSearchAction.SearchTimeProvider; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.action.support.TransportActions; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.AtomicArray; -import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.index.shard.ShardId; @@ -43,17 +43,20 @@ import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.transport.Transport; -import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.Executor; +import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Supplier; import java.util.stream.Collectors; import static org.elasticsearch.core.Strings.format; @@ -66,7 +69,7 @@ * The fan out and collect algorithm is traditionally used as the initial phase which can either be a query execution or collection of * distributed frequencies */ -abstract class AbstractSearchAsyncAction extends SearchPhase implements SearchPhaseContext { +abstract class AbstractSearchAsyncAction extends SearchPhase { private static final float DEFAULT_INDEX_BOOST = 1.0f; private final Logger logger; private final NamedWriteableRegistry namedWriteableRegistry; @@ -104,7 +107,8 @@ abstract class AbstractSearchAsyncAction exten private final boolean throttleConcurrentRequests; private final AtomicBoolean requestCancelled = new AtomicBoolean(); - private final List releasables = new ArrayList<>(); + // protected for tests + protected final List releasables = new ArrayList<>(); AbstractSearchAsyncAction( String name, @@ -192,7 +196,9 @@ protected void notifyListShards( ); } - @Override + /** + * Registers a {@link Releasable} that will be closed when the search request finishes or fails. + */ public void addReleasable(Releasable releasable) { releasables.add(releasable); } @@ -247,7 +253,12 @@ public final void run() { assert shardRoutings.skip() == false; assert shardIndexMap.containsKey(shardRoutings); int shardIndex = shardIndexMap.get(shardRoutings); - performPhaseOnShard(shardIndex, shardRoutings, shardRoutings.nextOrNull()); + final SearchShardTarget routing = shardRoutings.nextOrNull(); + if (routing == null) { + failOnUnavailable(shardIndex, shardRoutings); + } else { + performPhaseOnShard(shardIndex, shardRoutings, routing); + } } } } @@ -282,7 +293,7 @@ private static boolean assertExecuteOnStartThread() { int index = 0; assert stackTraceElements[index++].getMethodName().equals("getStackTrace"); assert stackTraceElements[index++].getMethodName().equals("assertExecuteOnStartThread"); - assert stackTraceElements[index++].getMethodName().equals("performPhaseOnShard"); + assert stackTraceElements[index++].getMethodName().equals("failOnUnavailable"); if (stackTraceElements[index].getMethodName().equals("performPhaseOnShard")) { assert stackTraceElements[index].getClassName().endsWith("CanMatchPreFilterSearchPhase"); index++; @@ -301,107 +312,69 @@ private static boolean assertExecuteOnStartThread() { } protected void performPhaseOnShard(final int shardIndex, final SearchShardIterator shardIt, final SearchShardTarget shard) { - /* - * We capture the thread that this phase is starting on. When we are called back after executing the phase, we are either on the - * same thread (because we never went async, or the same thread was selected from the thread pool) or a different thread. If we - * continue on the same thread in the case that we never went async and this happens repeatedly we will end up recursing deeply and - * could stack overflow. To prevent this, we fork if we are called back on the same thread that execution started on and otherwise - * we can continue (cf. InitialSearchPhase#maybeFork). - */ - if (shard == null) { - assert assertExecuteOnStartThread(); - SearchShardTarget unassignedShard = new SearchShardTarget(null, shardIt.shardId(), shardIt.getClusterAlias()); - onShardFailure(shardIndex, unassignedShard, shardIt, new NoShardAvailableActionException(shardIt.shardId())); + if (throttleConcurrentRequests) { + var pendingExecutions = pendingExecutionsPerNode.computeIfAbsent( + shard.getNodeId(), + n -> new PendingExecutions(maxConcurrentRequestsPerNode) + ); + pendingExecutions.submit(l -> doPerformPhaseOnShard(shardIndex, shardIt, shard, l)); } else { - final PendingExecutions pendingExecutions = throttleConcurrentRequests - ? pendingExecutionsPerNode.computeIfAbsent(shard.getNodeId(), n -> new PendingExecutions(maxConcurrentRequestsPerNode)) - : null; - Runnable r = () -> { - final Thread thread = Thread.currentThread(); - try { - executePhaseOnShard(shardIt, shard, new SearchActionListener<>(shard, shardIndex) { - @Override - public void innerOnResponse(Result result) { - try { - onShardResult(result, shardIt); - } catch (Exception exc) { - onShardFailure(shardIndex, shard, shardIt, exc); - } finally { - executeNext(pendingExecutions, thread); - } - } + doPerformPhaseOnShard(shardIndex, shardIt, shard, () -> {}); + } + } - @Override - public void onFailure(Exception t) { - try { - onShardFailure(shardIndex, shard, shardIt, t); - } finally { - executeNext(pendingExecutions, thread); - } - } - }); - } catch (final Exception e) { - try { - /* - * It is possible to run into connection exceptions here because we are getting the connection early and might - * run into nodes that are not connected. In this case, on shard failure will move us to the next shard copy. - */ - fork(() -> onShardFailure(shardIndex, shard, shardIt, e)); - } finally { - executeNext(pendingExecutions, thread); - } + private void doPerformPhaseOnShard(int shardIndex, SearchShardIterator shardIt, SearchShardTarget shard, Releasable releasable) { + var shardListener = new SearchActionListener(shard, shardIndex) { + @Override + public void innerOnResponse(Result result) { + try { + releasable.close(); + onShardResult(result, shardIt); + } catch (Exception exc) { + onShardFailure(shardIndex, shard, shardIt, exc); } - }; - if (throttleConcurrentRequests) { - pendingExecutions.tryRun(r); - } else { - r.run(); } + + @Override + public void onFailure(Exception e) { + releasable.close(); + onShardFailure(shardIndex, shard, shardIt, e); + } + }; + final Transport.Connection connection; + try { + connection = getConnection(shard.getClusterAlias(), shard.getNodeId()); + } catch (Exception e) { + shardListener.onFailure(e); + return; } + executePhaseOnShard(shardIt, connection, shardListener); + } + + private void failOnUnavailable(int shardIndex, SearchShardIterator shardIt) { + assert assertExecuteOnStartThread(); + SearchShardTarget unassignedShard = new SearchShardTarget(null, shardIt.shardId(), shardIt.getClusterAlias()); + onShardFailure(shardIndex, unassignedShard, shardIt, new NoShardAvailableActionException(shardIt.shardId())); } /** * Sends the request to the actual shard. * @param shardIt the shards iterator - * @param shard the shard routing to send the request for + * @param connection to node that the shard is located on * @param listener the listener to notify on response */ protected abstract void executePhaseOnShard( SearchShardIterator shardIt, - SearchShardTarget shard, + Transport.Connection connection, SearchActionListener listener ); - protected void fork(final Runnable runnable) { - executor.execute(new AbstractRunnable() { - @Override - public void onFailure(Exception e) { - logger.error(() -> "unexpected error during [" + task + "]", e); - assert false : e; - } - - @Override - public void onRejection(Exception e) { - // avoid leaks during node shutdown by executing on the current thread if the executor shuts down - assert e instanceof EsRejectedExecutionException esre && esre.isExecutorShutdown() : e; - doRun(); - } - - @Override - protected void doRun() { - runnable.run(); - } - - @Override - public boolean isForceExecution() { - // we can not allow a stuffed queue to reject execution here - return true; - } - }); - } - - @Override - public final void executeNextPhase(SearchPhase currentPhase, SearchPhase nextPhase) { + /** + * Processes the phase transition from on phase to another. This method handles all errors that happen during the initial run execution + * of the next phase. If there are no successful operations in the context when this method is executed the search is aborted and + * a response is returned to the user indicating that all shards have failed. + */ + protected void executeNextPhase(SearchPhase currentPhase, Supplier nextPhaseSupplier) { /* This is the main search phase transition where we move to the next phase. If all shards * failed or if there was a failure and partial results are not allowed, then we immediately * fail. Otherwise we continue to the next phase. @@ -445,6 +418,7 @@ public final void executeNextPhase(SearchPhase currentPhase, SearchPhase nextPha } return; } + var nextPhase = nextPhaseSupplier.get(); if (logger.isTraceEnabled()) { final String resultsFrom = results.getSuccessfulResults() .map(r -> r.getSearchShardTarget().toString()) @@ -536,8 +510,7 @@ protected void onShardGroupFailure(int shardIndex, SearchShardTarget shardTarget * @param shardTarget the shard target for this failure * @param e the failure reason */ - @Override - public final void onShardFailure(final int shardIndex, SearchShardTarget shardTarget, Exception e) { + void onShardFailure(final int shardIndex, SearchShardTarget shardTarget, Exception e) { if (TransportActions.isShardNotAvailableException(e)) { // Groups shard not available exceptions under a generic exception that returns a SERVICE_UNAVAILABLE(503) // temporary error. @@ -634,32 +607,45 @@ private void successfulShardExecution(SearchShardIterator shardsIt) { } } - @Override + /** + * Returns the total number of shards to the current search across all indices + */ public final int getNumShards() { return results.getNumShards(); } - @Override + /** + * Returns a logger for this context to prevent each individual phase to create their own logger. + */ public final Logger getLogger() { return logger; } - @Override + /** + * Returns the currently executing search task + */ public final SearchTask getTask() { return task; } - @Override + /** + * Returns the currently executing search request + */ public final SearchRequest getRequest() { return request; } - @Override + /** + * Returns the targeted {@link OriginalIndices} for the provided {@code shardIndex}. + */ public OriginalIndices getOriginalIndices(int shardIndex) { return shardIterators[shardIndex].getOriginalIndices(); } - @Override + /** + * Checks if the given context id is part of the point in time of this search (if exists). + * We should not release search contexts that belong to the point in time during or after searches. + */ public boolean isPartOfPointInTime(ShardSearchContextId contextId) { final PointInTimeBuilder pointInTimeBuilder = request.pointInTimeBuilder(); if (pointInTimeBuilder != null) { @@ -696,7 +682,12 @@ boolean buildPointInTimeFromSearchResults() { return false; } - @Override + /** + * Builds and sends the final search response back to the user. + * + * @param internalSearchResponse the internal search response + * @param queryResults the results of the query phase + */ public void sendSearchResponse(SearchResponseSections internalSearchResponse, AtomicArray queryResults) { ShardSearchFailure[] failures = buildShardFailures(); Boolean allowPartialResults = request.allowPartialSearchResults(); @@ -721,8 +712,14 @@ public void sendSearchResponse(SearchResponseSections internalSearchResponse, At } } - @Override - public final void onPhaseFailure(SearchPhase phase, String msg, Throwable cause) { + /** + * This method will communicate a fatal phase failure back to the user. In contrast to a shard failure + * will this method immediately fail the search request and return the failure to the issuer of the request + * @param phase the phase that failed + * @param msg an optional message + * @param cause the cause of the phase failure + */ + public void onPhaseFailure(SearchPhase phase, String msg, Throwable cause) { raisePhaseFailure(new SearchPhaseExecutionException(phase.getName(), msg, cause, buildShardFailures())); } @@ -739,7 +736,7 @@ private void raisePhaseFailure(SearchPhaseExecutionException exception) { try { SearchShardTarget searchShardTarget = entry.getSearchShardTarget(); Transport.Connection connection = getConnection(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId()); - sendReleaseSearchContext(entry.getContextId(), connection, getOriginalIndices(entry.getShardIndex())); + sendReleaseSearchContext(entry.getContextId(), connection); } catch (Exception inner) { inner.addSuppressed(exception); logger.trace("failed to release context", inner); @@ -749,16 +746,32 @@ private void raisePhaseFailure(SearchPhaseExecutionException exception) { listener.onFailure(exception); } + /** + * Releases a search context with the given context ID on the node the given connection is connected to. + * @see org.elasticsearch.search.query.QuerySearchResult#getContextId() + * @see org.elasticsearch.search.fetch.FetchSearchResult#getContextId() + * + */ + void sendReleaseSearchContext(ShardSearchContextId contextId, Transport.Connection connection) { + assert isPartOfPointInTime(contextId) == false : "Must not release point in time context [" + contextId + "]"; + if (connection != null) { + searchTransportService.sendFreeContext(connection, contextId, ActionListener.noop()); + } + } + /** * Executed once all shard results have been received and processed * @see #onShardFailure(int, SearchShardTarget, Exception) * @see #onShardResult(SearchPhaseResult, SearchShardIterator) */ - final void onPhaseDone() { // as a tribute to @kimchy aka. finishHim() - executeNextPhase(this, getNextPhase(results, this)); + private void onPhaseDone() { // as a tribute to @kimchy aka. finishHim() + executeNextPhase(this, this::getNextPhase); } - @Override + /** + * Returns a connection to the node if connected otherwise and {@link org.elasticsearch.transport.ConnectTransportException} will be + * thrown. + */ public final Transport.Connection getConnection(String clusterAlias, String nodeId) { Transport.Connection conn = nodeIdToConnection.apply(clusterAlias, nodeId); Version minVersion = request.minCompatibleShardNode(); @@ -768,21 +781,17 @@ public final Transport.Connection getConnection(String clusterAlias, String node return conn; } - @Override - public final SearchTransportService getSearchTransport() { + /** + * Returns the {@link SearchTransportService} to send shard request to other nodes + */ + public SearchTransportService getSearchTransport() { return searchTransportService; } - @Override public final void execute(Runnable command) { executor.execute(command); } - @Override - public final void onFailure(Exception e) { - listener.onFailure(e); - } - /** * Builds an request for the initial search phase. * @@ -817,67 +826,73 @@ protected final ShardSearchRequest buildShardSearchRequest(SearchShardIterator s /** * Returns the next phase based on the results of the initial search phase - * @param results the results of the initial search phase. Each non null element in the result array represent a successfully - * executed shard request - * @param context the search context for the next phase */ - protected abstract SearchPhase getNextPhase(SearchPhaseResults results, SearchPhaseContext context); - - private void executeNext(PendingExecutions pendingExecutions, Thread originalThread) { - executeNext(pendingExecutions == null ? null : pendingExecutions.finishAndGetNext(), originalThread); - } - - void executeNext(Runnable runnable, Thread originalThread) { - if (runnable != null) { - assert throttleConcurrentRequests; - if (originalThread == Thread.currentThread()) { - fork(runnable); - } else { - runnable.run(); - } - } - } + protected abstract SearchPhase getNextPhase(); private static final class PendingExecutions { - private final int permits; - private int permitsTaken = 0; - private final ArrayDeque queue = new ArrayDeque<>(); + private final Semaphore semaphore; + private final ConcurrentLinkedQueue> queue = new ConcurrentLinkedQueue<>(); PendingExecutions(int permits) { assert permits > 0 : "not enough permits: " + permits; - this.permits = permits; + semaphore = new Semaphore(permits); } - Runnable finishAndGetNext() { - synchronized (this) { - permitsTaken--; - assert permitsTaken >= 0 : "illegal taken permits: " + permitsTaken; + void submit(Consumer task) { + if (semaphore.tryAcquire()) { + executeAndRelease(task); + } else { + queue.add(task); + if (semaphore.tryAcquire()) { + task = pollNextTaskOrReleasePermit(); + if (task != null) { + executeAndRelease(task); + } + } } - return tryQueue(null); } - void tryRun(Runnable runnable) { - Runnable r = tryQueue(runnable); - if (r != null) { - r.run(); - } - } + private void executeAndRelease(Consumer task) { + do { + final SubscribableListener onDone = new SubscribableListener<>(); + task.accept(() -> onDone.onResponse(null)); + if (onDone.isDone()) { + // keep going on the current thread, no need to fork + task = pollNextTaskOrReleasePermit(); + } else { + onDone.addListener(new ActionListener<>() { + @Override + public void onResponse(Void unused) { + final Consumer nextTask = pollNextTaskOrReleasePermit(); + if (nextTask != null) { + executeAndRelease(nextTask); + } + } - private synchronized Runnable tryQueue(Runnable runnable) { - Runnable toExecute = null; - if (permitsTaken < permits) { - permitsTaken++; - toExecute = runnable; - if (toExecute == null) { // only poll if we don't have anything to execute - toExecute = queue.poll(); + @Override + public void onFailure(Exception e) { + assert false : e; + } + }); + return; } - if (toExecute == null) { - permitsTaken--; + } while (task != null); + } + + private Consumer pollNextTaskOrReleasePermit() { + var task = queue.poll(); + if (task == null) { + semaphore.release(); + while (queue.peek() != null && semaphore.tryAcquire()) { + task = queue.poll(); + if (task == null) { + semaphore.release(); + } else { + return task; + } } - } else if (runnable != null) { - queue.add(runnable); } - return toExecute; + return task; } } } diff --git a/server/src/main/java/org/elasticsearch/action/search/ArraySearchPhaseResults.java b/server/src/main/java/org/elasticsearch/action/search/ArraySearchPhaseResults.java index 96f10d7d8a30e..e4d1f9f63bd2b 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ArraySearchPhaseResults.java +++ b/server/src/main/java/org/elasticsearch/action/search/ArraySearchPhaseResults.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/BottomSortValuesCollector.java b/server/src/main/java/org/elasticsearch/action/search/BottomSortValuesCollector.java index 4461b71be9047..8ac2033e2ff19 100644 --- a/server/src/main/java/org/elasticsearch/action/search/BottomSortValuesCollector.java +++ b/server/src/main/java/org/elasticsearch/action/search/BottomSortValuesCollector.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/CCSSingleCoordinatorSearchProgressListener.java b/server/src/main/java/org/elasticsearch/action/search/CCSSingleCoordinatorSearchProgressListener.java index 0504d0cde8986..2663b6d9baaa6 100644 --- a/server/src/main/java/org/elasticsearch/action/search/CCSSingleCoordinatorSearchProgressListener.java +++ b/server/src/main/java/org/elasticsearch/action/search/CCSSingleCoordinatorSearchProgressListener.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/CanMatchNodeRequest.java b/server/src/main/java/org/elasticsearch/action/search/CanMatchNodeRequest.java index 30fa4caa79a15..ba1afaf4678fb 100644 --- a/server/src/main/java/org/elasticsearch/action/search/CanMatchNodeRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/CanMatchNodeRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/CanMatchNodeResponse.java b/server/src/main/java/org/elasticsearch/action/search/CanMatchNodeResponse.java index 9909ad60a949e..82c5784aaedd3 100644 --- a/server/src/main/java/org/elasticsearch/action/search/CanMatchNodeResponse.java +++ b/server/src/main/java/org/elasticsearch/action/search/CanMatchNodeResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java index 30460593849c5..a93f763933eeb 100644 --- a/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; @@ -131,7 +132,6 @@ private static boolean assertSearchCoordinationThread() { @Override public void run() { assert assertSearchCoordinationThread(); - checkNoMissingShards(); Version version = request.minCompatibleShardNode(); if (version != null && Version.CURRENT.minimumCompatibilityVersion().equals(version) == false) { if (checkMinimumVersion(shardsIts) == false) { @@ -184,7 +184,10 @@ private void runCoordinatorRewritePhase() { if (matchedShardLevelRequests.isEmpty()) { finishPhase(); } else { - new Round(new GroupShardsIterator<>(matchedShardLevelRequests)).run(); + GroupShardsIterator matchingShards = new GroupShardsIterator<>(matchedShardLevelRequests); + // verify missing shards only for the shards that we hit for the query + checkNoMissingShards(matchingShards); + new Round(matchingShards).run(); } } @@ -194,9 +197,9 @@ private void consumeResult(boolean canMatch, ShardSearchRequest request) { results.consumeResult(result, () -> {}); } - private void checkNoMissingShards() { + private void checkNoMissingShards(GroupShardsIterator shards) { assert assertSearchCoordinationThread(); - doCheckNoMissingShards(getName(), request, shardsIts); + doCheckNoMissingShards(getName(), request, shards); } private Map> groupByNode(GroupShardsIterator shards) { diff --git a/server/src/main/java/org/elasticsearch/action/search/ClearScrollController.java b/server/src/main/java/org/elasticsearch/action/search/ClearScrollController.java index 965b19a69b858..bbd290a06a7f0 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ClearScrollController.java +++ b/server/src/main/java/org/elasticsearch/action/search/ClearScrollController.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/ClearScrollRequest.java b/server/src/main/java/org/elasticsearch/action/search/ClearScrollRequest.java index 6309c0264a558..4909d8c4d229d 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ClearScrollRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/ClearScrollRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/ClearScrollRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/search/ClearScrollRequestBuilder.java index 16a81def8c67c..eae988389cc8e 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ClearScrollRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/search/ClearScrollRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/ClearScrollResponse.java b/server/src/main/java/org/elasticsearch/action/search/ClearScrollResponse.java index de0d7b98ef851..694f490985ea0 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ClearScrollResponse.java +++ b/server/src/main/java/org/elasticsearch/action/search/ClearScrollResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/ClosePointInTimeRequest.java b/server/src/main/java/org/elasticsearch/action/search/ClosePointInTimeRequest.java index c2afb8fc05c46..b36ec586535e8 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ClosePointInTimeRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/ClosePointInTimeRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/ClosePointInTimeResponse.java b/server/src/main/java/org/elasticsearch/action/search/ClosePointInTimeResponse.java index 09cd96289416f..91463d79a7c84 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ClosePointInTimeResponse.java +++ b/server/src/main/java/org/elasticsearch/action/search/ClosePointInTimeResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/CountOnlyQueryPhaseResultConsumer.java b/server/src/main/java/org/elasticsearch/action/search/CountOnlyQueryPhaseResultConsumer.java index 2c4cb31584323..d41a2561646b8 100644 --- a/server/src/main/java/org/elasticsearch/action/search/CountOnlyQueryPhaseResultConsumer.java +++ b/server/src/main/java/org/elasticsearch/action/search/CountOnlyQueryPhaseResultConsumer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/CountedCollector.java b/server/src/main/java/org/elasticsearch/action/search/CountedCollector.java index 0e6830dcfab0e..3d15e11a19d31 100644 --- a/server/src/main/java/org/elasticsearch/action/search/CountedCollector.java +++ b/server/src/main/java/org/elasticsearch/action/search/CountedCollector.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; @@ -21,9 +22,9 @@ final class CountedCollector { private final SearchPhaseResults resultConsumer; private final CountDown counter; private final Runnable onFinish; - private final SearchPhaseContext context; + private final AbstractSearchAsyncAction context; - CountedCollector(SearchPhaseResults resultConsumer, int expectedOps, Runnable onFinish, SearchPhaseContext context) { + CountedCollector(SearchPhaseResults resultConsumer, int expectedOps, Runnable onFinish, AbstractSearchAsyncAction context) { this.resultConsumer = resultConsumer; this.counter = new CountDown(expectedOps); this.onFinish = onFinish; @@ -49,7 +50,7 @@ void onResult(R result) { } /** - * Escalates the failure via {@link SearchPhaseContext#onShardFailure(int, SearchShardTarget, Exception)} + * Escalates the failure via {@link AbstractSearchAsyncAction#onShardFailure(int, SearchShardTarget, Exception)} * and then runs {@link #countDown()} */ void onFailure(final int shardIndex, @Nullable SearchShardTarget shardTarget, Exception e) { diff --git a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java index 7a33eaa59eb03..cc8c4becea9a9 100644 --- a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; @@ -43,7 +44,7 @@ final class DfsQueryPhase extends SearchPhase { private final AggregatedDfs dfs; private final List knnResults; private final Function, SearchPhase> nextPhaseFactory; - private final SearchPhaseContext context; + private final AbstractSearchAsyncAction context; private final SearchTransportService searchTransportService; private final SearchProgressListener progressListener; @@ -53,7 +54,7 @@ final class DfsQueryPhase extends SearchPhase { List knnResults, SearchPhaseResults queryResult, Function, SearchPhase> nextPhaseFactory, - SearchPhaseContext context + AbstractSearchAsyncAction context ) { super("dfs_query"); this.progressListener = context.getTask().getProgressListener(); @@ -64,10 +65,6 @@ final class DfsQueryPhase extends SearchPhase { this.nextPhaseFactory = nextPhaseFactory; this.context = context; this.searchTransportService = context.getSearchTransport(); - - // register the release of the query consumer to free up the circuit breaker memory - // at the end of the search - context.addReleasable(queryResult); } @Override @@ -77,21 +74,26 @@ public void run() { final CountedCollector counter = new CountedCollector<>( queryResult, searchResults.size(), - () -> context.executeNextPhase(this, nextPhaseFactory.apply(queryResult)), + () -> context.executeNextPhase(this, () -> nextPhaseFactory.apply(queryResult)), context ); for (final DfsSearchResult dfsResult : searchResults) { final SearchShardTarget shardTarget = dfsResult.getSearchShardTarget(); - Transport.Connection connection = context.getConnection(shardTarget.getClusterAlias(), shardTarget.getNodeId()); - ShardSearchRequest shardRequest = rewriteShardSearchRequest(dfsResult.getShardSearchRequest()); + final int shardIndex = dfsResult.getShardIndex(); QuerySearchRequest querySearchRequest = new QuerySearchRequest( - context.getOriginalIndices(dfsResult.getShardIndex()), + context.getOriginalIndices(shardIndex), dfsResult.getContextId(), - shardRequest, + rewriteShardSearchRequest(dfsResult.getShardSearchRequest()), dfs ); - final int shardIndex = dfsResult.getShardIndex(); + final Transport.Connection connection; + try { + connection = context.getConnection(shardTarget.getClusterAlias(), shardTarget.getNodeId()); + } catch (Exception e) { + shardFailure(e, querySearchRequest, shardIndex, shardTarget, counter); + continue; + } searchTransportService.sendExecuteQuery( connection, querySearchRequest, @@ -111,20 +113,13 @@ protected void innerOnResponse(QuerySearchResult response) { @Override public void onFailure(Exception exception) { try { - context.getLogger() - .debug(() -> "[" + querySearchRequest.contextId() + "] Failed to execute query phase", exception); - progressListener.notifyQueryFailure(shardIndex, shardTarget, exception); - counter.onFailure(shardIndex, shardTarget, exception); + shardFailure(exception, querySearchRequest, shardIndex, shardTarget, counter); } finally { if (context.isPartOfPointInTime(querySearchRequest.contextId()) == false) { // the query might not have been executed at all (for example because thread pool rejected // execution) and the search context that was created in dfs phase might not be released. // release it again to be in the safe side - context.sendReleaseSearchContext( - querySearchRequest.contextId(), - connection, - context.getOriginalIndices(shardIndex) - ); + context.sendReleaseSearchContext(querySearchRequest.contextId(), connection); } } } @@ -133,6 +128,18 @@ public void onFailure(Exception exception) { } } + private void shardFailure( + Exception exception, + QuerySearchRequest querySearchRequest, + int shardIndex, + SearchShardTarget shardTarget, + CountedCollector counter + ) { + context.getLogger().debug(() -> "[" + querySearchRequest.contextId() + "] Failed to execute query phase", exception); + progressListener.notifyQueryFailure(shardIndex, shardTarget, exception); + counter.onFailure(shardIndex, shardTarget, exception); + } + // package private for testing ShardSearchRequest rewriteShardSearchRequest(ShardSearchRequest request) { SearchSourceBuilder source = request.source(); diff --git a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java index e2385745149c1..e8d94c32bdcc7 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java @@ -1,14 +1,16 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.Maps; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.InnerHitBuilder; @@ -29,11 +31,11 @@ * forwards to the next phase immediately. */ final class ExpandSearchPhase extends SearchPhase { - private final SearchPhaseContext context; + private final AbstractSearchAsyncAction context; private final SearchHits searchHits; private final Supplier nextPhase; - ExpandSearchPhase(SearchPhaseContext context, SearchHits searchHits, Supplier nextPhase) { + ExpandSearchPhase(AbstractSearchAsyncAction context, SearchHits searchHits, Supplier nextPhase) { super("expand"); this.context = context; this.searchHits = searchHits; @@ -81,8 +83,15 @@ private void doRun() { CollapseBuilder innerCollapseBuilder = innerHitBuilder.getInnerCollapseBuilder(); SearchSourceBuilder sourceBuilder = buildExpandSearchSourceBuilder(innerHitBuilder, innerCollapseBuilder).query(groupQuery) .postFilter(searchRequest.source().postFilter()) - .runtimeMappings(searchRequest.source().runtimeMappings()); + .runtimeMappings(searchRequest.source().runtimeMappings()) + .pointInTimeBuilder(searchRequest.source().pointInTimeBuilder()); SearchRequest groupRequest = new SearchRequest(searchRequest); + if (searchRequest.pointInTimeBuilder() != null) { + // if the original request has a point in time, we propagate it to the inner search request + // and clear the indices and preference from the inner search request + groupRequest.indices(Strings.EMPTY_ARRAY); + groupRequest.preference(null); + } groupRequest.source(sourceBuilder); multiRequest.add(groupRequest); } @@ -93,7 +102,7 @@ private void doRun() { for (InnerHitBuilder innerHitBuilder : innerHitBuilders) { MultiSearchResponse.Item item = it.next(); if (item.isFailure()) { - context.onPhaseFailure(this, "failed to expand hits", item.getFailure()); + phaseFailure(item.getFailure()); return; } SearchHits innerHits = item.getResponse().getHits(); @@ -110,7 +119,11 @@ private void doRun() { } } onPhaseDone(); - }, context::onFailure)); + }, this::phaseFailure)); + } + + private void phaseFailure(Exception ex) { + context.onPhaseFailure(this, "failed to expand hits", ex); } private static SearchSourceBuilder buildExpandSearchSourceBuilder(InnerHitBuilder options, CollapseBuilder innerCollapseBuilder) { @@ -155,6 +168,6 @@ private static SearchSourceBuilder buildExpandSearchSourceBuilder(InnerHitBuilde } private void onPhaseDone() { - context.executeNextPhase(this, nextPhase.get()); + context.executeNextPhase(this, nextPhase); } } diff --git a/server/src/main/java/org/elasticsearch/action/search/FetchLookupFieldsPhase.java b/server/src/main/java/org/elasticsearch/action/search/FetchLookupFieldsPhase.java index 417e5e40f6fdc..d8671bcadf86d 100644 --- a/server/src/main/java/org/elasticsearch/action/search/FetchLookupFieldsPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/FetchLookupFieldsPhase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; @@ -32,11 +33,15 @@ * @see org.elasticsearch.index.mapper.LookupRuntimeFieldType */ final class FetchLookupFieldsPhase extends SearchPhase { - private final SearchPhaseContext context; + private final AbstractSearchAsyncAction context; private final SearchResponseSections searchResponse; private final AtomicArray queryResults; - FetchLookupFieldsPhase(SearchPhaseContext context, SearchResponseSections searchResponse, AtomicArray queryResults) { + FetchLookupFieldsPhase( + AbstractSearchAsyncAction context, + SearchResponseSections searchResponse, + AtomicArray queryResults + ) { super("fetch_lookup_fields"); this.context = context; this.searchResponse = searchResponse; diff --git a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java index c81f5a20bc2d1..8568b60916761 100644 --- a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; @@ -11,6 +12,7 @@ import org.apache.lucene.search.ScoreDoc; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.AtomicArray; +import org.elasticsearch.core.Nullable; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.dfs.AggregatedDfs; @@ -19,6 +21,7 @@ import org.elasticsearch.search.internal.ShardSearchContextId; import org.elasticsearch.search.rank.RankDoc; import org.elasticsearch.search.rank.RankDocShardInfo; +import org.elasticsearch.transport.Transport; import java.util.ArrayList; import java.util.HashMap; @@ -31,20 +34,21 @@ * Then it reaches out to all relevant shards to fetch the topN hits. */ final class FetchSearchPhase extends SearchPhase { - private final ArraySearchPhaseResults fetchResults; private final AtomicArray searchPhaseShardResults; private final BiFunction, SearchPhase> nextPhaseFactory; - private final SearchPhaseContext context; + private final AbstractSearchAsyncAction context; private final Logger logger; private final SearchProgressListener progressListener; private final AggregatedDfs aggregatedDfs; + @Nullable + private final SearchPhaseResults resultConsumer; private final SearchPhaseController.ReducedQueryPhase reducedQueryPhase; FetchSearchPhase( SearchPhaseResults resultConsumer, AggregatedDfs aggregatedDfs, - SearchPhaseContext context, - SearchPhaseController.ReducedQueryPhase reducedQueryPhase + AbstractSearchAsyncAction context, + @Nullable SearchPhaseController.ReducedQueryPhase reducedQueryPhase ) { this( resultConsumer, @@ -62,8 +66,8 @@ final class FetchSearchPhase extends SearchPhase { FetchSearchPhase( SearchPhaseResults resultConsumer, AggregatedDfs aggregatedDfs, - SearchPhaseContext context, - SearchPhaseController.ReducedQueryPhase reducedQueryPhase, + AbstractSearchAsyncAction context, + @Nullable SearchPhaseController.ReducedQueryPhase reducedQueryPhase, BiFunction, SearchPhase> nextPhaseFactory ) { super("fetch"); @@ -75,8 +79,6 @@ final class FetchSearchPhase extends SearchPhase { + resultConsumer.getNumShards() ); } - this.fetchResults = new ArraySearchPhaseResults<>(resultConsumer.getNumShards()); - context.addReleasable(fetchResults); this.searchPhaseShardResults = resultConsumer.getAtomicArray(); this.aggregatedDfs = aggregatedDfs; this.nextPhaseFactory = nextPhaseFactory; @@ -84,6 +86,7 @@ final class FetchSearchPhase extends SearchPhase { this.logger = context.getLogger(); this.progressListener = context.getTask().getProgressListener(); this.reducedQueryPhase = reducedQueryPhase; + this.resultConsumer = reducedQueryPhase == null ? resultConsumer : null; } @Override @@ -91,7 +94,7 @@ public void run() { context.execute(new AbstractRunnable() { @Override - protected void doRun() { + protected void doRun() throws Exception { innerRun(); } @@ -102,17 +105,21 @@ public void onFailure(Exception e) { }); } - private void innerRun() { + private void innerRun() throws Exception { + assert this.reducedQueryPhase == null ^ this.resultConsumer == null; + // depending on whether we executed the RankFeaturePhase we may or may not have the reduced query result computed already + final var reducedQueryPhase = this.reducedQueryPhase == null ? resultConsumer.reduce() : this.reducedQueryPhase; final int numShards = context.getNumShards(); // Usually when there is a single shard, we force the search type QUERY_THEN_FETCH. But when there's kNN, we might // still use DFS_QUERY_THEN_FETCH, which does not perform the "query and fetch" optimization during the query phase. final boolean queryAndFetchOptimization = searchPhaseShardResults.length() == 1 && context.getRequest().hasKnnSearch() == false - && reducedQueryPhase.queryPhaseRankCoordinatorContext() == null; + && reducedQueryPhase.queryPhaseRankCoordinatorContext() == null + && (context.getRequest().source() == null || context.getRequest().source().rankBuilder() == null); if (queryAndFetchOptimization) { assert assertConsistentWithQueryAndFetchOptimization(); // query AND fetch optimization - moveToNextPhase(searchPhaseShardResults); + moveToNextPhase(searchPhaseShardResults, reducedQueryPhase); } else { ScoreDoc[] scoreDocs = reducedQueryPhase.sortedTopDocs().scoreDocs(); // no docs to fetch -- sidestep everything and return @@ -120,48 +127,56 @@ private void innerRun() { // we have to release contexts here to free up resources searchPhaseShardResults.asList() .forEach(searchPhaseShardResult -> releaseIrrelevantSearchContext(searchPhaseShardResult, context)); - moveToNextPhase(fetchResults.getAtomicArray()); + moveToNextPhase(new AtomicArray<>(numShards), reducedQueryPhase); } else { - final boolean shouldExplainRank = shouldExplainRankScores(context.getRequest()); - final List> rankDocsPerShard = false == shouldExplainRank - ? null - : splitRankDocsPerShard(scoreDocs, numShards); - final ScoreDoc[] lastEmittedDocPerShard = context.getRequest().scroll() != null - ? SearchPhaseController.getLastEmittedDocPerShard(reducedQueryPhase, numShards) - : null; - final List[] docIdsToLoad = SearchPhaseController.fillDocIdsToLoad(numShards, scoreDocs); - final CountedCollector counter = new CountedCollector<>( - fetchResults, - docIdsToLoad.length, // we count down every shard in the result no matter if we got any results or not - () -> moveToNextPhase(fetchResults.getAtomicArray()), - context - ); - for (int i = 0; i < docIdsToLoad.length; i++) { - List entry = docIdsToLoad[i]; - RankDocShardInfo rankDocs = rankDocsPerShard == null || rankDocsPerShard.get(i).isEmpty() - ? null - : new RankDocShardInfo(rankDocsPerShard.get(i)); - SearchPhaseResult shardPhaseResult = searchPhaseShardResults.get(i); - if (entry == null) { // no results for this shard ID - if (shardPhaseResult != null) { - // if we got some hits from this shard we have to release the context there - // we do this as we go since it will free up resources and passing on the request on the - // transport layer is cheap. - releaseIrrelevantSearchContext(shardPhaseResult, context); - progressListener.notifyFetchResult(i); - } - // in any case we count down this result since we don't talk to this shard anymore - counter.countDown(); - } else { - executeFetch( - shardPhaseResult, - counter, - entry, - rankDocs, - (lastEmittedDocPerShard != null) ? lastEmittedDocPerShard[i] : null - ); - } + innerRunFetch(scoreDocs, numShards, reducedQueryPhase); + } + } + } + + private void innerRunFetch(ScoreDoc[] scoreDocs, int numShards, SearchPhaseController.ReducedQueryPhase reducedQueryPhase) { + ArraySearchPhaseResults fetchResults = new ArraySearchPhaseResults<>(numShards); + final List> rankDocsPerShard = false == shouldExplainRankScores(context.getRequest()) + ? null + : splitRankDocsPerShard(scoreDocs, numShards); + final ScoreDoc[] lastEmittedDocPerShard = context.getRequest().scroll() != null + ? SearchPhaseController.getLastEmittedDocPerShard(reducedQueryPhase, numShards) + : null; + final List[] docIdsToLoad = SearchPhaseController.fillDocIdsToLoad(numShards, scoreDocs); + final CountedCollector counter = new CountedCollector<>( + fetchResults, + docIdsToLoad.length, // we count down every shard in the result no matter if we got any results or not + () -> { + try (fetchResults) { + moveToNextPhase(fetchResults.getAtomicArray(), reducedQueryPhase); } + }, + context + ); + for (int i = 0; i < docIdsToLoad.length; i++) { + List entry = docIdsToLoad[i]; + RankDocShardInfo rankDocs = rankDocsPerShard == null || rankDocsPerShard.get(i).isEmpty() + ? null + : new RankDocShardInfo(rankDocsPerShard.get(i)); + SearchPhaseResult shardPhaseResult = searchPhaseShardResults.get(i); + if (entry == null) { // no results for this shard ID + if (shardPhaseResult != null) { + // if we got some hits from this shard we have to release the context there + // we do this as we go since it will free up resources and passing on the request on the + // transport layer is cheap. + releaseIrrelevantSearchContext(shardPhaseResult, context); + progressListener.notifyFetchResult(i); + } + // in any case we count down this result since we don't talk to this shard anymore + counter.countDown(); + } else { + executeFetch( + shardPhaseResult, + counter, + entry, + rankDocs, + (lastEmittedDocPerShard != null) ? lastEmittedDocPerShard[i] : null + ); } } } @@ -200,9 +215,41 @@ private void executeFetch( final ShardSearchContextId contextId = shardPhaseResult.queryResult() != null ? shardPhaseResult.queryResult().getContextId() : shardPhaseResult.rankFeatureResult().getContextId(); + var listener = new SearchActionListener(shardTarget, shardIndex) { + @Override + public void innerOnResponse(FetchSearchResult result) { + try { + progressListener.notifyFetchResult(shardIndex); + counter.onResult(result); + } catch (Exception e) { + context.onPhaseFailure(FetchSearchPhase.this, "", e); + } + } + + @Override + public void onFailure(Exception e) { + try { + logger.debug(() -> "[" + contextId + "] Failed to execute fetch phase", e); + progressListener.notifyFetchFailure(shardIndex, shardTarget, e); + counter.onFailure(shardIndex, shardTarget, e); + } finally { + // the search context might not be cleared on the node where the fetch was executed for example + // because the action was rejected by the thread pool. in this case we need to send a dedicated + // request to clear the search context. + releaseIrrelevantSearchContext(shardPhaseResult, context); + } + } + }; + final Transport.Connection connection; + try { + connection = context.getConnection(shardTarget.getClusterAlias(), shardTarget.getNodeId()); + } catch (Exception e) { + listener.onFailure(e); + return; + } context.getSearchTransport() .sendExecuteFetch( - context.getConnection(shardTarget.getClusterAlias(), shardTarget.getNodeId()), + connection, new ShardFetchSearchRequest( context.getOriginalIndices(shardPhaseResult.getShardIndex()), contextId, @@ -214,39 +261,19 @@ private void executeFetch( aggregatedDfs ), context.getTask(), - new SearchActionListener<>(shardTarget, shardIndex) { - @Override - public void innerOnResponse(FetchSearchResult result) { - try { - progressListener.notifyFetchResult(shardIndex); - counter.onResult(result); - } catch (Exception e) { - context.onPhaseFailure(FetchSearchPhase.this, "", e); - } - } - - @Override - public void onFailure(Exception e) { - try { - logger.debug(() -> "[" + contextId + "] Failed to execute fetch phase", e); - progressListener.notifyFetchFailure(shardIndex, shardTarget, e); - counter.onFailure(shardIndex, shardTarget, e); - } finally { - // the search context might not be cleared on the node where the fetch was executed for example - // because the action was rejected by the thread pool. in this case we need to send a dedicated - // request to clear the search context. - releaseIrrelevantSearchContext(shardPhaseResult, context); - } - } - } + listener ); } - private void moveToNextPhase(AtomicArray fetchResultsArr) { - var resp = SearchPhaseController.merge(context.getRequest().scroll() != null, reducedQueryPhase, fetchResultsArr); - context.addReleasable(resp::decRef); - fetchResults.close(); - context.executeNextPhase(this, nextPhaseFactory.apply(resp, searchPhaseShardResults)); + private void moveToNextPhase( + AtomicArray fetchResultsArr, + SearchPhaseController.ReducedQueryPhase reducedQueryPhase + ) { + context.executeNextPhase(this, () -> { + var resp = SearchPhaseController.merge(context.getRequest().scroll() != null, reducedQueryPhase, fetchResultsArr); + context.addReleasable(resp); + return nextPhaseFactory.apply(resp, searchPhaseShardResults); + }); } private boolean shouldExplainRankScores(SearchRequest request) { diff --git a/server/src/main/java/org/elasticsearch/action/search/MaxScoreCollector.java b/server/src/main/java/org/elasticsearch/action/search/MaxScoreCollector.java index c07810518ea29..e4af6c01f8262 100644 --- a/server/src/main/java/org/elasticsearch/action/search/MaxScoreCollector.java +++ b/server/src/main/java/org/elasticsearch/action/search/MaxScoreCollector.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java index fa22cb36b70ab..8467ee6fd86f3 100644 --- a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequestBuilder.java index 67c9a33a2e801..4e80ede924751 100644 --- a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java b/server/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java index 2388ee77f3efe..a9c512380e973 100644 --- a/server/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java +++ b/server/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeRequest.java b/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeRequest.java index 146418839f063..d68e2ce1b02b7 100644 --- a/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; @@ -62,7 +63,7 @@ public OpenPointInTimeRequest(StreamInput in) throws IOException { if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) { this.indexFilter = in.readOptionalNamedWriteable(QueryBuilder.class); } - if (in.getTransportVersion().onOrAfter(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { this.allowPartialSearchResults = in.readBoolean(); } } @@ -81,7 +82,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) { out.writeOptionalWriteable(indexFilter); } - if (out.getTransportVersion().onOrAfter(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeBoolean(allowPartialSearchResults); } else if (allowPartialSearchResults) { throw new IOException("[allow_partial_search_results] is not supported on nodes with version " + out.getTransportVersion()); diff --git a/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeResponse.java b/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeResponse.java index 4a4c0252fb109..b3ffc564d848c 100644 --- a/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeResponse.java +++ b/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; @@ -46,7 +47,7 @@ public OpenPointInTimeResponse( @Override public void writeTo(StreamOutput out) throws IOException { out.writeBytesReference(pointInTimeId); - if (out.getTransportVersion().onOrAfter(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeVInt(totalShards); out.writeVInt(successfulShards); out.writeVInt(failedShards); diff --git a/server/src/main/java/org/elasticsearch/action/search/ParsedScrollId.java b/server/src/main/java/org/elasticsearch/action/search/ParsedScrollId.java index a9f3502bfa631..85d3b8e875e51 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ParsedScrollId.java +++ b/server/src/main/java/org/elasticsearch/action/search/ParsedScrollId.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/QueryPhaseResultConsumer.java b/server/src/main/java/org/elasticsearch/action/search/QueryPhaseResultConsumer.java index 055274069c9c7..89411ac302b10 100644 --- a/server/src/main/java/org/elasticsearch/action/search/QueryPhaseResultConsumer.java +++ b/server/src/main/java/org/elasticsearch/action/search/QueryPhaseResultConsumer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java b/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java index 0f7cbd65a63c2..199228c9f992c 100644 --- a/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; @@ -23,6 +24,7 @@ import org.elasticsearch.search.rank.feature.RankFeatureDoc; import org.elasticsearch.search.rank.feature.RankFeatureResult; import org.elasticsearch.search.rank.feature.RankFeatureShardRequest; +import org.elasticsearch.transport.Transport; import java.util.List; @@ -36,20 +38,22 @@ public class RankFeaturePhase extends SearchPhase { private static final Logger logger = LogManager.getLogger(RankFeaturePhase.class); - private final SearchPhaseContext context; + private final AbstractSearchAsyncAction context; final SearchPhaseResults queryPhaseResults; final SearchPhaseResults rankPhaseResults; private final AggregatedDfs aggregatedDfs; private final SearchProgressListener progressListener; - private final Client client; + private final RankFeaturePhaseRankCoordinatorContext rankFeaturePhaseRankCoordinatorContext; RankFeaturePhase( SearchPhaseResults queryPhaseResults, AggregatedDfs aggregatedDfs, - SearchPhaseContext context, - Client client + AbstractSearchAsyncAction context, + RankFeaturePhaseRankCoordinatorContext rankFeaturePhaseRankCoordinatorContext ) { super("rank-feature"); + assert rankFeaturePhaseRankCoordinatorContext != null; + this.rankFeaturePhaseRankCoordinatorContext = rankFeaturePhaseRankCoordinatorContext; if (context.getNumShards() != queryPhaseResults.getNumShards()) { throw new IllegalStateException( "number of shards must match the length of the query results but doesn't:" @@ -64,7 +68,6 @@ public class RankFeaturePhase extends SearchPhase { this.rankPhaseResults = new ArraySearchPhaseResults<>(context.getNumShards()); context.addReleasable(rankPhaseResults); this.progressListener = context.getTask().getProgressListener(); - this.client = client; } @Override @@ -76,7 +79,7 @@ protected void doRun() throws Exception { // was set up at FetchSearchPhase. // we do the heavy lifting in this inner run method where we reduce aggs etc - innerRun(); + innerRun(rankFeaturePhaseRankCoordinatorContext); } @Override @@ -86,51 +89,39 @@ public void onFailure(Exception e) { }); } - void innerRun() throws Exception { + void innerRun(RankFeaturePhaseRankCoordinatorContext rankFeaturePhaseRankCoordinatorContext) throws Exception { // if the RankBuilder specifies a QueryPhaseCoordinatorContext, it will be called as part of the reduce call // to operate on the first `rank_window_size * num_shards` results and merge them appropriately. SearchPhaseController.ReducedQueryPhase reducedQueryPhase = queryPhaseResults.reduce(); - RankFeaturePhaseRankCoordinatorContext rankFeaturePhaseRankCoordinatorContext = coordinatorContext(context.getRequest().source()); - if (rankFeaturePhaseRankCoordinatorContext != null) { - ScoreDoc[] queryScoreDocs = reducedQueryPhase.sortedTopDocs().scoreDocs(); // rank_window_size - final List[] docIdsToLoad = SearchPhaseController.fillDocIdsToLoad(context.getNumShards(), queryScoreDocs); - final CountedCollector rankRequestCounter = new CountedCollector<>( - rankPhaseResults, - context.getNumShards(), - () -> onPhaseDone(rankFeaturePhaseRankCoordinatorContext, reducedQueryPhase), - context - ); + ScoreDoc[] queryScoreDocs = reducedQueryPhase.sortedTopDocs().scoreDocs(); // rank_window_size + final List[] docIdsToLoad = SearchPhaseController.fillDocIdsToLoad(context.getNumShards(), queryScoreDocs); + final CountedCollector rankRequestCounter = new CountedCollector<>( + rankPhaseResults, + context.getNumShards(), + () -> onPhaseDone(rankFeaturePhaseRankCoordinatorContext, reducedQueryPhase), + context + ); - // we send out a request to each shard in order to fetch the needed feature info - for (int i = 0; i < docIdsToLoad.length; i++) { - List entry = docIdsToLoad[i]; - SearchPhaseResult queryResult = queryPhaseResults.getAtomicArray().get(i); - if (entry == null || entry.isEmpty()) { - if (queryResult != null) { - releaseIrrelevantSearchContext(queryResult, context); - progressListener.notifyRankFeatureResult(i); - } - rankRequestCounter.countDown(); - } else { - executeRankFeatureShardPhase(queryResult, rankRequestCounter, entry); + // we send out a request to each shard in order to fetch the needed feature info + for (int i = 0; i < docIdsToLoad.length; i++) { + List entry = docIdsToLoad[i]; + SearchPhaseResult queryResult = queryPhaseResults.getAtomicArray().get(i); + if (entry == null || entry.isEmpty()) { + if (queryResult != null) { + releaseIrrelevantSearchContext(queryResult, context); + progressListener.notifyRankFeatureResult(i); } + rankRequestCounter.countDown(); + } else { + executeRankFeatureShardPhase(queryResult, rankRequestCounter, entry); } - } else { - moveToNextPhase(queryPhaseResults, reducedQueryPhase); } } - private RankFeaturePhaseRankCoordinatorContext coordinatorContext(SearchSourceBuilder source) { + static RankFeaturePhaseRankCoordinatorContext coordinatorContext(SearchSourceBuilder source, Client client) { return source == null || source.rankBuilder() == null ? null - : context.getRequest() - .source() - .rankBuilder() - .buildRankFeaturePhaseCoordinatorContext( - context.getRequest().source().size(), - context.getRequest().source().from(), - client - ); + : source.rankBuilder().buildRankFeaturePhaseCoordinatorContext(source.size(), source.from(), client); } private void executeRankFeatureShardPhase( @@ -141,9 +132,38 @@ private void executeRankFeatureShardPhase( final SearchShardTarget shardTarget = queryResult.queryResult().getSearchShardTarget(); final ShardSearchContextId contextId = queryResult.queryResult().getContextId(); final int shardIndex = queryResult.getShardIndex(); + var listener = new SearchActionListener(shardTarget, shardIndex) { + @Override + protected void innerOnResponse(RankFeatureResult response) { + try { + progressListener.notifyRankFeatureResult(shardIndex); + rankRequestCounter.onResult(response); + } catch (Exception e) { + context.onPhaseFailure(RankFeaturePhase.this, "", e); + } + } + + @Override + public void onFailure(Exception e) { + try { + logger.debug(() -> "[" + contextId + "] Failed to execute rank phase", e); + progressListener.notifyRankFeatureFailure(shardIndex, shardTarget, e); + rankRequestCounter.onFailure(shardIndex, shardTarget, e); + } finally { + releaseIrrelevantSearchContext(queryResult, context); + } + } + }; + final Transport.Connection connection; + try { + connection = context.getConnection(shardTarget.getClusterAlias(), shardTarget.getNodeId()); + } catch (Exception e) { + listener.onFailure(e); + return; + } context.getSearchTransport() .sendExecuteRankFeature( - context.getConnection(shardTarget.getClusterAlias(), shardTarget.getNodeId()), + connection, new RankFeatureShardRequest( context.getOriginalIndices(queryResult.getShardIndex()), queryResult.getContextId(), @@ -151,28 +171,7 @@ private void executeRankFeatureShardPhase( entry ), context.getTask(), - new SearchActionListener<>(shardTarget, shardIndex) { - @Override - protected void innerOnResponse(RankFeatureResult response) { - try { - progressListener.notifyRankFeatureResult(shardIndex); - rankRequestCounter.onResult(response); - } catch (Exception e) { - context.onPhaseFailure(RankFeaturePhase.this, "", e); - } - } - - @Override - public void onFailure(Exception e) { - try { - logger.debug(() -> "[" + contextId + "] Failed to execute rank phase", e); - progressListener.notifyRankFeatureFailure(shardIndex, shardTarget, e); - rankRequestCounter.onFailure(shardIndex, shardTarget, e); - } finally { - releaseIrrelevantSearchContext(queryResult, context); - } - } - } + listener ); } @@ -180,23 +179,25 @@ private void onPhaseDone( RankFeaturePhaseRankCoordinatorContext rankFeaturePhaseRankCoordinatorContext, SearchPhaseController.ReducedQueryPhase reducedQueryPhase ) { - assert rankFeaturePhaseRankCoordinatorContext != null; - ThreadedActionListener rankResultListener = new ThreadedActionListener<>(context, new ActionListener<>() { - @Override - public void onResponse(RankFeatureDoc[] docsWithUpdatedScores) { - RankFeatureDoc[] topResults = rankFeaturePhaseRankCoordinatorContext.rankAndPaginate(docsWithUpdatedScores); - SearchPhaseController.ReducedQueryPhase reducedRankFeaturePhase = newReducedQueryPhaseResults( - reducedQueryPhase, - topResults - ); - moveToNextPhase(rankPhaseResults, reducedRankFeaturePhase); - } + ThreadedActionListener rankResultListener = new ThreadedActionListener<>( + context::execute, + new ActionListener<>() { + @Override + public void onResponse(RankFeatureDoc[] docsWithUpdatedScores) { + RankFeatureDoc[] topResults = rankFeaturePhaseRankCoordinatorContext.rankAndPaginate(docsWithUpdatedScores); + SearchPhaseController.ReducedQueryPhase reducedRankFeaturePhase = newReducedQueryPhaseResults( + reducedQueryPhase, + topResults + ); + moveToNextPhase(rankPhaseResults, reducedRankFeaturePhase); + } - @Override - public void onFailure(Exception e) { - context.onPhaseFailure(RankFeaturePhase.this, "Computing updated ranks for results failed", e); + @Override + public void onFailure(Exception e) { + context.onPhaseFailure(RankFeaturePhase.this, "Computing updated ranks for results failed", e); + } } - }); + ); rankFeaturePhaseRankCoordinatorContext.computeRankScoresForGlobalResults( rankPhaseResults.getAtomicArray().asList().stream().map(SearchPhaseResult::rankFeatureResult).toList(), rankResultListener @@ -238,6 +239,6 @@ private float maxScore(ScoreDoc[] scoreDocs) { } void moveToNextPhase(SearchPhaseResults phaseResults, SearchPhaseController.ReducedQueryPhase reducedQueryPhase) { - context.executeNextPhase(this, new FetchSearchPhase(phaseResults, aggregatedDfs, context, reducedQueryPhase)); + context.executeNextPhase(this, () -> new FetchSearchPhase(phaseResults, aggregatedDfs, context, reducedQueryPhase)); } } diff --git a/server/src/main/java/org/elasticsearch/action/search/ReduceSearchPhaseException.java b/server/src/main/java/org/elasticsearch/action/search/ReduceSearchPhaseException.java index 94e545c472624..4768f9dfd020d 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ReduceSearchPhaseException.java +++ b/server/src/main/java/org/elasticsearch/action/search/ReduceSearchPhaseException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/RestClosePointInTimeAction.java b/server/src/main/java/org/elasticsearch/action/search/RestClosePointInTimeAction.java index 64702501581ea..0982c9190c1d1 100644 --- a/server/src/main/java/org/elasticsearch/action/search/RestClosePointInTimeAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/RestClosePointInTimeAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/RestOpenPointInTimeAction.java b/server/src/main/java/org/elasticsearch/action/search/RestOpenPointInTimeAction.java index 5966a1c924745..41c894063aaea 100644 --- a/server/src/main/java/org/elasticsearch/action/search/RestOpenPointInTimeAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/RestOpenPointInTimeAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchActionListener.java b/server/src/main/java/org/elasticsearch/action/search/SearchActionListener.java index 6b057c325c6b8..237449881fba1 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchActionListener.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchActionListener.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchContextId.java b/server/src/main/java/org/elasticsearch/action/search/SearchContextId.java index 2e4dc724413ea..c2f1510341fb0 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchContextId.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchContextId.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; @@ -62,14 +63,14 @@ public static BytesReference encode( TransportVersion version, ShardSearchFailure[] shardFailures ) { - assert shardFailures.length == 0 || version.onOrAfter(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT) + assert shardFailures.length == 0 || version.onOrAfter(TransportVersions.V_8_16_0) : "[allow_partial_search_results] cannot be enabled on a cluster that has not been fully upgraded to version [" - + TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT + + TransportVersions.V_8_16_0.toReleaseVersion() + "] or higher."; try (var out = new BytesStreamOutput()) { out.setTransportVersion(version); TransportVersion.writeVersion(version, out); - boolean allowNullContextId = out.getTransportVersion().onOrAfter(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT); + boolean allowNullContextId = out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0); int shardSize = searchPhaseResults.size() + (allowNullContextId ? shardFailures.length : 0); out.writeVInt(shardSize); for (var searchResult : searchPhaseResults) { diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchContextIdForNode.java b/server/src/main/java/org/elasticsearch/action/search/SearchContextIdForNode.java index a70ddf6ee14b9..f91a9d09f4bb4 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchContextIdForNode.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchContextIdForNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; @@ -36,7 +37,7 @@ public final class SearchContextIdForNode implements Writeable { } SearchContextIdForNode(StreamInput in) throws IOException { - boolean allowNull = in.getTransportVersion().onOrAfter(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT); + boolean allowNull = in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0); this.node = allowNull ? in.readOptionalString() : in.readString(); this.clusterAlias = in.readOptionalString(); this.searchContextId = allowNull ? in.readOptionalWriteable(ShardSearchContextId::new) : new ShardSearchContextId(in); @@ -44,7 +45,7 @@ public final class SearchContextIdForNode implements Writeable { @Override public void writeTo(StreamOutput out) throws IOException { - boolean allowNull = out.getTransportVersion().onOrAfter(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT); + boolean allowNull = out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0); if (allowNull) { out.writeOptionalString(node); } else { @@ -52,7 +53,7 @@ public void writeTo(StreamOutput out) throws IOException { // We should never set a null node if the cluster is not fully upgraded to a version that can handle it. throw new IOException( "Cannot write null node value to a node in version " - + out.getTransportVersion() + + out.getTransportVersion().toReleaseVersion() + ". The target node must be specified to retrieve the ShardSearchContextId." ); } @@ -66,7 +67,7 @@ public void writeTo(StreamOutput out) throws IOException { // We should never set a null search context id if the cluster is not fully upgraded to a version that can handle it. throw new IOException( "Cannot write null search context ID to a node in version " - + out.getTransportVersion() + + out.getTransportVersion().toReleaseVersion() + ". A valid search context ID is required to identify the shard's search context in this version." ); } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java index 85c99fc9032d4..25d59a06664da 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; @@ -83,19 +84,14 @@ final class SearchDfsQueryThenFetchAsyncAction extends AbstractSearchAsyncAction @Override protected void executePhaseOnShard( final SearchShardIterator shardIt, - final SearchShardTarget shard, + final Transport.Connection connection, final SearchActionListener listener ) { - getSearchTransport().sendExecuteDfs( - getConnection(shard.getClusterAlias(), shard.getNodeId()), - buildShardSearchRequest(shardIt, listener.requestIndex), - getTask(), - listener - ); + getSearchTransport().sendExecuteDfs(connection, buildShardSearchRequest(shardIt, listener.requestIndex), getTask(), listener); } @Override - protected SearchPhase getNextPhase(final SearchPhaseResults results, SearchPhaseContext context) { + protected SearchPhase getNextPhase() { final List dfsSearchResults = results.getAtomicArray().asList(); final AggregatedDfs aggregatedDfs = SearchPhaseController.aggregateDfs(dfsSearchResults); final List mergedKnnResults = SearchPhaseController.mergeKnnResults(getRequest(), dfsSearchResults); @@ -104,8 +100,8 @@ protected SearchPhase getNextPhase(final SearchPhaseResults res aggregatedDfs, mergedKnnResults, queryPhaseResultConsumer, - (queryResults) -> new RankFeaturePhase(queryResults, aggregatedDfs, context, client), - context + (queryResults) -> SearchQueryThenFetchAsyncAction.nextPhase(client, this, queryResults, aggregatedDfs), + this ); } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchExecutionStatsCollector.java b/server/src/main/java/org/elasticsearch/action/search/SearchExecutionStatsCollector.java index 4b5fbe22b28e5..d0a1fdb288b03 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchExecutionStatsCollector.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchExecutionStatsCollector.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhase.java index da8479873a4b6..cc92eaca21ca7 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhase.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; @@ -73,7 +74,7 @@ protected void doCheckNoMissingShards(String phaseName, SearchRequest request, G /** * Releases shard targets that are not used in the docsIdsToLoad. */ - protected void releaseIrrelevantSearchContext(SearchPhaseResult searchPhaseResult, SearchPhaseContext context) { + protected static void releaseIrrelevantSearchContext(SearchPhaseResult searchPhaseResult, AbstractSearchAsyncAction context) { // we only release search context that we did not fetch from, if we are not scrolling // or using a PIT and if it has at least one hit that didn't make it to the global topDocs if (searchPhaseResult == null) { @@ -91,11 +92,7 @@ protected void releaseIrrelevantSearchContext(SearchPhaseResult searchPhaseResul context.getLogger().trace("trying to release search context [{}]", phaseResult.getContextId()); SearchShardTarget shardTarget = phaseResult.getSearchShardTarget(); Transport.Connection connection = context.getConnection(shardTarget.getClusterAlias(), shardTarget.getNodeId()); - context.sendReleaseSearchContext( - phaseResult.getContextId(), - connection, - context.getOriginalIndices(phaseResult.getShardIndex()) - ); + context.sendReleaseSearchContext(phaseResult.getContextId(), connection); } catch (Exception e) { context.getLogger().trace("failed to release context", e); } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseContext.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseContext.java deleted file mode 100644 index c77c7e58efc7d..0000000000000 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseContext.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -package org.elasticsearch.action.search; - -import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.OriginalIndices; -import org.elasticsearch.common.util.concurrent.AtomicArray; -import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.Releasable; -import org.elasticsearch.search.SearchPhaseResult; -import org.elasticsearch.search.SearchShardTarget; -import org.elasticsearch.search.internal.ShardSearchContextId; -import org.elasticsearch.transport.Transport; - -import java.util.concurrent.Executor; - -/** - * This class provide contextual state and access to resources across multiple search phases. - */ -interface SearchPhaseContext extends Executor { - // TODO maybe we can make this concrete later - for now we just implement this in the base class for all initial phases - - /** - * Returns the total number of shards to the current search across all indices - */ - int getNumShards(); - - /** - * Returns a logger for this context to prevent each individual phase to create their own logger. - */ - Logger getLogger(); - - /** - * Returns the currently executing search task - */ - SearchTask getTask(); - - /** - * Returns the currently executing search request - */ - SearchRequest getRequest(); - - /** - * Returns the targeted {@link OriginalIndices} for the provided {@code shardIndex}. - */ - OriginalIndices getOriginalIndices(int shardIndex); - - /** - * Checks if the given context id is part of the point in time of this search (if exists). - * We should not release search contexts that belong to the point in time during or after searches. - */ - boolean isPartOfPointInTime(ShardSearchContextId contextId); - - /** - * Builds and sends the final search response back to the user. - * - * @param internalSearchResponse the internal search response - * @param queryResults the results of the query phase - */ - void sendSearchResponse(SearchResponseSections internalSearchResponse, AtomicArray queryResults); - - /** - * Notifies the top-level listener of the provided exception - */ - void onFailure(Exception e); - - /** - * This method will communicate a fatal phase failure back to the user. In contrast to a shard failure - * will this method immediately fail the search request and return the failure to the issuer of the request - * @param phase the phase that failed - * @param msg an optional message - * @param cause the cause of the phase failure - */ - void onPhaseFailure(SearchPhase phase, String msg, Throwable cause); - - /** - * This method will record a shard failure for the given shard index. In contrast to a phase failure - * ({@link #onPhaseFailure(SearchPhase, String, Throwable)}) this method will immediately return to the user but will record - * a shard failure for the given shard index. This should be called if a shard failure happens after we successfully retrieved - * a result from that shard in a previous phase. - */ - void onShardFailure(int shardIndex, @Nullable SearchShardTarget shardTarget, Exception e); - - /** - * Returns a connection to the node if connected otherwise and {@link org.elasticsearch.transport.ConnectTransportException} will be - * thrown. - */ - Transport.Connection getConnection(String clusterAlias, String nodeId); - - /** - * Returns the {@link SearchTransportService} to send shard request to other nodes - */ - SearchTransportService getSearchTransport(); - - /** - * Releases a search context with the given context ID on the node the given connection is connected to. - * @see org.elasticsearch.search.query.QuerySearchResult#getContextId() - * @see org.elasticsearch.search.fetch.FetchSearchResult#getContextId() - * - */ - default void sendReleaseSearchContext( - ShardSearchContextId contextId, - Transport.Connection connection, - OriginalIndices originalIndices - ) { - assert isPartOfPointInTime(contextId) == false : "Must not release point in time context [" + contextId + "]"; - if (connection != null) { - getSearchTransport().sendFreeContext(connection, contextId, originalIndices); - } - } - - /** - * Processes the phase transition from on phase to another. This method handles all errors that happen during the initial run execution - * of the next phase. If there are no successful operations in the context when this method is executed the search is aborted and - * a response is returned to the user indicating that all shards have failed. - */ - void executeNextPhase(SearchPhase currentPhase, SearchPhase nextPhase); - - /** - * Registers a {@link Releasable} that will be closed when the search request finishes or fails. - */ - void addReleasable(Releasable releasable); -} diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index 82c498c64e1c9..74786dff1648d 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; @@ -35,6 +36,7 @@ import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchService; +import org.elasticsearch.search.SearchSortValues; import org.elasticsearch.search.aggregations.AggregationReduceContext; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.InternalAggregations; @@ -50,6 +52,7 @@ import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.rank.RankDoc; import org.elasticsearch.search.rank.context.QueryPhaseRankCoordinatorContext; +import org.elasticsearch.search.sort.ShardDocSortField; import org.elasticsearch.search.suggest.Suggest; import org.elasticsearch.search.suggest.Suggest.Suggestion; import org.elasticsearch.search.suggest.completion.CompletionSuggestion; @@ -459,10 +462,16 @@ private static SearchHits getHits( : "not enough hits fetched. index [" + index + "] length: " + fetchResult.hits().getHits().length; SearchHit searchHit = fetchResult.hits().getHits()[index]; searchHit.shard(fetchResult.getSearchShardTarget()); - if (reducedQueryPhase.queryPhaseRankCoordinatorContext != null) { - assert shardDoc instanceof RankDoc; + if (shardDoc instanceof RankDoc) { searchHit.setRank(((RankDoc) shardDoc).rank); searchHit.score(shardDoc.score); + long shardAndDoc = ShardDocSortField.encodeShardAndDoc(shardDoc.shardIndex, shardDoc.doc); + searchHit.sortValues( + new SearchSortValues( + new Object[] { shardDoc.score, shardAndDoc }, + new DocValueFormat[] { DocValueFormat.RAW, DocValueFormat.RAW } + ) + ); } else if (sortedTopDocs.isSortedByField) { FieldDoc fieldDoc = (FieldDoc) shardDoc; searchHit.sortValues(fieldDoc.fields, reducedQueryPhase.sortValueFormats); @@ -725,6 +734,12 @@ static int getTopDocsSize(SearchRequest request) { return DEFAULT_SIZE; } SearchSourceBuilder source = request.source(); + if (source.rankBuilder() != null) { + // if we have a RankBuilder defined, it needs to have access to all the documents in order to rerank them + // so we override size here and keep all `rank_window_size` docs. + // Pagination is taking place later through RankFeaturePhaseRankCoordinatorContext#rankAndPaginate + return source.rankBuilder().rankWindowSize(); + } return (source.size() == -1 ? DEFAULT_SIZE : source.size()) + (source.from() == -1 ? SearchService.DEFAULT_FROM : source.from()); } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java index 022c3304bc865..fc79e85f9326d 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; @@ -72,15 +73,21 @@ public RestStatus status() { // on coordinator node. so get the status from cause instead of returning SERVICE_UNAVAILABLE blindly return getCause() == null ? RestStatus.SERVICE_UNAVAILABLE : ExceptionsHelper.status(getCause()); } - RestStatus status = shardFailures[0].status(); - if (shardFailures.length > 1) { - for (int i = 1; i < shardFailures.length; i++) { - if (shardFailures[i].status().getStatus() >= RestStatus.INTERNAL_SERVER_ERROR.getStatus()) { - status = shardFailures[i].status(); - } + RestStatus status = null; + for (ShardSearchFailure shardFailure : shardFailures) { + RestStatus shardStatus = shardFailure.status(); + int statusCode = shardStatus.getStatus(); + + // Return if it's an error that can be retried. + // These currently take precedence over other status code(s). + if (statusCode >= 502 && statusCode <= 504) { + return shardStatus; + } else if (statusCode >= 500) { + status = shardStatus; } } - return status; + + return status == null ? shardFailures[0].status() : status; } public ShardSearchFailure[] shardFailures() { diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseResults.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseResults.java index 28606ecc09f90..73bb0d545a2e0 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseResults.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseResults.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchProgressActionListener.java b/server/src/main/java/org/elasticsearch/action/search/SearchProgressActionListener.java index 95e9b4cedeba5..2c09ec55c505a 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchProgressActionListener.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchProgressActionListener.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java b/server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java index 3b5e03cb5ac4a..a7f92700435a4 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java index 9537140657c95..f75b84abc2f0f 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; @@ -17,6 +18,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchShardTarget; +import org.elasticsearch.search.dfs.AggregatedDfs; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.internal.ShardSearchRequest; @@ -89,11 +91,11 @@ class SearchQueryThenFetchAsyncAction extends AbstractSearchAsyncAction listener ) { ShardSearchRequest request = rewriteShardSearchRequest(super.buildShardSearchRequest(shardIt, listener.requestIndex)); - getSearchTransport().sendExecuteQuery(getConnection(shard.getClusterAlias(), shard.getNodeId()), request, getTask(), listener); + getSearchTransport().sendExecuteQuery(connection, request, getTask(), listener); } @Override @@ -124,9 +126,22 @@ && getRequest().scroll() == null super.onShardResult(result, shardIt); } + static SearchPhase nextPhase( + Client client, + AbstractSearchAsyncAction context, + SearchPhaseResults queryResults, + AggregatedDfs aggregatedDfs + ) { + var rankFeaturePhaseCoordCtx = RankFeaturePhase.coordinatorContext(context.getRequest().source(), client); + if (rankFeaturePhaseCoordCtx == null) { + return new FetchSearchPhase(queryResults, aggregatedDfs, context, null); + } + return new RankFeaturePhase(queryResults, aggregatedDfs, context, rankFeaturePhaseCoordCtx); + } + @Override - protected SearchPhase getNextPhase(final SearchPhaseResults results, SearchPhaseContext context) { - return new RankFeaturePhase(results, null, this, client); + protected SearchPhase getNextPhase() { + return nextPhase(client, this, results, null); } private ShardSearchRequest rewriteShardSearchRequest(ShardSearchRequest request) { diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index 415a2ba7da9f8..9961c3770fa86 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java index 5be696da5c6b1..afbfe129c302e 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java index 8d70e2dd6bb66..83ee6c216ad49 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; @@ -13,13 +14,12 @@ import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.common.xcontent.ChunkedToXContentObject; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.RefCounted; @@ -382,39 +382,17 @@ public Clusters getClusters() { @Override public Iterator toXContentChunked(ToXContent.Params params) { assert hasReferences(); - return Iterators.concat( - ChunkedToXContentHelper.startObject(), - this.innerToXContentChunked(params), - ChunkedToXContentHelper.endObject() - ); + return ChunkedToXContent.builder(params).xContentObject(innerToXContentChunked(params)); } public Iterator innerToXContentChunked(ToXContent.Params params) { - return Iterators.concat( - ChunkedToXContentHelper.singleChunk(SearchResponse.this::headerToXContent), - Iterators.single(clusters), - Iterators.concat( - Iterators.flatMap(Iterators.single(hits), r -> r.toXContentChunked(params)), - Iterators.single((ToXContent) (b, p) -> { - if (aggregations != null) { - aggregations.toXContent(b, p); - } - return b; - }), - Iterators.single((b, p) -> { - if (suggest != null) { - suggest.toXContent(b, p); - } - return b; - }), - Iterators.single((b, p) -> { - if (profileResults != null) { - profileResults.toXContent(b, p); - } - return b; - }) - ) - ); + return ChunkedToXContent.builder(params) + .append(SearchResponse.this::headerToXContent) + .append(clusters) + .append(hits) + .appendIfPresent(aggregations) + .appendIfPresent(suggest) + .appendIfPresent(profileResults); } public XContentBuilder headerToXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponseMerger.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponseMerger.java index d393adc4e26d1..bbb4ce45c47dc 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchResponseMerger.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponseMerger.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java index d7f7849578996..9d85348b80d62 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java @@ -1,21 +1,20 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; -import org.elasticsearch.core.RefCounted; -import org.elasticsearch.core.SimpleRefCounted; +import org.elasticsearch.core.Releasable; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.profile.SearchProfileResults; import org.elasticsearch.search.profile.SearchProfileShardResult; import org.elasticsearch.search.suggest.Suggest; -import org.elasticsearch.transport.LeakTracker; import java.util.Collections; import java.util.Map; @@ -24,7 +23,7 @@ * Holds some sections that a search response is composed of (hits, aggs, suggestions etc.) during some steps of the search response * building. */ -public class SearchResponseSections implements RefCounted { +public class SearchResponseSections implements Releasable { public static final SearchResponseSections EMPTY_WITH_TOTAL_HITS = new SearchResponseSections( SearchHits.EMPTY_WITH_TOTAL_HITS, @@ -52,8 +51,6 @@ public class SearchResponseSections implements RefCounted { protected final Boolean terminatedEarly; protected final int numReducePhases; - private final RefCounted refCounted; - public SearchResponseSections( SearchHits hits, InternalAggregations aggregations, @@ -71,7 +68,6 @@ public SearchResponseSections( this.timedOut = timedOut; this.terminatedEarly = terminatedEarly; this.numReducePhases = numReducePhases; - refCounted = hits.getHits().length > 0 ? LeakTracker.wrap(new SimpleRefCounted()) : ALWAYS_REFERENCED; } public final SearchHits hits() { @@ -96,26 +92,7 @@ public final Map profile() { } @Override - public void incRef() { - refCounted.incRef(); - } - - @Override - public boolean tryIncRef() { - return refCounted.tryIncRef(); - } - - @Override - public boolean decRef() { - if (refCounted.decRef()) { - hits.decRef(); - return true; - } - return false; - } - - @Override - public boolean hasReferences() { - return refCounted.hasReferences(); + public void close() { + hits.decRef(); } } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java index 92c26d2db7a33..2231f791384fa 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchScrollAsyncAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; @@ -245,8 +246,7 @@ protected final void sendResponse( if (request.scroll() != null) { scrollId = request.scrollId(); } - var sections = SearchPhaseController.merge(true, queryPhase, fetchResults); - try { + try (var sections = SearchPhaseController.merge(true, queryPhase, fetchResults)) { ActionListener.respondAndRelease( listener, new SearchResponse( @@ -261,8 +261,6 @@ protected final void sendResponse( null ) ); - } finally { - sections.decRef(); } } catch (Exception e) { listener.onFailure(new ReduceSearchPhaseException("fetch", "inner finish failed", e, buildShardFailures())); diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryAndFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryAndFetchAsyncAction.java index c80c62f3387ea..7b5ba21c80220 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryAndFetchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryAndFetchAsyncAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java index 56472e877ed48..8c33e3ca7da4b 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java index c49518b0523eb..71b88b03a5463 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchScrollRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchScrollRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/search/SearchScrollRequestBuilder.java index f75dc52b5cd7c..24dac98166ce0 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchScrollRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchScrollRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchShard.java b/server/src/main/java/org/elasticsearch/action/search/SearchShard.java index e9dc628410f95..1704f295915e5 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchShard.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchShard.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java b/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java index 2b1bbffa7e36f..b543ff922886f 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchShardTask.java b/server/src/main/java/org/elasticsearch/action/search/SearchShardTask.java index 0c9ead5c32fb6..0e1e8305f111b 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchShardTask.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchShardTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchShardsGroup.java b/server/src/main/java/org/elasticsearch/action/search/SearchShardsGroup.java index 19ba56be5749a..bdaf7e29ae0bc 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchShardsGroup.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchShardsGroup.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchShardsRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchShardsRequest.java index 4e545dcf58392..be1b194803709 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchShardsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchShardsRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchShardsResponse.java b/server/src/main/java/org/elasticsearch/action/search/SearchShardsResponse.java index 71ff3e74fe027..ed1a1d5ae6cc4 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchShardsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchShardsResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTask.java b/server/src/main/java/org/elasticsearch/action/search/SearchTask.java index cc5d60ad0b0c0..fe91f04b4a25b 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchTask.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportAPMMetrics.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportAPMMetrics.java deleted file mode 100644 index 9f8896f169350..0000000000000 --- a/server/src/main/java/org/elasticsearch/action/search/SearchTransportAPMMetrics.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.action.search; - -import org.elasticsearch.telemetry.metric.LongHistogram; -import org.elasticsearch.telemetry.metric.MeterRegistry; - -public class SearchTransportAPMMetrics { - public static final String SEARCH_ACTION_LATENCY_BASE_METRIC = "es.search.nodes.transport_actions.latency.histogram"; - public static final String ACTION_ATTRIBUTE_NAME = "action"; - - public static final String QUERY_CAN_MATCH_NODE_METRIC = "shards_can_match"; - public static final String DFS_ACTION_METRIC = "dfs_query_then_fetch/shard_dfs_phase"; - public static final String QUERY_ID_ACTION_METRIC = "dfs_query_then_fetch/shard_query_phase"; - public static final String QUERY_ACTION_METRIC = "query_then_fetch/shard_query_phase"; - public static final String RANK_SHARD_FEATURE_ACTION_METRIC = "rank/shard_feature_phase"; - public static final String FREE_CONTEXT_ACTION_METRIC = "shard_release_context"; - public static final String FETCH_ID_ACTION_METRIC = "shard_fetch_phase"; - public static final String QUERY_SCROLL_ACTION_METRIC = "scroll/shard_query_phase"; - public static final String FETCH_ID_SCROLL_ACTION_METRIC = "scroll/shard_fetch_phase"; - public static final String QUERY_FETCH_SCROLL_ACTION_METRIC = "scroll/shard_query_and_fetch_phase"; - public static final String FREE_CONTEXT_SCROLL_ACTION_METRIC = "scroll/shard_release_context"; - public static final String CLEAR_SCROLL_CONTEXTS_ACTION_METRIC = "scroll/shard_release_contexts"; - - private final LongHistogram actionLatencies; - - public SearchTransportAPMMetrics(MeterRegistry meterRegistry) { - this( - meterRegistry.registerLongHistogram( - SEARCH_ACTION_LATENCY_BASE_METRIC, - "Transport action execution times at the node level, expressed as a histogram", - "millis" - ) - ); - } - - private SearchTransportAPMMetrics(LongHistogram actionLatencies) { - this.actionLatencies = actionLatencies; - } - - public LongHistogram getActionLatencies() { - return actionLatencies; - } -} diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java index 52d4542faaf77..2041754bc2bcc 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; @@ -12,12 +13,10 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListenerResponseHandler; -import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.get.TransportGetTaskAction; import org.elasticsearch.action.support.ChannelActionListener; -import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -66,20 +65,6 @@ import java.util.concurrent.Executor; import java.util.function.BiFunction; -import static org.elasticsearch.action.search.SearchTransportAPMMetrics.ACTION_ATTRIBUTE_NAME; -import static org.elasticsearch.action.search.SearchTransportAPMMetrics.CLEAR_SCROLL_CONTEXTS_ACTION_METRIC; -import static org.elasticsearch.action.search.SearchTransportAPMMetrics.DFS_ACTION_METRIC; -import static org.elasticsearch.action.search.SearchTransportAPMMetrics.FETCH_ID_ACTION_METRIC; -import static org.elasticsearch.action.search.SearchTransportAPMMetrics.FETCH_ID_SCROLL_ACTION_METRIC; -import static org.elasticsearch.action.search.SearchTransportAPMMetrics.FREE_CONTEXT_ACTION_METRIC; -import static org.elasticsearch.action.search.SearchTransportAPMMetrics.FREE_CONTEXT_SCROLL_ACTION_METRIC; -import static org.elasticsearch.action.search.SearchTransportAPMMetrics.QUERY_ACTION_METRIC; -import static org.elasticsearch.action.search.SearchTransportAPMMetrics.QUERY_CAN_MATCH_NODE_METRIC; -import static org.elasticsearch.action.search.SearchTransportAPMMetrics.QUERY_FETCH_SCROLL_ACTION_METRIC; -import static org.elasticsearch.action.search.SearchTransportAPMMetrics.QUERY_ID_ACTION_METRIC; -import static org.elasticsearch.action.search.SearchTransportAPMMetrics.QUERY_SCROLL_ACTION_METRIC; -import static org.elasticsearch.action.search.SearchTransportAPMMetrics.RANK_SHARD_FEATURE_ACTION_METRIC; - /** * An encapsulation of {@link org.elasticsearch.search.SearchService} operations exposed through * transport. @@ -137,24 +122,6 @@ public SearchTransportService( this.responseWrapper = responseWrapper; } - private static final ActionListenerResponseHandler SEND_FREE_CONTEXT_LISTENER = - new ActionListenerResponseHandler<>( - ActionListener.noop(), - SearchFreeContextResponse::readFrom, - TransportResponseHandler.TRANSPORT_WORKER - ); - - public void sendFreeContext(Transport.Connection connection, final ShardSearchContextId contextId, OriginalIndices originalIndices) { - transportService.sendRequest( - connection, - FREE_CONTEXT_ACTION_NAME, - new SearchFreeContextRequest(originalIndices, contextId), - TransportRequestOptions.EMPTY, - // no need to respond if it was freed or not - SEND_FREE_CONTEXT_LISTENER - ); - } - public void sendFreeContext( Transport.Connection connection, ShardSearchContextId contextId, @@ -383,43 +350,6 @@ private static class ClearScrollContextsRequest extends TransportRequest { } } - static class SearchFreeContextRequest extends ScrollFreeContextRequest implements IndicesRequest { - private final OriginalIndices originalIndices; - - SearchFreeContextRequest(OriginalIndices originalIndices, ShardSearchContextId id) { - super(id); - this.originalIndices = originalIndices; - } - - SearchFreeContextRequest(StreamInput in) throws IOException { - super(in); - originalIndices = OriginalIndices.readOriginalIndices(in); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - OriginalIndices.writeOriginalIndices(originalIndices, out); - } - - @Override - public String[] indices() { - if (originalIndices == null) { - return null; - } - return originalIndices.indices(); - } - - @Override - public IndicesOptions indicesOptions() { - if (originalIndices == null) { - return null; - } - return originalIndices.indicesOptions(); - } - - } - public static class SearchFreeContextResponse extends TransportResponse { private static final SearchFreeContextResponse FREED = new SearchFreeContextResponse(true); @@ -449,11 +379,7 @@ public void writeTo(StreamOutput out) throws IOException { } } - public static void registerRequestHandler( - TransportService transportService, - SearchService searchService, - SearchTransportAPMMetrics searchTransportMetrics - ) { + public static void registerRequestHandler(TransportService transportService, SearchService searchService) { final TransportRequestHandler freeContextHandler = (request, channel, task) -> { logger.trace("releasing search context [{}]", request.id()); boolean freed = searchService.freeReaderContext(request.id()); @@ -464,7 +390,7 @@ public static void registerRequestHandler( FREE_CONTEXT_SCROLL_ACTION_NAME, freeContextExecutor, ScrollFreeContextRequest::new, - instrumentedHandler(FREE_CONTEXT_SCROLL_ACTION_METRIC, transportService, searchTransportMetrics, freeContextHandler) + freeContextHandler ); TransportActionProxy.registerProxyAction( transportService, @@ -473,22 +399,23 @@ public static void registerRequestHandler( SearchFreeContextResponse::readFrom ); - transportService.registerRequestHandler( - FREE_CONTEXT_ACTION_NAME, - freeContextExecutor, - SearchFreeContextRequest::new, - instrumentedHandler(FREE_CONTEXT_ACTION_METRIC, transportService, searchTransportMetrics, freeContextHandler) - ); + // TODO: remove this handler once the lowest compatible version stops using it + transportService.registerRequestHandler(FREE_CONTEXT_ACTION_NAME, freeContextExecutor, in -> { + var res = new ScrollFreeContextRequest(in); + // this handler exists for BwC purposes only, we don't need the original indices to free the context + OriginalIndices.readOriginalIndices(in); + return res; + }, freeContextHandler); TransportActionProxy.registerProxyAction(transportService, FREE_CONTEXT_ACTION_NAME, false, SearchFreeContextResponse::readFrom); transportService.registerRequestHandler( CLEAR_SCROLL_CONTEXTS_ACTION_NAME, freeContextExecutor, ClearScrollContextsRequest::new, - instrumentedHandler(CLEAR_SCROLL_CONTEXTS_ACTION_METRIC, transportService, searchTransportMetrics, (request, channel, task) -> { + (request, channel, task) -> { searchService.freeAllScrollContexts(); channel.sendResponse(TransportResponse.Empty.INSTANCE); - }) + } ); TransportActionProxy.registerProxyAction( transportService, @@ -501,16 +428,7 @@ public static void registerRequestHandler( DFS_ACTION_NAME, EsExecutors.DIRECT_EXECUTOR_SERVICE, ShardSearchRequest::new, - instrumentedHandler( - DFS_ACTION_METRIC, - transportService, - searchTransportMetrics, - (request, channel, task) -> searchService.executeDfsPhase( - request, - (SearchShardTask) task, - new ChannelActionListener<>(channel) - ) - ) + (request, channel, task) -> searchService.executeDfsPhase(request, (SearchShardTask) task, new ChannelActionListener<>(channel)) ); TransportActionProxy.registerProxyAction(transportService, DFS_ACTION_NAME, true, DfsSearchResult::new); @@ -518,15 +436,10 @@ public static void registerRequestHandler( QUERY_ACTION_NAME, EsExecutors.DIRECT_EXECUTOR_SERVICE, ShardSearchRequest::new, - instrumentedHandler( - QUERY_ACTION_METRIC, - transportService, - searchTransportMetrics, - (request, channel, task) -> searchService.executeQueryPhase( - request, - (SearchShardTask) task, - new ChannelActionListener<>(channel) - ) + (request, channel, task) -> searchService.executeQueryPhase( + request, + (SearchShardTask) task, + new ChannelActionListener<>(channel) ) ); TransportActionProxy.registerProxyActionWithDynamicResponseType( @@ -540,15 +453,11 @@ public static void registerRequestHandler( QUERY_ID_ACTION_NAME, EsExecutors.DIRECT_EXECUTOR_SERVICE, QuerySearchRequest::new, - instrumentedHandler( - QUERY_ID_ACTION_METRIC, - transportService, - searchTransportMetrics, - (request, channel, task) -> searchService.executeQueryPhase( - request, - (SearchShardTask) task, - new ChannelActionListener<>(channel) - ) + (request, channel, task) -> searchService.executeQueryPhase( + request, + (SearchShardTask) task, + new ChannelActionListener<>(channel), + channel.getVersion() ) ); TransportActionProxy.registerProxyAction(transportService, QUERY_ID_ACTION_NAME, true, QuerySearchResult::new); @@ -557,15 +466,11 @@ public static void registerRequestHandler( QUERY_SCROLL_ACTION_NAME, EsExecutors.DIRECT_EXECUTOR_SERVICE, InternalScrollSearchRequest::new, - instrumentedHandler( - QUERY_SCROLL_ACTION_METRIC, - transportService, - searchTransportMetrics, - (request, channel, task) -> searchService.executeQueryPhase( - request, - (SearchShardTask) task, - new ChannelActionListener<>(channel) - ) + (request, channel, task) -> searchService.executeQueryPhase( + request, + (SearchShardTask) task, + new ChannelActionListener<>(channel), + channel.getVersion() ) ); TransportActionProxy.registerProxyAction(transportService, QUERY_SCROLL_ACTION_NAME, true, ScrollQuerySearchResult::new); @@ -574,15 +479,10 @@ public static void registerRequestHandler( QUERY_FETCH_SCROLL_ACTION_NAME, EsExecutors.DIRECT_EXECUTOR_SERVICE, InternalScrollSearchRequest::new, - instrumentedHandler( - QUERY_FETCH_SCROLL_ACTION_METRIC, - transportService, - searchTransportMetrics, - (request, channel, task) -> searchService.executeFetchPhase( - request, - (SearchShardTask) task, - new ChannelActionListener<>(channel) - ) + (request, channel, task) -> searchService.executeFetchPhase( + request, + (SearchShardTask) task, + new ChannelActionListener<>(channel) ) ); TransportActionProxy.registerProxyAction(transportService, QUERY_FETCH_SCROLL_ACTION_NAME, true, ScrollQueryFetchSearchResult::new); @@ -593,7 +493,7 @@ public static void registerRequestHandler( RANK_FEATURE_SHARD_ACTION_NAME, EsExecutors.DIRECT_EXECUTOR_SERVICE, RankFeatureShardRequest::new, - instrumentedHandler(RANK_SHARD_FEATURE_ACTION_METRIC, transportService, searchTransportMetrics, rankShardFeatureRequest) + rankShardFeatureRequest ); TransportActionProxy.registerProxyAction(transportService, RANK_FEATURE_SHARD_ACTION_NAME, true, RankFeatureResult::new); @@ -603,7 +503,7 @@ public static void registerRequestHandler( FETCH_ID_SCROLL_ACTION_NAME, EsExecutors.DIRECT_EXECUTOR_SERVICE, ShardFetchRequest::new, - instrumentedHandler(FETCH_ID_SCROLL_ACTION_METRIC, transportService, searchTransportMetrics, shardFetchRequestHandler) + shardFetchRequestHandler ); TransportActionProxy.registerProxyAction(transportService, FETCH_ID_SCROLL_ACTION_NAME, true, FetchSearchResult::new); @@ -613,7 +513,7 @@ public static void registerRequestHandler( true, true, ShardFetchSearchRequest::new, - instrumentedHandler(FETCH_ID_ACTION_METRIC, transportService, searchTransportMetrics, shardFetchRequestHandler) + shardFetchRequestHandler ); TransportActionProxy.registerProxyAction(transportService, FETCH_ID_ACTION_NAME, true, FetchSearchResult::new); @@ -621,12 +521,7 @@ public static void registerRequestHandler( QUERY_CAN_MATCH_NODE_NAME, transportService.getThreadPool().executor(ThreadPool.Names.SEARCH_COORDINATION), CanMatchNodeRequest::new, - instrumentedHandler( - QUERY_CAN_MATCH_NODE_METRIC, - transportService, - searchTransportMetrics, - (request, channel, task) -> searchService.canMatch(request, new ChannelActionListener<>(channel)) - ) + (request, channel, task) -> searchService.canMatch(request, new ChannelActionListener<>(channel)) ); TransportActionProxy.registerProxyAction(transportService, QUERY_CAN_MATCH_NODE_NAME, true, CanMatchNodeResponse::new); } @@ -657,26 +552,6 @@ public void onFailure(Exception e) { }); } - private static TransportRequestHandler instrumentedHandler( - String actionQualifier, - TransportService transportService, - SearchTransportAPMMetrics searchTransportMetrics, - TransportRequestHandler transportRequestHandler - ) { - var threadPool = transportService.getThreadPool(); - var latencies = searchTransportMetrics.getActionLatencies(); - Map attributes = Map.of(ACTION_ATTRIBUTE_NAME, actionQualifier); - return (request, channel, task) -> { - var startTime = threadPool.relativeTimeInMillis(); - try { - transportRequestHandler.messageReceived(request, channel, task); - } finally { - var elapsedTime = threadPool.relativeTimeInMillis() - startTime; - latencies.record(elapsedTime, attributes); - } - }; - } - /** * Returns a connection to the given node on the provided cluster. If the cluster alias is null the node will be resolved * against the local cluster. diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchType.java b/server/src/main/java/org/elasticsearch/action/search/SearchType.java index 8e6511db62136..8aaa80a2417d0 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchType.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchType.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java b/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java index 5fad89a241189..5b2b2dccd6d10 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java +++ b/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; @@ -140,12 +141,9 @@ public static ShardSearchFailure fromXContent(XContentParser parser) throws IOEx if (SHARD_FIELD.equals(currentFieldName)) { shardId = parser.intValue(); } else if (INDEX_FIELD.equals(currentFieldName)) { - indexName = parser.text(); - int indexOf = indexName.indexOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR); - if (indexOf > 0) { - clusterAlias = indexName.substring(0, indexOf); - indexName = indexName.substring(indexOf + 1); - } + String[] split = RemoteClusterAware.splitIndexName(parser.text()); + clusterAlias = split[0]; + indexName = split[1]; } else if (NODE_FIELD.equals(currentFieldName)) { nodeId = parser.text(); } else { diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java index cc6223453195f..e09eb41695b39 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportClosePointInTimeAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportClosePointInTimeAction.java index 7fd81a7b8ea29..8fc954ca81ebf 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportClosePointInTimeAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportClosePointInTimeAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java index 9d2f97e4f051c..7f10ddc7998ce 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java index 717b1805547be..75dbfaf1f9623 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; @@ -27,7 +28,6 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.shard.ShardId; @@ -35,7 +35,6 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchService; -import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.internal.ShardSearchContextId; @@ -105,8 +104,7 @@ public TransportOpenPointInTimeAction( protected void doExecute(Task task, OpenPointInTimeRequest request, ActionListener listener) { final ClusterState clusterState = clusterService.state(); // Check if all the nodes in this cluster know about the service - if (request.allowPartialSearchResults() - && clusterState.getMinTransportVersion().before(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT)) { + if (request.allowPartialSearchResults() && clusterState.getMinTransportVersion().before(TransportVersions.V_8_16_0)) { listener.onFailure( new ElasticsearchStatusException( format( @@ -253,19 +251,13 @@ protected String missingShardsErrorMessage(StringBuilder missingShards) { @Override protected void executePhaseOnShard( SearchShardIterator shardIt, - SearchShardTarget shard, + Transport.Connection connection, SearchActionListener phaseListener ) { - final ShardOpenReaderRequest shardRequest = new ShardOpenReaderRequest( - shardIt.shardId(), - shardIt.getOriginalIndices(), - pitRequest.keepAlive() - ); - Transport.Connection connection = connectionLookup.apply(shardIt.getClusterAlias(), shard.getNodeId()); transportService.sendChildRequest( connection, OPEN_SHARD_READER_CONTEXT_NAME, - shardRequest, + new ShardOpenReaderRequest(shardIt.shardId(), shardIt.getOriginalIndices(), pitRequest.keepAlive()), task, new ActionListenerResponseHandler<>( phaseListener, @@ -276,31 +268,11 @@ protected void executePhaseOnShard( } @Override - protected SearchPhase getNextPhase(SearchPhaseResults results, SearchPhaseContext context) { + protected SearchPhase getNextPhase() { return new SearchPhase(getName()) { - - private void onExecuteFailure(Exception e) { - onPhaseFailure(this, "sending response failed", e); - } - @Override public void run() { - execute(new AbstractRunnable() { - @Override - public void onFailure(Exception e) { - onExecuteFailure(e); - } - - @Override - protected void doRun() { - sendSearchResponse(SearchResponseSections.EMPTY_WITH_TOTAL_HITS, results.getAtomicArray()); - } - - @Override - public boolean isForceExecution() { - return true; // we already created the PIT, no sense in rejecting the task that sends the response. - } - }); + sendSearchResponse(SearchResponseSections.EMPTY_WITH_TOTAL_HITS, results.getAtomicArray()); } }; } diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index 30faae9c1a5fb..c9985bb58d4bf 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; @@ -36,6 +37,7 @@ import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.GroupShardsIterator; @@ -51,9 +53,9 @@ import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.ArrayUtils; import org.elasticsearch.common.util.CollectionUtils; -import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.util.concurrent.CountDown; import org.elasticsearch.common.util.concurrent.EsExecutors; @@ -109,6 +111,7 @@ import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.LongSupplier; +import java.util.stream.Collectors; import static org.elasticsearch.action.search.SearchType.DFS_QUERY_THEN_FETCH; import static org.elasticsearch.action.search.SearchType.QUERY_THEN_FETCH; @@ -127,8 +130,6 @@ public class TransportSearchAction extends HandledTransportAction SHARD_COUNT_LIMIT_SETTING = Setting.longSetting( "action.search.shard_count.limit", @@ -161,6 +162,7 @@ public class TransportSearchAction extends HandledTransportAction buildPerIndexOriginalIndices( ClusterState clusterState, - Set indicesAndAliases, + Set indicesAndAliases, String[] indices, IndicesOptions indicesOptions ) { @@ -210,6 +212,9 @@ private Map buildPerIndexOriginalIndices( var blocks = clusterState.blocks(); // optimization: mostly we do not have any blocks so there's no point in the expensive per-index checking boolean hasBlocks = blocks.global().isEmpty() == false || blocks.indices().isEmpty() == false; + // Get a distinct set of index abstraction names present from the resolved expressions to help with the reverse resolution from + // concrete index to the expression that produced it. + Set indicesAndAliasesResources = indicesAndAliases.stream().map(ResolvedExpression::resource).collect(Collectors.toSet()); for (String index : indices) { if (hasBlocks) { blocks.indexBlockedRaiseException(ClusterBlockLevel.READ, index); @@ -226,8 +231,8 @@ private Map buildPerIndexOriginalIndices( String[] finalIndices = Strings.EMPTY_ARRAY; if (aliases == null || aliases.length == 0 - || indicesAndAliases.contains(index) - || hasDataStreamRef(clusterState, indicesAndAliases, index)) { + || indicesAndAliasesResources.contains(index) + || hasDataStreamRef(clusterState, indicesAndAliasesResources, index)) { finalIndices = new String[] { index }; } if (aliases != null) { @@ -246,7 +251,11 @@ private static boolean hasDataStreamRef(ClusterState clusterState, Set i return indicesAndAliases.contains(ret.getParentDataStream().getName()); } - Map buildIndexAliasFilters(ClusterState clusterState, Set indicesAndAliases, Index[] concreteIndices) { + Map buildIndexAliasFilters( + ClusterState clusterState, + Set indicesAndAliases, + Index[] concreteIndices + ) { final Map aliasFilterMap = new HashMap<>(); for (Index index : concreteIndices) { clusterState.blocks().indexBlockedRaiseException(ClusterBlockLevel.READ, index.getName()); @@ -371,15 +380,15 @@ void executeRequest( searchPhaseProvider.apply(delegate) ); } else { - if ((listener instanceof TelemetryListener tl) && CCS_TELEMETRY_FEATURE_FLAG.isEnabled()) { + if (listener instanceof TelemetryListener tl) { tl.setRemotes(resolvedIndices.getRemoteClusterIndices().size()); if (task.isAsync()) { tl.setFeature(CCSUsageTelemetry.ASYNC_FEATURE); } - String client = task.getHeader(Task.X_ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER); - if (client != null) { - tl.setClient(client); + if (original.pointInTimeBuilder() != null) { + tl.setFeature(CCSUsageTelemetry.PIT_FEATURE); } + tl.setClient(task); // Check if any of the index patterns are wildcard patterns var localIndices = resolvedIndices.getLocalIndices(); if (localIndices != null && Arrays.stream(localIndices.indices()).anyMatch(Regex::isSimpleMatchPattern)) { @@ -394,7 +403,7 @@ void executeRequest( } final TaskId parentTaskId = task.taskInfo(clusterService.localNode().getId(), false).taskId(); if (shouldMinimizeRoundtrips(rewritten)) { - if ((listener instanceof TelemetryListener tl) && CCS_TELEMETRY_FEATURE_FLAG.isEnabled()) { + if (listener instanceof TelemetryListener tl) { tl.setFeature(CCSUsageTelemetry.MRT_FEATURE); } final AggregationReduceContext.Builder aggregationReduceContextBuilder = rewritten.source() != null @@ -496,8 +505,11 @@ void executeRequest( } } }); + final SearchSourceBuilder source = original.source(); if (shouldOpenPIT(source)) { + // disabling shard reordering for request + original.setPreFilterShardSize(Integer.MAX_VALUE); openPIT(client, original, searchService.getDefaultKeepAliveInMillis(), listener.delegateFailureAndWrap((delegate, resp) -> { // We set the keep alive to -1 to indicate that we don't need the pit id in the response. // This is needed since we delete the pit prior to sending the response so the id doesn't exist anymore. @@ -1231,7 +1243,10 @@ private void executeSearch( } else { final Index[] indices = resolvedIndices.getConcreteLocalIndices(); concreteLocalIndices = Arrays.stream(indices).map(Index::getName).toArray(String[]::new); - final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(clusterState, searchRequest.indices()); + final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions( + clusterState, + searchRequest.indices() + ); aliasFilter = buildIndexAliasFilters(clusterState, indicesAndAliases, indices); aliasFilter.putAll(remoteAliasMap); localShardIterators = getLocalShardsIterator( @@ -1241,6 +1256,29 @@ private void executeSearch( indicesAndAliases, concreteLocalIndices ); + + // localShardIterators is empty since there are no matching indices. In such cases, + // we update the local cluster's status from RUNNING to SUCCESSFUL right away. Before + // we attempt to do that, we must ensure that the local cluster was specified in the user's + // search request. This is done by trying to fetch the local cluster via getCluster() and + // checking for a non-null return value. If the local cluster was never specified, its status + // update can be skipped. + if (localShardIterators.isEmpty() + && clusters != SearchResponse.Clusters.EMPTY + && clusters.getCluster(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY) != null) { + clusters.swapCluster( + RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, + (alias, v) -> new SearchResponse.Cluster.Builder(v).setStatus(SearchResponse.Cluster.Status.SUCCESSFUL) + .setTotalShards(0) + .setSuccessfulShards(0) + .setSkippedShards(0) + .setFailedShards(0) + .setFailures(Collections.emptyList()) + .setTook(TimeValue.timeValueMillis(0)) + .setTimedOut(false) + .build() + ); + } } final GroupShardsIterator shardIterators = mergeShardsIterators(localShardIterators, remoteShardIterators); @@ -1429,6 +1467,8 @@ public SearchPhase newSearchPhase( SearchResponse.Clusters clusters ) { if (preFilter) { + // only for aggs we need to contact shards even if there are no matches + boolean requireAtLeastOneMatch = searchRequest.source() != null && searchRequest.source().aggregations() != null; return new CanMatchPreFilterSearchPhase( logger, searchTransportService, @@ -1440,7 +1480,7 @@ public SearchPhase newSearchPhase( shardIterators, timeProvider, task, - true, + requireAtLeastOneMatch, searchService.getCoordinatorRewriteContextProvider(timeProvider::absoluteStartMillis), listener.delegateFailureAndWrap( (l, iters) -> newSearchPhase( @@ -1804,7 +1844,7 @@ List getLocalShardsIterator( ClusterState clusterState, SearchRequest searchRequest, String clusterAlias, - Set indicesAndAliases, + Set indicesAndAliases, String[] concreteIndices ) { var routingMap = indexNameExpressionResolver.resolveSearchRouting(clusterState, searchRequest.routing(), searchRequest.indices()); @@ -1840,7 +1880,7 @@ private interface TelemetryListener { void setFeature(String feature); - void setClient(String client); + void setClient(Task task); } private class SearchResponseActionListener implements ActionListener, TelemetryListener { @@ -1862,7 +1902,7 @@ private class SearchResponseActionListener implements ActionListener 0; + return SearchService.CCS_COLLECT_TELEMETRY.get(settings) && usageBuilder.getRemotesCount() > 0; } public void setRemotes(int count) { @@ -1875,8 +1915,8 @@ public void setFeature(String feature) { } @Override - public void setClient(String client) { - usageBuilder.setClient(client); + public void setClient(Task task) { + usageBuilder.setClientFromTask(task); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java index 7cf48245c9a94..4e3544f0170cb 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; @@ -109,15 +110,9 @@ private static SearchContextIdForNode readSearchContextIdForNodeExcludingContext private static SearchContextIdForNode innerReadSearchContextIdForNode(String contextUUID, StreamInput in) throws IOException { long id = in.readLong(); - String target = in.readString(); - String clusterAlias; - final int index = target.indexOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR); - if (index == -1) { - clusterAlias = null; - } else { - clusterAlias = target.substring(0, index); - target = target.substring(index + 1); - } + String[] split = RemoteClusterAware.splitIndexName(in.readString()); + String clusterAlias = split[0]; + String target = split[1]; return new SearchContextIdForNode(clusterAlias, target, new ShardSearchContextId(contextUUID, id)); } diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java index 1e2c445b22f3c..cffba76988f7d 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchShardsAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchShardsAction.java index 81b999a1239e5..b94bd95c93d8a 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchShardsAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchShardsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; @@ -16,6 +17,7 @@ import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.index.Index; @@ -126,7 +128,10 @@ public void searchShards(Task task, SearchShardsRequest searchShardsRequest, Act searchService.getRewriteContext(timeProvider::absoluteStartMillis, resolvedIndices, null), listener.delegateFailureAndWrap((delegate, searchRequest) -> { Index[] concreteIndices = resolvedIndices.getConcreteLocalIndices(); - final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(clusterState, searchRequest.indices()); + final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions( + clusterState, + searchRequest.indices() + ); final Map aliasFilters = transportSearchAction.buildIndexAliasFilters( clusterState, indicesAndAliases, diff --git a/server/src/main/java/org/elasticsearch/action/search/VersionMismatchException.java b/server/src/main/java/org/elasticsearch/action/search/VersionMismatchException.java index 4ca3744526e1d..69ea4484ae691 100644 --- a/server/src/main/java/org/elasticsearch/action/search/VersionMismatchException.java +++ b/server/src/main/java/org/elasticsearch/action/search/VersionMismatchException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.search; diff --git a/server/src/main/java/org/elasticsearch/action/search/package-info.java b/server/src/main/java/org/elasticsearch/action/search/package-info.java index aa6db7f4a177f..00853a7400fc1 100644 --- a/server/src/main/java/org/elasticsearch/action/search/package-info.java +++ b/server/src/main/java/org/elasticsearch/action/search/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/server/src/main/java/org/elasticsearch/action/support/AbstractThreadedActionListener.java b/server/src/main/java/org/elasticsearch/action/support/AbstractThreadedActionListener.java index 54f01abc63833..a22b369858edc 100644 --- a/server/src/main/java/org/elasticsearch/action/support/AbstractThreadedActionListener.java +++ b/server/src/main/java/org/elasticsearch/action/support/AbstractThreadedActionListener.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/ActionFilter.java b/server/src/main/java/org/elasticsearch/action/support/ActionFilter.java index 3a5c3517d6e59..4583c43101b46 100644 --- a/server/src/main/java/org/elasticsearch/action/support/ActionFilter.java +++ b/server/src/main/java/org/elasticsearch/action/support/ActionFilter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/ActionFilterChain.java b/server/src/main/java/org/elasticsearch/action/support/ActionFilterChain.java index 4c078b2548dbe..758ca2b92c7da 100644 --- a/server/src/main/java/org/elasticsearch/action/support/ActionFilterChain.java +++ b/server/src/main/java/org/elasticsearch/action/support/ActionFilterChain.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/ActionFilters.java b/server/src/main/java/org/elasticsearch/action/support/ActionFilters.java index bea1e7e72d851..261083f286024 100644 --- a/server/src/main/java/org/elasticsearch/action/support/ActionFilters.java +++ b/server/src/main/java/org/elasticsearch/action/support/ActionFilters.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/ActiveShardCount.java b/server/src/main/java/org/elasticsearch/action/support/ActiveShardCount.java index f6934e8d20ae6..8c096ec3b777c 100644 --- a/server/src/main/java/org/elasticsearch/action/support/ActiveShardCount.java +++ b/server/src/main/java/org/elasticsearch/action/support/ActiveShardCount.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/ActiveShardsObserver.java b/server/src/main/java/org/elasticsearch/action/support/ActiveShardsObserver.java index ffe8bde231f22..2f6c9f1b30e38 100644 --- a/server/src/main/java/org/elasticsearch/action/support/ActiveShardsObserver.java +++ b/server/src/main/java/org/elasticsearch/action/support/ActiveShardsObserver.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java b/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java index eb374a4ae9258..5b3c69f3de6b1 100644 --- a/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java +++ b/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/CancellableFanOut.java b/server/src/main/java/org/elasticsearch/action/support/CancellableFanOut.java index 363feb6830cfa..f6491cf1d437b 100644 --- a/server/src/main/java/org/elasticsearch/action/support/CancellableFanOut.java +++ b/server/src/main/java/org/elasticsearch/action/support/CancellableFanOut.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/ChannelActionListener.java b/server/src/main/java/org/elasticsearch/action/support/ChannelActionListener.java index c839db5720048..69a590ddcdeb8 100644 --- a/server/src/main/java/org/elasticsearch/action/support/ChannelActionListener.java +++ b/server/src/main/java/org/elasticsearch/action/support/ChannelActionListener.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/ContextPreservingActionListener.java b/server/src/main/java/org/elasticsearch/action/support/ContextPreservingActionListener.java index e4ac4fcf7a410..41166ace510fd 100644 --- a/server/src/main/java/org/elasticsearch/action/support/ContextPreservingActionListener.java +++ b/server/src/main/java/org/elasticsearch/action/support/ContextPreservingActionListener.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/CountDownActionListener.java b/server/src/main/java/org/elasticsearch/action/support/CountDownActionListener.java index 90b5826bf80ea..5a4a0005a3d2e 100644 --- a/server/src/main/java/org/elasticsearch/action/support/CountDownActionListener.java +++ b/server/src/main/java/org/elasticsearch/action/support/CountDownActionListener.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java b/server/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java index 6702f1f4f77ff..f7140fd10928d 100644 --- a/server/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java +++ b/server/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/DestructiveOperations.java b/server/src/main/java/org/elasticsearch/action/support/DestructiveOperations.java index 7c9d359718035..e2a68aa8f7200 100644 --- a/server/src/main/java/org/elasticsearch/action/support/DestructiveOperations.java +++ b/server/src/main/java/org/elasticsearch/action/support/DestructiveOperations.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/GroupedActionListener.java b/server/src/main/java/org/elasticsearch/action/support/GroupedActionListener.java index 47fbf1f381eb1..6e4b2ee546bd6 100644 --- a/server/src/main/java/org/elasticsearch/action/support/GroupedActionListener.java +++ b/server/src/main/java/org/elasticsearch/action/support/GroupedActionListener.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java b/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java index b9200a0e32736..ce414f47a9a63 100644 --- a/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/IndexComponentSelector.java b/server/src/main/java/org/elasticsearch/action/support/IndexComponentSelector.java new file mode 100644 index 0000000000000..910be151d1bf5 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/support/IndexComponentSelector.java @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.support; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.core.Nullable; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * We define as index components the two different sets of indices a data stream could consist of: + * - DATA: represents the backing indices + * - FAILURES: represent the failing indices + * - ALL: represents all available in this expression components, meaning if it's a data stream both backing and failure indices and if it's + * an index only the index itself. + * Note: An index is its own DATA component, but it cannot have a FAILURE component. + */ +public enum IndexComponentSelector implements Writeable { + DATA("data", (byte) 0), + FAILURES("failures", (byte) 1), + ALL_APPLICABLE("*", (byte) 2); + + private final String key; + private final byte id; + + IndexComponentSelector(String key, byte id) { + this.key = key; + this.id = id; + } + + public String getKey() { + return key; + } + + public byte getId() { + return id; + } + + private static final Map KEY_REGISTRY; + private static final Map ID_REGISTRY; + + static { + Map keyRegistry = new HashMap<>(IndexComponentSelector.values().length); + for (IndexComponentSelector value : IndexComponentSelector.values()) { + keyRegistry.put(value.getKey(), value); + } + KEY_REGISTRY = Collections.unmodifiableMap(keyRegistry); + Map idRegistry = new HashMap<>(IndexComponentSelector.values().length); + for (IndexComponentSelector value : IndexComponentSelector.values()) { + idRegistry.put(value.getId(), value); + } + ID_REGISTRY = Collections.unmodifiableMap(idRegistry); + } + + /** + * Retrieves the respective selector when the suffix key is recognised + * @param key the suffix key, probably parsed from an expression + * @return the selector or null if the key was not recognised. + */ + @Nullable + public static IndexComponentSelector getByKey(String key) { + return KEY_REGISTRY.get(key); + } + + public static IndexComponentSelector read(StreamInput in) throws IOException { + return getById(in.readByte()); + } + + // Visible for testing + static IndexComponentSelector getById(byte id) { + IndexComponentSelector indexComponentSelector = ID_REGISTRY.get(id); + if (indexComponentSelector == null) { + throw new IllegalArgumentException( + "Unknown id of index component selector [" + id + "], available options are: " + ID_REGISTRY + ); + } + return indexComponentSelector; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeByte(id); + } + + public boolean shouldIncludeData() { + return this == ALL_APPLICABLE || this == DATA; + } + + public boolean shouldIncludeFailures() { + return this == ALL_APPLICABLE || this == FAILURES; + } +} diff --git a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java index 7c9dcb608ec84..ebbd47336e3da 100644 --- a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java +++ b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; @@ -30,7 +31,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.function.Consumer; import java.util.stream.Collectors; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; @@ -47,24 +47,36 @@ * @param gatekeeperOptions, applies to all the resolved indices and defines if throttled will be included and if certain type of * aliases or indices are allowed, or they will throw an error. It acts as a gatekeeper when an action * does not support certain options. - * @param failureStoreOptions, applies to all indices already matched and controls the type of indices that will be returned. Currently, - * there are two types, data stream failure indices (only certain data streams have them) and data stream - * backing indices or stand-alone indices. + * @param selectorOptions, applies to all resolved expressions, and it specifies the index component that should be included, if there + * is no index component defined on the expression level. */ public record IndicesOptions( ConcreteTargetOptions concreteTargetOptions, WildcardOptions wildcardOptions, GatekeeperOptions gatekeeperOptions, - FailureStoreOptions failureStoreOptions + SelectorOptions selectorOptions ) implements ToXContentFragment { - public IndicesOptions( - ConcreteTargetOptions concreteTargetOptions, - WildcardOptions wildcardOptions, - GatekeeperOptions gatekeeperOptions - ) { - this(concreteTargetOptions, wildcardOptions, gatekeeperOptions, FailureStoreOptions.DEFAULT); - } + /** + * @deprecated this query param will be replaced by the selector `::` on the expression level + */ + @Deprecated + public static final String FAILURE_STORE_QUERY_PARAM = "failure_store"; + /** + * @deprecated this value will be replaced by the selector `::*` on the expression level + */ + @Deprecated + public static final String INCLUDE_ALL = "include"; + /** + * @deprecated this value will be replaced by the selector `::data` on the expression level + */ + @Deprecated + public static final String INCLUDE_ONLY_REGULAR_INDICES = "exclude"; + /** + * @deprecated this value will be replaced by the selector `::failures` on the expression level + */ + @Deprecated + public static final String INCLUDE_ONLY_FAILURE_INDICES = "only"; public static IndicesOptions.Builder builder() { return new Builder(); @@ -309,7 +321,7 @@ public static Builder builder(WildcardOptions wildcardOptions) { * - The "allow*" flags, which purpose is to enable actions to define certain conditions that need to apply on the concrete indices * they accept. For example, single-index actions will set allowAliasToMultipleIndices to false, while search will not accept a * closed index etc. These options are not configurable by the end-user. - * - The ignoreThrottled flag, which is a depricared flag that will filter out frozen indices. + * - The ignoreThrottled flag, which is a deprecated flag that will filter out frozen indices. * @param allowAliasToMultipleIndices, allow aliases to multiple indices, true by default. * @param allowClosedIndices, allow closed indices, true by default. * @param allowFailureIndices, allow failure indices in the response, true by default @@ -407,97 +419,47 @@ public static Builder builder(GatekeeperOptions gatekeeperOptions) { } /** - * Applies to all indices already matched and controls the type of indices that will be returned. There are two types, data stream - * failure indices (only certain data streams have them) and data stream backing indices or stand-alone indices. - * @param includeRegularIndices, when true regular or data stream backing indices will be retrieved. - * @param includeFailureIndices, when true data stream failure indices will be included. + * Defines which selectors should be used by default for an index operation in the event that no selectors are provided. */ - public record FailureStoreOptions(boolean includeRegularIndices, boolean includeFailureIndices) - implements - Writeable, - ToXContentFragment { - - public static final String FAILURE_STORE = "failure_store"; - public static final String INCLUDE_ALL = "include"; - public static final String INCLUDE_ONLY_REGULAR_INDICES = "exclude"; - public static final String INCLUDE_ONLY_FAILURE_INDICES = "only"; - - public static final FailureStoreOptions DEFAULT = new FailureStoreOptions(true, false); - - public static FailureStoreOptions read(StreamInput in) throws IOException { - return new FailureStoreOptions(in.readBoolean(), in.readBoolean()); - } + public record SelectorOptions(IndexComponentSelector defaultSelector) implements Writeable { - public static FailureStoreOptions parseParameters(Object failureStoreValue, FailureStoreOptions defaultOptions) { - if (failureStoreValue == null) { - return defaultOptions; - } - FailureStoreOptions.Builder builder = defaultOptions == null - ? new FailureStoreOptions.Builder() - : new FailureStoreOptions.Builder(defaultOptions); - return switch (failureStoreValue.toString()) { - case INCLUDE_ALL -> builder.includeRegularIndices(true).includeFailureIndices(true).build(); - case INCLUDE_ONLY_REGULAR_INDICES -> builder.includeRegularIndices(true).includeFailureIndices(false).build(); - case INCLUDE_ONLY_FAILURE_INDICES -> builder.includeRegularIndices(false).includeFailureIndices(true).build(); - default -> throw new IllegalArgumentException("No valid " + FAILURE_STORE + " value [" + failureStoreValue + "]"); - }; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return builder.field(FAILURE_STORE, displayValue()); - } - - public String displayValue() { - if (includeRegularIndices && includeFailureIndices) { - return INCLUDE_ALL; - } else if (includeRegularIndices) { - return INCLUDE_ONLY_REGULAR_INDICES; + public static final SelectorOptions ALL_APPLICABLE = new SelectorOptions(IndexComponentSelector.ALL_APPLICABLE); + public static final SelectorOptions DATA = new SelectorOptions(IndexComponentSelector.DATA); + public static final SelectorOptions FAILURES = new SelectorOptions(IndexComponentSelector.FAILURES); + /** + * Default instance. Uses
    ::data
    as the default selector if none are present in an index expression. + */ + public static final SelectorOptions DEFAULT = DATA; + + public static SelectorOptions read(StreamInput in) throws IOException { + if (in.getTransportVersion().before(TransportVersions.INTRODUCE_ALL_APPLICABLE_SELECTOR)) { + EnumSet set = in.readEnumSet(IndexComponentSelector.class); + if (set.isEmpty() || set.size() == 2) { + assert set.contains(IndexComponentSelector.DATA) && set.contains(IndexComponentSelector.FAILURES) + : "The enum set only supported ::data and ::failures"; + return SelectorOptions.ALL_APPLICABLE; + } else if (set.contains(IndexComponentSelector.DATA)) { + return SelectorOptions.DATA; + } else { + return SelectorOptions.FAILURES; + } + } else { + return new SelectorOptions(IndexComponentSelector.read(in)); } - return INCLUDE_ONLY_FAILURE_INDICES; } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeBoolean(includeRegularIndices); - out.writeBoolean(includeFailureIndices); - } - - public static class Builder { - private boolean includeRegularIndices; - private boolean includeFailureIndices; - - public Builder() { - this(DEFAULT); - } - - Builder(FailureStoreOptions options) { - includeRegularIndices = options.includeRegularIndices; - includeFailureIndices = options.includeFailureIndices; - } - - public Builder includeRegularIndices(boolean includeRegularIndices) { - this.includeRegularIndices = includeRegularIndices; - return this; - } - - public Builder includeFailureIndices(boolean includeFailureIndices) { - this.includeFailureIndices = includeFailureIndices; - return this; - } - - public FailureStoreOptions build() { - return new FailureStoreOptions(includeRegularIndices, includeFailureIndices); + if (out.getTransportVersion().before(TransportVersions.INTRODUCE_ALL_APPLICABLE_SELECTOR)) { + switch (defaultSelector) { + case ALL_APPLICABLE -> out.writeEnumSet(EnumSet.of(IndexComponentSelector.DATA, IndexComponentSelector.FAILURES)); + case DATA -> out.writeEnumSet(EnumSet.of(IndexComponentSelector.DATA)); + case FAILURES -> out.writeEnumSet(EnumSet.of(IndexComponentSelector.FAILURES)); + } + } else { + defaultSelector.writeTo(out); } } - - public static Builder builder() { - return new Builder(); - } - - public static Builder builder(FailureStoreOptions failureStoreOptions) { - return new Builder(failureStoreOptions); - } } /** @@ -549,7 +511,7 @@ private enum Option { ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS, WildcardOptions.DEFAULT, GatekeeperOptions.DEFAULT, - FailureStoreOptions.DEFAULT + SelectorOptions.DEFAULT ); public static final IndicesOptions STRICT_EXPAND_OPEN = IndicesOptions.builder() @@ -569,7 +531,7 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false)) + .selectorOptions(SelectorOptions.DATA) .build(); public static final IndicesOptions STRICT_EXPAND_OPEN_FAILURE_STORE = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -588,7 +550,7 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true)) + .selectorOptions(SelectorOptions.ALL_APPLICABLE) .build(); public static final IndicesOptions LENIENT_EXPAND_OPEN = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS) @@ -607,7 +569,25 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false)) + .selectorOptions(SelectorOptions.DATA) + .build(); + public static final IndicesOptions LENIENT_EXPAND_OPEN_NO_SELECTORS = IndicesOptions.builder() + .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS) + .wildcardOptions( + WildcardOptions.builder() + .matchOpen(true) + .matchClosed(false) + .includeHidden(false) + .allowEmptyExpressions(true) + .resolveAliases(true) + ) + .gatekeeperOptions( + GatekeeperOptions.builder() + .allowAliasToMultipleIndices(true) + .allowClosedIndices(true) + .allowFailureIndices(false) + .ignoreThrottled(false) + ) .build(); public static final IndicesOptions LENIENT_EXPAND_OPEN_HIDDEN = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS) @@ -626,7 +606,7 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false)) + .selectorOptions(SelectorOptions.DATA) .build(); public static final IndicesOptions LENIENT_EXPAND_OPEN_CLOSED = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS) @@ -645,7 +625,7 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false)) + .selectorOptions(SelectorOptions.DATA) .build(); public static final IndicesOptions LENIENT_EXPAND_OPEN_CLOSED_HIDDEN = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS) @@ -659,7 +639,20 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false)) + .selectorOptions(SelectorOptions.DATA) + .build(); + public static final IndicesOptions LENIENT_EXPAND_OPEN_CLOSED_HIDDEN_NO_SELECTOR = IndicesOptions.builder() + .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS) + .wildcardOptions( + WildcardOptions.builder().matchOpen(true).matchClosed(true).includeHidden(true).allowEmptyExpressions(true).resolveAliases(true) + ) + .gatekeeperOptions( + GatekeeperOptions.builder() + .allowAliasToMultipleIndices(true) + .allowClosedIndices(true) + .allowFailureIndices(false) + .ignoreThrottled(false) + ) .build(); public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -678,7 +671,7 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false)) + .selectorOptions(SelectorOptions.DATA) .build(); public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED_HIDDEN = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -692,7 +685,20 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false)) + .selectorOptions(SelectorOptions.DATA) + .build(); + public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED_HIDDEN_NO_SELECTORS = IndicesOptions.builder() + .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) + .wildcardOptions( + WildcardOptions.builder().matchOpen(true).matchClosed(true).includeHidden(true).allowEmptyExpressions(true).resolveAliases(true) + ) + .gatekeeperOptions( + GatekeeperOptions.builder() + .allowAliasToMultipleIndices(true) + .allowClosedIndices(true) + .allowFailureIndices(false) + .ignoreThrottled(false) + ) .build(); public static final IndicesOptions LENIENT_EXPAND_OPEN_CLOSED_FAILURE_STORE = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS) @@ -711,7 +717,7 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true)) + .selectorOptions(SelectorOptions.ALL_APPLICABLE) .build(); public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED_HIDDEN_FAILURE_STORE = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -725,7 +731,7 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true)) + .selectorOptions(SelectorOptions.ALL_APPLICABLE) .build(); public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED_FAILURE_STORE = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -744,7 +750,7 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true)) + .selectorOptions(SelectorOptions.ALL_APPLICABLE) .build(); public static final IndicesOptions STRICT_EXPAND_OPEN_FORBID_CLOSED = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -763,7 +769,7 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false)) + .selectorOptions(SelectorOptions.DATA) .build(); public static final IndicesOptions STRICT_EXPAND_OPEN_HIDDEN_FORBID_CLOSED = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -782,7 +788,7 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false)) + .selectorOptions(SelectorOptions.DATA) .build(); public static final IndicesOptions STRICT_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -801,7 +807,7 @@ private enum Option { .allowFailureIndices(true) .allowAliasToMultipleIndices(true) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false)) + .selectorOptions(SelectorOptions.DATA) .build(); public static final IndicesOptions STRICT_SINGLE_INDEX_NO_EXPAND_FORBID_CLOSED = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -820,7 +826,7 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false)) + .selectorOptions(SelectorOptions.DATA) .build(); public static final IndicesOptions STRICT_NO_EXPAND_FORBID_CLOSED = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -839,7 +845,7 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false)) + .selectorOptions(SelectorOptions.DATA) .build(); /** @@ -897,7 +903,7 @@ public boolean forbidClosedIndices() { } /** - * @return Whether execution on closed indices is allowed. + * @return Whether execution on failure indices is allowed. */ public boolean allowFailureIndices() { return gatekeeperOptions.allowFailureIndices(); @@ -928,14 +934,14 @@ public boolean ignoreThrottled() { * @return whether regular indices (stand-alone or backing indices) will be included in the response */ public boolean includeRegularIndices() { - return failureStoreOptions().includeRegularIndices(); + return selectorOptions().defaultSelector().shouldIncludeData(); } /** * @return whether failure indices (only supported by certain data streams) will be included in the response */ public boolean includeFailureIndices() { - return failureStoreOptions().includeFailureIndices(); + return selectorOptions().defaultSelector().shouldIncludeFailures(); } public void writeIndicesOptions(StreamOutput out) throws IOException { @@ -976,8 +982,12 @@ public void writeIndicesOptions(StreamOutput out) throws IOException { states.add(WildcardStates.HIDDEN); } out.writeEnumSet(states); - if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { - failureStoreOptions.writeTo(out); + if (out.getTransportVersion().between(TransportVersions.V_8_14_0, TransportVersions.V_8_16_0)) { + out.writeBoolean(includeRegularIndices()); + out.writeBoolean(includeFailureIndices()); + } + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { + selectorOptions.writeTo(out); } } @@ -998,16 +1008,29 @@ public static IndicesOptions readIndicesOptions(StreamInput in) throws IOExcepti .allowFailureIndices(allowFailureIndices) .ignoreThrottled(options.contains(Option.IGNORE_THROTTLED)) .build(); - FailureStoreOptions failureStoreOptions = in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) - ? FailureStoreOptions.read(in) - : FailureStoreOptions.DEFAULT; + SelectorOptions selectorOptions = SelectorOptions.DEFAULT; + if (in.getTransportVersion().between(TransportVersions.V_8_14_0, TransportVersions.V_8_16_0)) { + // Reading from an older node, which will be sending two booleans that we must read out and ignore. + var includeData = in.readBoolean(); + var includeFailures = in.readBoolean(); + if (includeData && includeFailures) { + selectorOptions = SelectorOptions.ALL_APPLICABLE; + } else if (includeData) { + selectorOptions = SelectorOptions.DATA; + } else { + selectorOptions = SelectorOptions.FAILURES; + } + } + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { + selectorOptions = SelectorOptions.read(in); + } return new IndicesOptions( options.contains(Option.ALLOW_UNAVAILABLE_CONCRETE_TARGETS) ? ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS : ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS, wildcardOptions, gatekeeperOptions, - failureStoreOptions + selectorOptions ); } @@ -1015,7 +1038,7 @@ public static class Builder { private ConcreteTargetOptions concreteTargetOptions; private WildcardOptions wildcardOptions; private GatekeeperOptions gatekeeperOptions; - private FailureStoreOptions failureStoreOptions; + private SelectorOptions selectorOptions; Builder() { this(DEFAULT); @@ -1025,7 +1048,7 @@ public static class Builder { concreteTargetOptions = indicesOptions.concreteTargetOptions; wildcardOptions = indicesOptions.wildcardOptions; gatekeeperOptions = indicesOptions.gatekeeperOptions; - failureStoreOptions = indicesOptions.failureStoreOptions; + selectorOptions = indicesOptions.selectorOptions; } public Builder concreteTargetOptions(ConcreteTargetOptions concreteTargetOptions) { @@ -1053,25 +1076,13 @@ public Builder gatekeeperOptions(GatekeeperOptions.Builder generalOptions) { return this; } - public Builder failureStoreOptions(FailureStoreOptions failureStoreOptions) { - this.failureStoreOptions = failureStoreOptions; - return this; - } - - public Builder failureStoreOptions(FailureStoreOptions.Builder failureStoreOptions) { - this.failureStoreOptions = failureStoreOptions.build(); - return this; - } - - public Builder failureStoreOptions(Consumer failureStoreOptionsConfig) { - FailureStoreOptions.Builder failureStoreOptionsBuilder = FailureStoreOptions.builder(failureStoreOptions); - failureStoreOptionsConfig.accept(failureStoreOptionsBuilder); - this.failureStoreOptions = failureStoreOptionsBuilder.build(); + public Builder selectorOptions(SelectorOptions selectorOptions) { + this.selectorOptions = selectorOptions; return this; } public IndicesOptions build() { - return new IndicesOptions(concreteTargetOptions, wildcardOptions, gatekeeperOptions, failureStoreOptions); + return new IndicesOptions(concreteTargetOptions, wildcardOptions, gatekeeperOptions, selectorOptions); } } @@ -1170,11 +1181,12 @@ public static IndicesOptions fromOptions( .allowClosedIndices(forbidClosedIndices == false) .ignoreThrottled(ignoreThrottled) .build(); + final SelectorOptions selectorOptions = SelectorOptions.DEFAULT; return new IndicesOptions( ignoreUnavailable ? ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS : ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS, wildcards, gatekeeperOptions, - FailureStoreOptions.DEFAULT + selectorOptions ); } @@ -1188,9 +1200,7 @@ public static IndicesOptions fromRequest(RestRequest request, IndicesOptions def request.param(ConcreteTargetOptions.IGNORE_UNAVAILABLE), request.param(WildcardOptions.ALLOW_NO_INDICES), request.param(GatekeeperOptions.IGNORE_THROTTLED), - DataStream.isFailureStoreFeatureFlagEnabled() - ? request.param(FailureStoreOptions.FAILURE_STORE) - : FailureStoreOptions.INCLUDE_ONLY_REGULAR_INDICES, + DataStream.isFailureStoreFeatureFlagEnabled() ? request.param(FAILURE_STORE_QUERY_PARAM) : INCLUDE_ONLY_REGULAR_INDICES, defaultSettings ); } @@ -1206,7 +1216,7 @@ public static IndicesOptions fromMap(Map map, IndicesOptions def map.containsKey(GatekeeperOptions.IGNORE_THROTTLED) ? map.get(GatekeeperOptions.IGNORE_THROTTLED) : map.get("ignoreThrottled"), - map.containsKey(FailureStoreOptions.FAILURE_STORE) ? map.get(FailureStoreOptions.FAILURE_STORE) : map.get("failureStore"), + map.containsKey(FAILURE_STORE_QUERY_PARAM) ? map.get(FAILURE_STORE_QUERY_PARAM) : map.get("failureStore"), defaultSettings ); } @@ -1234,7 +1244,7 @@ public static boolean isIndicesOptions(String name) { || "ignoreThrottled".equals(name) || WildcardOptions.ALLOW_NO_INDICES.equals(name) || "allowNoIndices".equals(name) - || (DataStream.isFailureStoreFeatureFlagEnabled() && FailureStoreOptions.FAILURE_STORE.equals(name)) + || (DataStream.isFailureStoreFeatureFlagEnabled() && FAILURE_STORE_QUERY_PARAM.equals(name)) || (DataStream.isFailureStoreFeatureFlagEnabled() && "failureStore".equals(name)); } @@ -1266,26 +1276,51 @@ public static IndicesOptions fromParameters( WildcardOptions wildcards = WildcardOptions.parseParameters(wildcardsString, allowNoIndicesString, defaultSettings.wildcardOptions); GatekeeperOptions gatekeeperOptions = GatekeeperOptions.parseParameter(ignoreThrottled, defaultSettings.gatekeeperOptions); - FailureStoreOptions failureStoreOptions = DataStream.isFailureStoreFeatureFlagEnabled() - ? FailureStoreOptions.parseParameters(failureStoreString, defaultSettings.failureStoreOptions) - : FailureStoreOptions.DEFAULT; + SelectorOptions selectorOptions = DataStream.isFailureStoreFeatureFlagEnabled() + ? parseFailureStoreParameters(failureStoreString, defaultSettings.selectorOptions) + : SelectorOptions.DEFAULT; // note that allowAliasesToMultipleIndices is not exposed, always true (only for internal use) return IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.fromParameter(ignoreUnavailableString, defaultSettings.concreteTargetOptions)) .wildcardOptions(wildcards) .gatekeeperOptions(gatekeeperOptions) - .failureStoreOptions(failureStoreOptions) + .selectorOptions(selectorOptions) .build(); } + /** + * @deprecated This method parses the query parameter failure_store. This is a deprecated param, and it will be replaced + * the selector suffix, for example `my-data-stream::data` or `my-data-stream::failures` + */ + @Deprecated + private static SelectorOptions parseFailureStoreParameters(Object failureStoreValue, SelectorOptions defaultOptions) { + if (failureStoreValue == null) { + return defaultOptions; + } + return switch (failureStoreValue.toString()) { + case INCLUDE_ALL -> SelectorOptions.ALL_APPLICABLE; + case INCLUDE_ONLY_REGULAR_INDICES -> SelectorOptions.DATA; + case INCLUDE_ONLY_FAILURE_INDICES -> SelectorOptions.FAILURES; + default -> throw new IllegalArgumentException("No valid " + FAILURE_STORE_QUERY_PARAM + " value [" + failureStoreValue + "]"); + }; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { concreteTargetOptions.toXContent(builder, params); wildcardOptions.toXContent(builder, params); gatekeeperOptions.toXContent(builder, params); if (DataStream.isFailureStoreFeatureFlagEnabled()) { - failureStoreOptions.toXContent(builder, params); + String displayValue; + if (SelectorOptions.ALL_APPLICABLE.equals(selectorOptions())) { + displayValue = INCLUDE_ALL; + } else if (SelectorOptions.DATA.equals(selectorOptions())) { + displayValue = INCLUDE_ONLY_REGULAR_INDICES; + } else { + displayValue = INCLUDE_ONLY_FAILURE_INDICES; + } + builder.field(FAILURE_STORE_QUERY_PARAM, displayValue); } return builder; } @@ -1294,7 +1329,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par private static final ParseField IGNORE_UNAVAILABLE_FIELD = new ParseField(ConcreteTargetOptions.IGNORE_UNAVAILABLE); private static final ParseField IGNORE_THROTTLED_FIELD = new ParseField(GatekeeperOptions.IGNORE_THROTTLED).withAllDeprecated(); private static final ParseField ALLOW_NO_INDICES_FIELD = new ParseField(WildcardOptions.ALLOW_NO_INDICES); - private static final ParseField FAILURE_STORE_FIELD = new ParseField(FailureStoreOptions.FAILURE_STORE); + private static final ParseField FAILURE_STORE_FIELD = new ParseField(FAILURE_STORE_QUERY_PARAM); public static IndicesOptions fromXContent(XContentParser parser) throws IOException { return fromXContent(parser, null); @@ -1305,7 +1340,7 @@ public static IndicesOptions fromXContent(XContentParser parser, @Nullable Indic WildcardOptions.Builder wildcards = defaults == null ? null : WildcardOptions.builder(defaults.wildcardOptions()); GatekeeperOptions.Builder generalOptions = GatekeeperOptions.builder() .ignoreThrottled(defaults != null && defaults.gatekeeperOptions().ignoreThrottled()); - FailureStoreOptions failureStoreOptions = defaults == null ? FailureStoreOptions.DEFAULT : defaults.failureStoreOptions(); + SelectorOptions selectorOptions = defaults == null ? SelectorOptions.DEFAULT : defaults.selectorOptions(); Boolean allowNoIndices = defaults == null ? null : defaults.allowNoIndices(); Boolean ignoreUnavailable = defaults == null ? null : defaults.ignoreUnavailable(); Token token = parser.currentToken() == Token.START_OBJECT ? parser.currentToken() : parser.nextToken(); @@ -1357,7 +1392,7 @@ public static IndicesOptions fromXContent(XContentParser parser, @Nullable Indic generalOptions.ignoreThrottled(parser.booleanValue()); } else if (DataStream.isFailureStoreFeatureFlagEnabled() && FAILURE_STORE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - failureStoreOptions = FailureStoreOptions.parseParameters(parser.text(), failureStoreOptions); + selectorOptions = parseFailureStoreParameters(parser.text(), selectorOptions); } else { throw new ElasticsearchParseException( "could not read indices options. Unexpected index option [" + currentFieldName + "]" @@ -1388,7 +1423,7 @@ public static IndicesOptions fromXContent(XContentParser parser, @Nullable Indic .concreteTargetOptions(new ConcreteTargetOptions(ignoreUnavailable)) .wildcardOptions(wildcards) .gatekeeperOptions(generalOptions) - .failureStoreOptions(failureStoreOptions) + .selectorOptions(selectorOptions) .build(); } diff --git a/server/src/main/java/org/elasticsearch/action/support/ListenableActionFuture.java b/server/src/main/java/org/elasticsearch/action/support/ListenableActionFuture.java index 2c31cc123fa46..a03cbd7bea3da 100644 --- a/server/src/main/java/org/elasticsearch/action/support/ListenableActionFuture.java +++ b/server/src/main/java/org/elasticsearch/action/support/ListenableActionFuture.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/ListenerTimeouts.java b/server/src/main/java/org/elasticsearch/action/support/ListenerTimeouts.java index a4e2121c56aaf..939cdcdc92955 100644 --- a/server/src/main/java/org/elasticsearch/action/support/ListenerTimeouts.java +++ b/server/src/main/java/org/elasticsearch/action/support/ListenerTimeouts.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/MappedActionFilter.java b/server/src/main/java/org/elasticsearch/action/support/MappedActionFilter.java index deff2d371e633..42c34d8e4683a 100644 --- a/server/src/main/java/org/elasticsearch/action/support/MappedActionFilter.java +++ b/server/src/main/java/org/elasticsearch/action/support/MappedActionFilter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/MappedActionFilters.java b/server/src/main/java/org/elasticsearch/action/support/MappedActionFilters.java index 89384bc4f9abf..6d90c1f5d499b 100644 --- a/server/src/main/java/org/elasticsearch/action/support/MappedActionFilters.java +++ b/server/src/main/java/org/elasticsearch/action/support/MappedActionFilters.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/PlainActionFuture.java b/server/src/main/java/org/elasticsearch/action/support/PlainActionFuture.java index 4022ef3873771..2e0d54805660e 100644 --- a/server/src/main/java/org/elasticsearch/action/support/PlainActionFuture.java +++ b/server/src/main/java/org/elasticsearch/action/support/PlainActionFuture.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/RefCountAwareThreadedActionListener.java b/server/src/main/java/org/elasticsearch/action/support/RefCountAwareThreadedActionListener.java index 3b13f37d238a0..322623dcaa1f5 100644 --- a/server/src/main/java/org/elasticsearch/action/support/RefCountAwareThreadedActionListener.java +++ b/server/src/main/java/org/elasticsearch/action/support/RefCountAwareThreadedActionListener.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/RefCountingListener.java b/server/src/main/java/org/elasticsearch/action/support/RefCountingListener.java index ff5c3115e569b..70b95b4354768 100644 --- a/server/src/main/java/org/elasticsearch/action/support/RefCountingListener.java +++ b/server/src/main/java/org/elasticsearch/action/support/RefCountingListener.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/RefCountingRunnable.java b/server/src/main/java/org/elasticsearch/action/support/RefCountingRunnable.java index 8dcc801f10c30..53653536e8c89 100644 --- a/server/src/main/java/org/elasticsearch/action/support/RefCountingRunnable.java +++ b/server/src/main/java/org/elasticsearch/action/support/RefCountingRunnable.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/ReservedStateAwareHandledTransportAction.java b/server/src/main/java/org/elasticsearch/action/support/ReservedStateAwareHandledTransportAction.java index dfe950b3e32c8..9b34a20149234 100644 --- a/server/src/main/java/org/elasticsearch/action/support/ReservedStateAwareHandledTransportAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/ReservedStateAwareHandledTransportAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/RetryableAction.java b/server/src/main/java/org/elasticsearch/action/support/RetryableAction.java index 3e5114ddcfa9a..0bc378fb4d979 100644 --- a/server/src/main/java/org/elasticsearch/action/support/RetryableAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/RetryableAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/SubscribableListener.java b/server/src/main/java/org/elasticsearch/action/support/SubscribableListener.java index 6a2673e3276fc..c6c240e3b6759 100644 --- a/server/src/main/java/org/elasticsearch/action/support/SubscribableListener.java +++ b/server/src/main/java/org/elasticsearch/action/support/SubscribableListener.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/ThreadedActionListener.java b/server/src/main/java/org/elasticsearch/action/support/ThreadedActionListener.java index 5c13d8f93746e..a06440111c7ff 100644 --- a/server/src/main/java/org/elasticsearch/action/support/ThreadedActionListener.java +++ b/server/src/main/java/org/elasticsearch/action/support/ThreadedActionListener.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/TransportAction.java b/server/src/main/java/org/elasticsearch/action/support/TransportAction.java index 65a7e2302b9ae..ecfe80d3062df 100644 --- a/server/src/main/java/org/elasticsearch/action/support/TransportAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/TransportAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/TransportActions.java b/server/src/main/java/org/elasticsearch/action/support/TransportActions.java index f7c0450ad843e..7ac7a9d380bc6 100644 --- a/server/src/main/java/org/elasticsearch/action/support/TransportActions.java +++ b/server/src/main/java/org/elasticsearch/action/support/TransportActions.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/TransportLocalClusterStateAction.java b/server/src/main/java/org/elasticsearch/action/support/TransportLocalClusterStateAction.java index 6af5a3a1e8384..d2754bf11f532 100644 --- a/server/src/main/java/org/elasticsearch/action/support/TransportLocalClusterStateAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/TransportLocalClusterStateAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/UnsafePlainActionFuture.java b/server/src/main/java/org/elasticsearch/action/support/UnsafePlainActionFuture.java index b76dfe07e18ed..8fc8592cbda4d 100644 --- a/server/src/main/java/org/elasticsearch/action/support/UnsafePlainActionFuture.java +++ b/server/src/main/java/org/elasticsearch/action/support/UnsafePlainActionFuture.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/WriteRequest.java b/server/src/main/java/org/elasticsearch/action/support/WriteRequest.java index 64355a32c3a63..a26d49f9a1c37 100644 --- a/server/src/main/java/org/elasticsearch/action/support/WriteRequest.java +++ b/server/src/main/java/org/elasticsearch/action/support/WriteRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/WriteRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/WriteRequestBuilder.java index 659582d13e22e..9ee7bb88d2800 100644 --- a/server/src/main/java/org/elasticsearch/action/support/WriteRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/support/WriteRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/WriteResponse.java b/server/src/main/java/org/elasticsearch/action/support/WriteResponse.java index dc4823f49122f..d4f9d619bbe93 100644 --- a/server/src/main/java/org/elasticsearch/action/support/WriteResponse.java +++ b/server/src/main/java/org/elasticsearch/action/support/WriteResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support; diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/BaseBroadcastResponse.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/BaseBroadcastResponse.java index 3a27d6ac58534..c7e4de245d4d9 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/BaseBroadcastResponse.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/BaseBroadcastResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.broadcast; diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastOperationRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastOperationRequestBuilder.java index 7702a1e30fe30..12e30c4bfa088 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastOperationRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastOperationRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.broadcast; diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastRequest.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastRequest.java index 569e407f7f601..e0c1d94ed1de3 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastRequest.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.broadcast; diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastResponse.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastResponse.java index fefc0603e7ce8..c01b44aff7886 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastResponse.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.broadcast; diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastShardOperationFailedException.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastShardOperationFailedException.java index b1594bf5ba935..357e32d78641d 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastShardOperationFailedException.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastShardOperationFailedException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.broadcast; diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastShardRequest.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastShardRequest.java index 0fe9f46c9969b..5ad20ebdd5d1a 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastShardRequest.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastShardRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.broadcast; diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastShardResponse.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastShardResponse.java index ad89e0e8ec430..ed4b948189a49 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastShardResponse.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastShardResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.broadcast; diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/ChunkedBroadcastResponse.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/ChunkedBroadcastResponse.java index 6564e30799ce8..4b65af23c622a 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/ChunkedBroadcastResponse.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/ChunkedBroadcastResponse.java @@ -1,15 +1,16 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.broadcast; import org.elasticsearch.action.support.DefaultShardOperationFailedException; -import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.common.xcontent.ChunkedToXContentObject; import org.elasticsearch.rest.action.RestActions; import org.elasticsearch.xcontent.ToXContent; @@ -34,11 +35,8 @@ public ChunkedBroadcastResponse( @Override public final Iterator toXContentChunked(ToXContent.Params params) { - return Iterators.concat(Iterators.single((b, p) -> { - b.startObject(); - RestActions.buildBroadcastShardsHeader(b, p, this); - return b; - }), customXContentChunks(params), Iterators.single((builder, p) -> builder.endObject())); + return ChunkedToXContent.builder(params) + .object(ob -> ob.append((b, p) -> RestActions.buildBroadcastShardsHeader(b, p, this)).append(this::customXContentChunks)); } protected abstract Iterator customXContentChunks(ToXContent.Params params); diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java index cd1d9ac293823..836eedd960f10 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.broadcast; diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java index 828230a12c0da..5c6171dc22536 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.broadcast.node; diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/BroadcastUnpromotableRequest.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/BroadcastUnpromotableRequest.java index 312a9843c9e2b..ac5fff1d1700f 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/BroadcastUnpromotableRequest.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/BroadcastUnpromotableRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.broadcast.unpromotable; diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java index 1e7e07d9135fe..1255dbdf7419d 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.broadcast.unpromotable; diff --git a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java index a55467fbfadf8..11872cd85d9ba 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java @@ -1,14 +1,14 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.master; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.cluster.ack.AckedRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.TimeValue; @@ -22,9 +22,7 @@ * Abstract base class for action requests that track acknowledgements of cluster state updates: such a request is acknowledged only once * the cluster state update is committed and all relevant nodes have applied it and acknowledged its application to the elected master.. */ -public abstract class AcknowledgedRequest> extends MasterNodeRequest - implements - AckedRequest { +public abstract class AcknowledgedRequest> extends MasterNodeRequest { public static final TimeValue DEFAULT_ACK_TIMEOUT = timeValueSeconds(30); @@ -73,7 +71,6 @@ public final Request ackTimeout(TimeValue ackTimeout) { /** * @return the current ack timeout as a {@link TimeValue} */ - @Override public final TimeValue ackTimeout() { return ackTimeout; } diff --git a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequestBuilder.java index a749be99e2808..47a4570cafa17 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.master; diff --git a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java index 892db8d4a6d04..89e3c98ea003b 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.master; diff --git a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedTransportMasterNodeAction.java b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedTransportMasterNodeAction.java index ce76a64009030..f49aecbbfc1e5 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedTransportMasterNodeAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedTransportMasterNodeAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.master; diff --git a/server/src/main/java/org/elasticsearch/action/support/master/IsAcknowledgedSupplier.java b/server/src/main/java/org/elasticsearch/action/support/master/IsAcknowledgedSupplier.java index da33d640324ed..ff218598b72f1 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/IsAcknowledgedSupplier.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/IsAcknowledgedSupplier.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.master; diff --git a/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeOperationRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeOperationRequestBuilder.java index 3f5e89eb3b18c..cd3097de20bed 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeOperationRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeOperationRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.master; diff --git a/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeReadOperationRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeReadOperationRequestBuilder.java index 482f81004b9b8..c43d25ee7cc96 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeReadOperationRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeReadOperationRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.master; diff --git a/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeReadRequest.java b/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeReadRequest.java index 92788f53279d5..e4a635cd2b811 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeReadRequest.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeReadRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.master; diff --git a/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeRequest.java b/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeRequest.java index d547f26bd2061..10d8fe02e18a9 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeRequest.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.master; @@ -75,7 +76,7 @@ protected MasterNodeRequest(TimeValue masterNodeTimeout) { protected MasterNodeRequest(StreamInput in) throws IOException { super(in); masterNodeTimeout = in.readTimeValue(); - if (in.getTransportVersion().onOrAfter(TransportVersions.VERSIONED_MASTER_NODE_REQUESTS)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { masterTerm = in.readVLong(); } else { masterTerm = 0L; @@ -89,7 +90,7 @@ public void writeTo(StreamOutput out) throws IOException { assert masterTerm <= newMasterTerm : masterTerm + " vs " + newMasterTerm; super.writeTo(out); out.writeTimeValue(masterNodeTimeout); - if (out.getTransportVersion().onOrAfter(TransportVersions.VERSIONED_MASTER_NODE_REQUESTS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { out.writeVLong(newMasterTerm); } // else no protection against routing loops in older versions } diff --git a/server/src/main/java/org/elasticsearch/action/support/master/ShardsAcknowledgedResponse.java b/server/src/main/java/org/elasticsearch/action/support/master/ShardsAcknowledgedResponse.java index a00495605dbb5..72bf0a1a41f3e 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/ShardsAcknowledgedResponse.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/ShardsAcknowledgedResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.master; diff --git a/server/src/main/java/org/elasticsearch/action/support/master/TermOverridingMasterNodeRequest.java b/server/src/main/java/org/elasticsearch/action/support/master/TermOverridingMasterNodeRequest.java index 5fee54f3eb29d..be3e749d058c9 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/TermOverridingMasterNodeRequest.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/TermOverridingMasterNodeRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.master; diff --git a/server/src/main/java/org/elasticsearch/action/support/master/TermOverridingStreamOutput.java b/server/src/main/java/org/elasticsearch/action/support/master/TermOverridingStreamOutput.java index 45cf52bfbe208..61f7a9555d398 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/TermOverridingStreamOutput.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/TermOverridingStreamOutput.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.master; diff --git a/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java b/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java index 0cbbdb0792890..30f57f1a459ee 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.master; diff --git a/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeReadAction.java b/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeReadAction.java index 89de477ed0785..2a5fd1fab86d9 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeReadAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeReadAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.master; diff --git a/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequest.java b/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequest.java index 94ba504c8b175..943b03588d4ba 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequest.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.master.info; diff --git a/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequestBuilder.java index 3dab327ed8452..94db1c1dd6977 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.master.info; diff --git a/server/src/main/java/org/elasticsearch/action/support/master/info/TransportClusterInfoAction.java b/server/src/main/java/org/elasticsearch/action/support/master/info/TransportClusterInfoAction.java index b0ab3cddb9729..d521cf9e23b61 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/info/TransportClusterInfoAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/info/TransportClusterInfoAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.master.info; diff --git a/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodeResponse.java b/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodeResponse.java index 8c8db37a5d2cb..ab76d99c35fee 100644 --- a/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodeResponse.java +++ b/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodeResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.nodes; diff --git a/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesRequest.java b/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesRequest.java index d8628db4047e6..87aa1044f2c6b 100644 --- a/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.nodes; diff --git a/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesResponse.java b/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesResponse.java index 986496bc78f6a..eb3958654a375 100644 --- a/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.nodes; diff --git a/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesXContentResponse.java b/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesXContentResponse.java index 3b0f246d8f30e..b9fb2807a05c6 100644 --- a/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesXContentResponse.java +++ b/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesXContentResponse.java @@ -1,17 +1,17 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.nodes; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.common.collect.Iterators; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.common.xcontent.ChunkedToXContentObject; import org.elasticsearch.rest.action.RestActions; import org.elasticsearch.xcontent.ToXContent; @@ -29,11 +29,12 @@ protected BaseNodesXContentResponse(ClusterName clusterName, List @Override public final Iterator toXContentChunked(ToXContent.Params params) { - return Iterators.concat(Iterators.single((b, p) -> { - b.startObject(); - RestActions.buildNodesHeader(b, p, this); - return b.field("cluster_name", getClusterName().value()); - }), xContentChunks(params), ChunkedToXContentHelper.endObject()); + return ChunkedToXContent.builder(params) + .object( + ob -> ob.append((b, p) -> RestActions.buildNodesHeader(b, p, this)) + .field("cluster_name", getClusterName().value()) + .append(xContentChunks(params)) + ); } protected abstract Iterator xContentChunks(ToXContent.Params outerParams); diff --git a/server/src/main/java/org/elasticsearch/action/support/nodes/NodesOperationRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/nodes/NodesOperationRequestBuilder.java index 1de7d3c0d93c7..290e0bf97aec9 100644 --- a/server/src/main/java/org/elasticsearch/action/support/nodes/NodesOperationRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/support/nodes/NodesOperationRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.nodes; diff --git a/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java b/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java index 347edd0916fc5..89b7ec01c0406 100644 --- a/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.nodes; @@ -29,6 +30,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Releasables; import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.tasks.CancellableTask; @@ -52,7 +54,8 @@ public abstract class TransportNodesAction< NodesRequest extends BaseNodesRequest, NodesResponse extends BaseNodesResponse, NodeRequest extends TransportRequest, - NodeResponse extends BaseNodeResponse> extends TransportAction { + NodeResponse extends BaseNodeResponse, + ActionContext> extends TransportAction { private static final Logger logger = LogManager.getLogger(TransportNodesAction.class); @@ -98,6 +101,7 @@ protected void doExecute(Task task, NodesRequest request, ActionListener, Exception>>() { + final ActionContext actionContext = createActionContext(task, request); final ArrayList responses = new ArrayList<>(concreteNodes.length); final ArrayList exceptions = new ArrayList<>(0); @@ -165,7 +169,7 @@ protected CheckedConsumer, Exception> onCompletion // ref releases all happen-before here so no need to be synchronized return l -> { try (var ignored = Releasables.wrap(Iterators.map(responses.iterator(), r -> r::decRef))) { - newResponseAsync(task, request, responses, exceptions, l); + newResponseAsync(task, request, actionContext, responses, exceptions, l); } }; } @@ -186,6 +190,16 @@ private Writeable.Reader nodeResponseReader(DiscoveryNode discover return in -> TransportNodesAction.this.newNodeResponse(in, discoveryNode); } + /** + * Create an (optional) {@link ActionContext}: called when starting to execute this action, and the result passed to + * {@link #newResponseAsync} on completion. NB runs on the transport worker thread, must not do anything expensive without dispatching + * to a different executor. + */ + @Nullable + protected ActionContext createActionContext(Task task, NodesRequest request) { + return null; + } + /** * Create a new {@link NodesResponse}. This method is executed on {@link #finalExecutor}. * @@ -210,6 +224,7 @@ private Writeable.Reader nodeResponseReader(DiscoveryNode discover protected void newResponseAsync( Task task, NodesRequest request, + ActionContext actionContext, List responses, List failures, ActionListener listener diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/BasicReplicationRequest.java b/server/src/main/java/org/elasticsearch/action/support/replication/BasicReplicationRequest.java index 2e3b733f1c0c6..8d04a31101d0c 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/BasicReplicationRequest.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/BasicReplicationRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.replication; diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/PendingReplicationActions.java b/server/src/main/java/org/elasticsearch/action/support/replication/PendingReplicationActions.java index d26ec436af540..75cea50016362 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/PendingReplicationActions.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/PendingReplicationActions.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.replication; diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/PostWriteRefresh.java b/server/src/main/java/org/elasticsearch/action/support/replication/PostWriteRefresh.java index 67793fb525644..7414aeeb2c405 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/PostWriteRefresh.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/PostWriteRefresh.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.replication; @@ -18,7 +19,6 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.translog.Translog; @@ -52,9 +52,7 @@ public void refreshShard( case WAIT_UNTIL -> waitUntil(indexShard, location, new ActionListener<>() { @Override public void onResponse(Boolean forced) { - // Fast refresh indices do not depend on the unpromotables being refreshed - boolean fastRefresh = IndexSettings.INDEX_FAST_REFRESH_SETTING.get(indexShard.indexSettings().getSettings()); - if (location != null && (indexShard.routingEntry().isSearchable() == false && fastRefresh == false)) { + if (location != null && indexShard.routingEntry().isSearchable() == false) { refreshUnpromotables(indexShard, location, listener, forced, postWriteRefreshTimeout); } else { listener.onResponse(forced); @@ -67,9 +65,7 @@ public void onFailure(Exception e) { } }); case IMMEDIATE -> immediate(indexShard, listener.delegateFailureAndWrap((l, r) -> { - // Fast refresh indices do not depend on the unpromotables being refreshed - boolean fastRefresh = IndexSettings.INDEX_FAST_REFRESH_SETTING.get(indexShard.indexSettings().getSettings()); - if (indexShard.getReplicationGroup().getRoutingTable().unpromotableShards().size() > 0 && fastRefresh == false) { + if (indexShard.getReplicationGroup().getRoutingTable().unpromotableShards().size() > 0) { sendUnpromotableRequests(indexShard, r.generation(), true, l, postWriteRefreshTimeout); } else { l.onResponse(true); diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicatedWriteRequest.java b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicatedWriteRequest.java index e90d71c26cf52..94f4275614dae 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicatedWriteRequest.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicatedWriteRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.replication; diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java index b38a067e8b316..5c4195bc7b069 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.replication; diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java index 8d388e7c6d4d6..debc64914a171 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.replication; @@ -209,7 +210,12 @@ public void writeThin(StreamOutput out) throws IOException { @Override public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { - return new ReplicationTask(id, type, action, getDescription(), parentTaskId, headers); + return new ReplicationTask(id, type, action, "", parentTaskId, headers) { + @Override + public String getDescription() { + return ReplicationRequest.this.getDescription(); + } + }; } @Override diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequestBuilder.java index a918618fc78ec..30a3ff5b9e994 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.replication; diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java index 06e9018c8983d..3f207ff43f86a 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.replication; diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationTask.java b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationTask.java index 019c34f32661f..a5afc73a9b7db 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationTask.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.replication; diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java index 602d5dd620c90..dfe0d5fc8dea9 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.replication; diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java index 7e3e5bdee206d..dc921bb0d1111 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.replication; diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java index 90b636ed69e24..8372302091c9d 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.replication; diff --git a/server/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequest.java b/server/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequest.java index 51952059d7d94..fd95f3079f6c9 100644 --- a/server/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequest.java +++ b/server/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.single.instance; diff --git a/server/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequestBuilder.java index 25278c71e1fe2..1379dfd915b86 100644 --- a/server/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.single.instance; diff --git a/server/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java b/server/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java index fb62f0a2b1d61..c0727bc70b0af 100644 --- a/server/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.single.instance; diff --git a/server/src/main/java/org/elasticsearch/action/support/single/shard/SingleShardOperationRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/single/shard/SingleShardOperationRequestBuilder.java index 98f58f13b8bea..d01120a182ed8 100644 --- a/server/src/main/java/org/elasticsearch/action/support/single/shard/SingleShardOperationRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/support/single/shard/SingleShardOperationRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.single.shard; diff --git a/server/src/main/java/org/elasticsearch/action/support/single/shard/SingleShardRequest.java b/server/src/main/java/org/elasticsearch/action/support/single/shard/SingleShardRequest.java index 950102a5debca..cb410d8bf19f6 100644 --- a/server/src/main/java/org/elasticsearch/action/support/single/shard/SingleShardRequest.java +++ b/server/src/main/java/org/elasticsearch/action/support/single/shard/SingleShardRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.single.shard; diff --git a/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java b/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java index 180aa3b336149..1f9b1b4a95cab 100644 --- a/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.single.shard; @@ -36,7 +37,6 @@ import org.elasticsearch.transport.TransportChannel; import org.elasticsearch.transport.TransportException; import org.elasticsearch.transport.TransportRequestHandler; -import org.elasticsearch.transport.TransportResponseHandler; import org.elasticsearch.transport.TransportService; import java.io.IOException; @@ -182,7 +182,7 @@ public void start() { clusterService.localNode(), transportShardAction, internalRequest.request(), - new ActionListenerResponseHandler<>(listener, reader, TransportResponseHandler.TRANSPORT_WORKER) + new ActionListenerResponseHandler<>(listener, reader, executor) ); } else { perform(null); @@ -235,7 +235,7 @@ private void perform(@Nullable final Exception currentFailure) { node, transportShardAction, internalRequest.request(), - new ActionListenerResponseHandler<>(listener, reader, TransportResponseHandler.TRANSPORT_WORKER) { + new ActionListenerResponseHandler<>(listener, reader, executor) { @Override public void handleException(TransportException exp) { onFailure(shardRouting, exp); diff --git a/server/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksRequest.java b/server/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksRequest.java index 56f2f1f16ed1b..ba348f8b62b79 100644 --- a/server/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksRequest.java +++ b/server/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.tasks; diff --git a/server/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksResponse.java b/server/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksResponse.java index 0aa565c87b4cd..63c3445bf1487 100644 --- a/server/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksResponse.java +++ b/server/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.tasks; diff --git a/server/src/main/java/org/elasticsearch/action/support/tasks/TasksRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/tasks/TasksRequestBuilder.java index aa542788c6d30..16676106e3228 100644 --- a/server/src/main/java/org/elasticsearch/action/support/tasks/TasksRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/support/tasks/TasksRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.tasks; diff --git a/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java b/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java index b367d33adb908..fb4ce6eeb493c 100644 --- a/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.support.tasks; diff --git a/server/src/main/java/org/elasticsearch/action/synonyms/AbstractSynonymsPagedResultAction.java b/server/src/main/java/org/elasticsearch/action/synonyms/AbstractSynonymsPagedResultAction.java index e234fd35eebc2..c1e2efb0834e3 100644 --- a/server/src/main/java/org/elasticsearch/action/synonyms/AbstractSynonymsPagedResultAction.java +++ b/server/src/main/java/org/elasticsearch/action/synonyms/AbstractSynonymsPagedResultAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.synonyms; diff --git a/server/src/main/java/org/elasticsearch/action/synonyms/DeleteSynonymRuleAction.java b/server/src/main/java/org/elasticsearch/action/synonyms/DeleteSynonymRuleAction.java index cc98da3648709..9f52f1552e421 100644 --- a/server/src/main/java/org/elasticsearch/action/synonyms/DeleteSynonymRuleAction.java +++ b/server/src/main/java/org/elasticsearch/action/synonyms/DeleteSynonymRuleAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.synonyms; diff --git a/server/src/main/java/org/elasticsearch/action/synonyms/DeleteSynonymsAction.java b/server/src/main/java/org/elasticsearch/action/synonyms/DeleteSynonymsAction.java index 52bbb40eb52f8..4901ea518c569 100644 --- a/server/src/main/java/org/elasticsearch/action/synonyms/DeleteSynonymsAction.java +++ b/server/src/main/java/org/elasticsearch/action/synonyms/DeleteSynonymsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.synonyms; diff --git a/server/src/main/java/org/elasticsearch/action/synonyms/GetSynonymRuleAction.java b/server/src/main/java/org/elasticsearch/action/synonyms/GetSynonymRuleAction.java index 6a1c3e17a70ad..dd3239372c5bf 100644 --- a/server/src/main/java/org/elasticsearch/action/synonyms/GetSynonymRuleAction.java +++ b/server/src/main/java/org/elasticsearch/action/synonyms/GetSynonymRuleAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.synonyms; diff --git a/server/src/main/java/org/elasticsearch/action/synonyms/GetSynonymsAction.java b/server/src/main/java/org/elasticsearch/action/synonyms/GetSynonymsAction.java index e328cda060470..335ab0566572b 100644 --- a/server/src/main/java/org/elasticsearch/action/synonyms/GetSynonymsAction.java +++ b/server/src/main/java/org/elasticsearch/action/synonyms/GetSynonymsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.synonyms; diff --git a/server/src/main/java/org/elasticsearch/action/synonyms/GetSynonymsSetsAction.java b/server/src/main/java/org/elasticsearch/action/synonyms/GetSynonymsSetsAction.java index 6be73c94e32ac..6d74127114eb7 100644 --- a/server/src/main/java/org/elasticsearch/action/synonyms/GetSynonymsSetsAction.java +++ b/server/src/main/java/org/elasticsearch/action/synonyms/GetSynonymsSetsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.synonyms; diff --git a/server/src/main/java/org/elasticsearch/action/synonyms/PutSynonymRuleAction.java b/server/src/main/java/org/elasticsearch/action/synonyms/PutSynonymRuleAction.java index 66d4b59755d28..6d0d24dbfee05 100644 --- a/server/src/main/java/org/elasticsearch/action/synonyms/PutSynonymRuleAction.java +++ b/server/src/main/java/org/elasticsearch/action/synonyms/PutSynonymRuleAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.synonyms; diff --git a/server/src/main/java/org/elasticsearch/action/synonyms/PutSynonymsAction.java b/server/src/main/java/org/elasticsearch/action/synonyms/PutSynonymsAction.java index 6bb6f484e54d3..5e806ef2ef548 100644 --- a/server/src/main/java/org/elasticsearch/action/synonyms/PutSynonymsAction.java +++ b/server/src/main/java/org/elasticsearch/action/synonyms/PutSynonymsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.synonyms; diff --git a/server/src/main/java/org/elasticsearch/action/synonyms/SynonymUpdateResponse.java b/server/src/main/java/org/elasticsearch/action/synonyms/SynonymUpdateResponse.java index 7bbddfca79075..8f40b48d2002e 100644 --- a/server/src/main/java/org/elasticsearch/action/synonyms/SynonymUpdateResponse.java +++ b/server/src/main/java/org/elasticsearch/action/synonyms/SynonymUpdateResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.synonyms; diff --git a/server/src/main/java/org/elasticsearch/action/synonyms/TransportDeleteSynonymRuleAction.java b/server/src/main/java/org/elasticsearch/action/synonyms/TransportDeleteSynonymRuleAction.java index f0c4e17abb382..0d55774489c4c 100644 --- a/server/src/main/java/org/elasticsearch/action/synonyms/TransportDeleteSynonymRuleAction.java +++ b/server/src/main/java/org/elasticsearch/action/synonyms/TransportDeleteSynonymRuleAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.synonyms; diff --git a/server/src/main/java/org/elasticsearch/action/synonyms/TransportDeleteSynonymsAction.java b/server/src/main/java/org/elasticsearch/action/synonyms/TransportDeleteSynonymsAction.java index c22bb308f42de..a6dc43cb55064 100644 --- a/server/src/main/java/org/elasticsearch/action/synonyms/TransportDeleteSynonymsAction.java +++ b/server/src/main/java/org/elasticsearch/action/synonyms/TransportDeleteSynonymsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.synonyms; diff --git a/server/src/main/java/org/elasticsearch/action/synonyms/TransportGetSynonymRuleAction.java b/server/src/main/java/org/elasticsearch/action/synonyms/TransportGetSynonymRuleAction.java index 5a574def93e0d..44e029c448f81 100644 --- a/server/src/main/java/org/elasticsearch/action/synonyms/TransportGetSynonymRuleAction.java +++ b/server/src/main/java/org/elasticsearch/action/synonyms/TransportGetSynonymRuleAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.synonyms; diff --git a/server/src/main/java/org/elasticsearch/action/synonyms/TransportGetSynonymsAction.java b/server/src/main/java/org/elasticsearch/action/synonyms/TransportGetSynonymsAction.java index 09d087e85e9e2..a9d361cb6cf6e 100644 --- a/server/src/main/java/org/elasticsearch/action/synonyms/TransportGetSynonymsAction.java +++ b/server/src/main/java/org/elasticsearch/action/synonyms/TransportGetSynonymsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.synonyms; diff --git a/server/src/main/java/org/elasticsearch/action/synonyms/TransportGetSynonymsSetsAction.java b/server/src/main/java/org/elasticsearch/action/synonyms/TransportGetSynonymsSetsAction.java index 13177f475354d..73307b7b93487 100644 --- a/server/src/main/java/org/elasticsearch/action/synonyms/TransportGetSynonymsSetsAction.java +++ b/server/src/main/java/org/elasticsearch/action/synonyms/TransportGetSynonymsSetsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.synonyms; diff --git a/server/src/main/java/org/elasticsearch/action/synonyms/TransportPutSynonymRuleAction.java b/server/src/main/java/org/elasticsearch/action/synonyms/TransportPutSynonymRuleAction.java index eb0b87ae0211a..33baa7b9081f2 100644 --- a/server/src/main/java/org/elasticsearch/action/synonyms/TransportPutSynonymRuleAction.java +++ b/server/src/main/java/org/elasticsearch/action/synonyms/TransportPutSynonymRuleAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.synonyms; diff --git a/server/src/main/java/org/elasticsearch/action/synonyms/TransportPutSynonymsAction.java b/server/src/main/java/org/elasticsearch/action/synonyms/TransportPutSynonymsAction.java index 4f5589bd85ed7..b0095ebee6a05 100644 --- a/server/src/main/java/org/elasticsearch/action/synonyms/TransportPutSynonymsAction.java +++ b/server/src/main/java/org/elasticsearch/action/synonyms/TransportPutSynonymsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.synonyms; diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsAction.java b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsAction.java index 0f5917cdcb259..ce76558324621 100644 --- a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsAction.java +++ b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.termvectors; diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsItemResponse.java b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsItemResponse.java index 931354b6c508a..b9aabd0094f16 100644 --- a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsItemResponse.java +++ b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsItemResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.termvectors; diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsRequest.java b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsRequest.java index 505f907ad6548..d5b5ef77db92a 100644 --- a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.termvectors; diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsRequestBuilder.java index aa3346851917e..54fe4f5bb56d0 100644 --- a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.termvectors; diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java index 5789c4910db09..2beeaf1a26f0f 100644 --- a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.termvectors; diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsShardRequest.java b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsShardRequest.java index 3fce5deb73f62..0ff5dcec7f8bf 100644 --- a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsShardRequest.java +++ b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsShardRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.termvectors; diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsShardResponse.java b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsShardResponse.java index 1f79223695de5..85a46f66961ad 100644 --- a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsShardResponse.java +++ b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsShardResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.termvectors; diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsAction.java b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsAction.java index 492673c18245a..102b3850e6546 100644 --- a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsAction.java +++ b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.termvectors; diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsFields.java b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsFields.java index 805bd055c97d7..1940c6007a5ab 100644 --- a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsFields.java +++ b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsFields.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.termvectors; diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsFilter.java b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsFilter.java index 56f6390caaf42..6bc7a3c34be6e 100644 --- a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsFilter.java +++ b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsFilter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.termvectors; diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java index 284831ef18060..a36158d11b5b3 100644 --- a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.termvectors; diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequestBuilder.java index 308f87e07d1e2..b707457a9833c 100644 --- a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.termvectors; diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java index d54c593e4579f..2d70af94d53d1 100644 --- a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.termvectors; diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsWriter.java b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsWriter.java index 1a676bf30a24d..2ce533297dd6a 100644 --- a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsWriter.java +++ b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsWriter.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.termvectors; diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java b/server/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java index b4095d04baff2..b3230fae9834a 100644 --- a/server/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java +++ b/server/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.termvectors; @@ -16,6 +17,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.routing.OperationRouting; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.common.util.concurrent.EsExecutors; @@ -71,8 +73,12 @@ protected void doExecute(Task task, final MultiTermVectorsRequest request, final clusterState.metadata().resolveIndexRouting(termVectorsRequest.routing(), termVectorsRequest.index()) ); String concreteSingleIndex = indexNameExpressionResolver.concreteSingleIndex(clusterState, termVectorsRequest).getName(); - shardId = clusterService.operationRouting() - .shardId(clusterState, concreteSingleIndex, termVectorsRequest.id(), termVectorsRequest.routing()); + shardId = OperationRouting.shardId( + clusterState, + concreteSingleIndex, + termVectorsRequest.id(), + termVectorsRequest.routing() + ); } catch (RoutingMissingException e) { responses.set( i, diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java b/server/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java index 84f6d433af882..7840af1baad3a 100644 --- a/server/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java +++ b/server/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.termvectors; diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java b/server/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java index 75bbbf96136a9..02479a9f8d143 100644 --- a/server/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java +++ b/server/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.termvectors; @@ -61,16 +62,27 @@ public TransportTermVectorsAction( @Override protected ShardIterator shards(ClusterState state, InternalRequest request) { + final var operationRouting = clusterService.operationRouting(); if (request.request().doc() != null && request.request().routing() == null) { // artificial document without routing specified, ignore its "id" and use either random shard or according to preference - GroupShardsIterator groupShardsIter = clusterService.operationRouting() - .searchShards(state, new String[] { request.concreteIndex() }, null, request.request().preference()); + GroupShardsIterator groupShardsIter = operationRouting.searchShards( + state, + new String[] { request.concreteIndex() }, + null, + request.request().preference() + ); return groupShardsIter.iterator().next(); } - ShardIterator shards = clusterService.operationRouting() - .getShards(state, request.concreteIndex(), request.request().id(), request.request().routing(), request.request().preference()); - return clusterService.operationRouting().useOnlyPromotableShardsForStateless(shards); + return operationRouting.useOnlyPromotableShardsForStateless( + operationRouting.getShards( + state, + request.concreteIndex(), + request.request().id(), + request.request().routing(), + request.request().preference() + ) + ); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/package-info.java b/server/src/main/java/org/elasticsearch/action/termvectors/package-info.java index 1cbde725032e1..dc1ef29ce9be4 100644 --- a/server/src/main/java/org/elasticsearch/action/termvectors/package-info.java +++ b/server/src/main/java/org/elasticsearch/action/termvectors/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java b/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java index 00d33ec9d2eca..ee84bcd15824d 100644 --- a/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java +++ b/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.update; @@ -43,6 +44,7 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.mapper.InferenceFieldMapper; +import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.shard.IndexShard; @@ -186,11 +188,12 @@ protected void shardOperation(final UpdateRequest request, final ActionListener< final ShardId shardId = request.getShardId(); final IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); final IndexShard indexShard = indexService.getShard(shardId.getId()); + final MappingLookup mappingLookup = indexShard.mapperService().mappingLookup(); final UpdateHelper.Result result = deleteInferenceResults( request, updateHelper.prepare(request, indexShard, threadPool::absoluteTimeInMillis), indexService.getMetadata(), - indexShard.mapperService().mappingLookup() + mappingLookup ); switch (result.getResponseResult()) { @@ -220,6 +223,7 @@ protected void shardOperation(final UpdateRequest request, final ActionListener< UpdateHelper.extractGetResult( request, request.concreteIndex(), + mappingLookup, response.getSeqNo(), response.getPrimaryTerm(), response.getVersion(), @@ -256,6 +260,7 @@ protected void shardOperation(final UpdateRequest request, final ActionListener< UpdateHelper.extractGetResult( request, request.concreteIndex(), + mappingLookup, response.getSeqNo(), response.getPrimaryTerm(), response.getVersion(), @@ -287,6 +292,7 @@ protected void shardOperation(final UpdateRequest request, final ActionListener< UpdateHelper.extractGetResult( request, request.concreteIndex(), + mappingLookup, response.getSeqNo(), response.getPrimaryTerm(), response.getVersion(), @@ -369,7 +375,7 @@ private static UpdateHelper.Result deleteInferenceResults( IndexMetadata indexMetadata, MappingLookup mappingLookup ) { - if (result.getResponseResult() != DocWriteResponse.Result.UPDATED) { + if (result.getResponseResult() != DocWriteResponse.Result.UPDATED || InferenceMetadataFieldsMapper.isEnabled(mappingLookup)) { return result; } @@ -398,7 +404,7 @@ private static UpdateHelper.Result deleteInferenceResults( String inferenceFieldName = entry.getKey(); Mapper mapper = mappingLookup.getMapper(inferenceFieldName); - if (mapper instanceof InferenceFieldMapper inferenceFieldMapper) { + if (mapper instanceof InferenceFieldMapper) { String[] sourceFields = entry.getValue().getSourceFields(); for (String sourceField : sourceFields) { if (sourceField.equals(inferenceFieldName) == false @@ -407,7 +413,7 @@ private static UpdateHelper.Result deleteInferenceResults( // This has two important side effects: // - The inference field value will remain parsable by its mapper // - The inference results will be removed, forcing them to be re-generated downstream - updatedSource.put(inferenceFieldName, inferenceFieldMapper.getOriginalValue(updatedSource)); + updatedSource.put(inferenceFieldName, getOriginalValueLegacy(inferenceFieldName, updatedSource)); updatedSourceModified = true; break; } @@ -430,4 +436,24 @@ private static UpdateHelper.Result deleteInferenceResults( return returnedResult; } + + /** + * Get the field's original value (i.e. the value the user specified) from the provided source. + * + * @param sourceAsMap The source as a map + * @return The field's original value, or {@code null} if none was provided + */ + private static Object getOriginalValueLegacy(String fullPath, Map sourceAsMap) { + // TODO: Fix bug here when semantic text field is in an object + Object fieldValue = sourceAsMap.get(fullPath); + if (fieldValue == null) { + return null; + } else if (fieldValue instanceof Map == false) { + // Don't try to further validate the non-map value, that will be handled when the source is fully parsed + return fieldValue; + } + + Map fieldValueMap = XContentMapValues.nodeMapValue(fieldValue, "Field [" + fullPath + "]"); + return XContentMapValues.extractValue("text", fieldValueMap); + } } diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java b/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java index 056eb70c53269..aff1db629ab6d 100644 --- a/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java +++ b/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.update; @@ -23,6 +24,7 @@ import org.elasticsearch.index.engine.DocumentSourceMissingException; import org.elasticsearch.index.get.GetResult; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.RoutingFieldMapper; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; @@ -34,6 +36,7 @@ import org.elasticsearch.script.UpdateScript; import org.elasticsearch.script.UpsertCtxMap; import org.elasticsearch.search.lookup.Source; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.xcontent.XContentType; import java.io.IOException; @@ -60,27 +63,35 @@ public UpdateHelper(ScriptService scriptService, DocumentParsingProvider documen * Prepares an update request by converting it into an index or delete request or an update response (no action). */ public Result prepare(UpdateRequest request, IndexShard indexShard, LongSupplier nowInMillis) throws IOException { - final GetResult getResult = indexShard.getService().getForUpdate(request.id(), request.ifSeqNo(), request.ifPrimaryTerm()); - return prepare(indexShard.shardId(), request, getResult, nowInMillis); + // TODO: Don't hard-code gFields + return prepare(request, indexShard, nowInMillis, new String[] { RoutingFieldMapper.NAME }); + } + + /** + * Prepares an update request by converting it into an index or delete request or an update response (no action). + */ + public Result prepare(UpdateRequest request, IndexShard indexShard, LongSupplier nowInMillis, String[] gFields) throws IOException { + final GetResult getResult = indexShard.getService().getForUpdate(request.id(), request.ifSeqNo(), request.ifPrimaryTerm(), gFields); + return prepare(indexShard, request, getResult, nowInMillis); } /** * Prepares an update request by converting it into an index or delete request or an update response (no action, in the event of a * noop). */ - protected Result prepare(ShardId shardId, UpdateRequest request, final GetResult getResult, LongSupplier nowInMillis) { + protected Result prepare(IndexShard indexShard, UpdateRequest request, final GetResult getResult, LongSupplier nowInMillis) { if (getResult.isExists() == false) { // If the document didn't exist, execute the update request as an upsert - return prepareUpsert(shardId, request, getResult, nowInMillis); + return prepareUpsert(indexShard.shardId(), request, getResult, nowInMillis); } else if (getResult.internalSourceRef() == null) { // no source, we can't do anything, throw a failure... - throw new DocumentSourceMissingException(shardId, request.id()); + throw new DocumentSourceMissingException(indexShard.shardId(), request.id()); } else if (request.script() == null && request.doc() != null) { // The request has no script, it is a new doc that should be merged with the old document - return prepareUpdateIndexRequest(shardId, request, getResult, request.detectNoop()); + return prepareUpdateIndexRequest(indexShard, request, getResult, request.detectNoop()); } else { // The request has a script (or empty script), execute the script and prepare a new index request - return prepareUpdateScriptRequest(shardId, request, getResult, nowInMillis); + return prepareUpdateScriptRequest(indexShard, request, getResult, nowInMillis); } } @@ -178,7 +189,7 @@ static String calculateRouting(GetResult getResult, @Nullable IndexRequest updat * Prepare the request for merging the existing document with a new one, can optionally detect a noop change. Returns a {@code Result} * containing a new {@code IndexRequest} to be executed on the primary and replicas. */ - Result prepareUpdateIndexRequest(ShardId shardId, UpdateRequest request, GetResult getResult, boolean detectNoop) { + Result prepareUpdateIndexRequest(IndexShard indexShard, UpdateRequest request, GetResult getResult, boolean detectNoop) { final IndexRequest currentRequest = request.doc(); final String routing = calculateRouting(getResult, currentRequest); final XContentMeteringParserDecorator meteringParserDecorator = documentParsingProvider.newMeteringParserDecorator(request); @@ -196,7 +207,7 @@ Result prepareUpdateIndexRequest(ShardId shardId, UpdateRequest request, GetResu // where users repopulating multi-fields or adding synonyms, etc. if (detectNoop && noop) { UpdateResponse update = new UpdateResponse( - shardId, + indexShard.shardId(), getResult.getId(), getResult.getSeqNo(), getResult.getPrimaryTerm(), @@ -207,6 +218,7 @@ Result prepareUpdateIndexRequest(ShardId shardId, UpdateRequest request, GetResu extractGetResult( request, request.index(), + indexShard.mapperService().mappingLookup(), getResult.getSeqNo(), getResult.getPrimaryTerm(), getResult.getVersion(), @@ -225,9 +237,7 @@ Result prepareUpdateIndexRequest(ShardId shardId, UpdateRequest request, GetResu .setIfPrimaryTerm(getResult.getPrimaryTerm()) .waitForActiveShards(request.waitForActiveShards()) .timeout(request.timeout()) - .setRefreshPolicy(request.getRefreshPolicy()) - .setOriginatesFromUpdateByDoc(true); - finalIndexRequest.setNormalisedBytesParsed(meteringParserDecorator.meteredDocumentSize().ingestedBytes()); + .setRefreshPolicy(request.getRefreshPolicy()); return new Result(finalIndexRequest, DocWriteResponse.Result.UPDATED, updatedSourceAsMap, updateSourceContentType); } } @@ -237,7 +247,7 @@ Result prepareUpdateIndexRequest(ShardId shardId, UpdateRequest request, GetResu * either a new {@code IndexRequest} or {@code DeleteRequest} (depending on the script's returned "op" value) to be executed on the * primary and replicas. */ - Result prepareUpdateScriptRequest(ShardId shardId, UpdateRequest request, GetResult getResult, LongSupplier nowInMillis) { + Result prepareUpdateScriptRequest(IndexShard indexShard, UpdateRequest request, GetResult getResult, LongSupplier nowInMillis) { final IndexRequest currentRequest = request.doc(); final String routing = calculateRouting(getResult, currentRequest); final Tuple> sourceAndContent = XContentHelper.convertToMap(getResult.internalSourceRef(), true); @@ -269,8 +279,7 @@ Result prepareUpdateScriptRequest(ShardId shardId, UpdateRequest request, GetRes .setIfPrimaryTerm(getResult.getPrimaryTerm()) .waitForActiveShards(request.waitForActiveShards()) .timeout(request.timeout()) - .setRefreshPolicy(request.getRefreshPolicy()) - .setOriginatesFromUpdateByScript(true); + .setRefreshPolicy(request.getRefreshPolicy()); return new Result(indexRequest, DocWriteResponse.Result.UPDATED, updatedSourceAsMap, updateSourceContentType); } case DELETE -> { @@ -287,7 +296,7 @@ Result prepareUpdateScriptRequest(ShardId shardId, UpdateRequest request, GetRes default -> { // If it was neither an INDEX or DELETE operation, treat it as a noop UpdateResponse update = new UpdateResponse( - shardId, + indexShard.shardId(), getResult.getId(), getResult.getSeqNo(), getResult.getPrimaryTerm(), @@ -298,6 +307,7 @@ Result prepareUpdateScriptRequest(ShardId shardId, UpdateRequest request, GetRes extractGetResult( request, request.index(), + indexShard.mapperService().mappingLookup(), getResult.getSeqNo(), getResult.getPrimaryTerm(), getResult.getVersion(), @@ -331,6 +341,7 @@ private T executeScript(Script script, T ctxMap) { public static GetResult extractGetResult( final UpdateRequest request, String concreteIndex, + final MappingLookup mappingLookup, long seqNo, long primaryTerm, long version, @@ -342,8 +353,9 @@ public static GetResult extractGetResult( return null; } BytesReference sourceFilteredAsBytes = sourceAsBytes; - if (request.fetchSource().hasFilter()) { - sourceFilteredAsBytes = Source.fromMap(source, sourceContentType).filter(request.fetchSource().filter()).internalSourceRef(); + SourceFilter sourceFilter = request.fetchSource().filter(); + if (sourceFilter != null) { + sourceFilteredAsBytes = Source.fromMap(source, sourceContentType).filter(sourceFilter).internalSourceRef(); } // TODO when using delete/none, we can still return the source as bytes by generating it (using the sourceContentType) diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java index 211daf2369d99..657ad029626af 100644 --- a/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.update; @@ -682,11 +683,6 @@ public boolean isRequireDataStream() { return false; } - @Override - public void process(IndexRouting indexRouting) { - // Nothing to do - } - @Override public int route(IndexRouting indexRouting) { return indexRouting.updateShard(id, routing); diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java index 587ed2ef75eba..ae1335008f64a 100644 --- a/server/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.update; diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateResponse.java b/server/src/main/java/org/elasticsearch/action/update/UpdateResponse.java index 8c9ae3f43d5c4..4d696da6d151f 100644 --- a/server/src/main/java/org/elasticsearch/action/update/UpdateResponse.java +++ b/server/src/main/java/org/elasticsearch/action/update/UpdateResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.action.update; diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java index a9ef2e68a6762..4c7fb96c5b1d5 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.bootstrap; @@ -16,6 +17,7 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.env.Environment; import org.elasticsearch.node.NodeValidationException; +import org.elasticsearch.plugins.PluginsLoader; import java.io.PrintStream; @@ -31,6 +33,7 @@ class Bootstrap { // arguments from the CLI process private final ServerArgs args; + private final boolean useEntitlements; // controller for spawning component subprocesses private final Spawner spawner = new Spawner(); @@ -41,10 +44,14 @@ class Bootstrap { // the loaded settings for the node, not valid until after phase 2 of initialization private final SetOnce nodeEnv = new SetOnce<>(); - Bootstrap(PrintStream out, PrintStream err, ServerArgs args) { + // loads information about plugins required for entitlements in phase 2, used by plugins service in phase 3 + private final SetOnce pluginsLoader = new SetOnce<>(); + + Bootstrap(PrintStream out, PrintStream err, ServerArgs args, boolean useEntitlements) { this.out = out; this.err = err; this.args = args; + this.useEntitlements = useEntitlements; } ServerArgs args() { @@ -55,6 +62,10 @@ Spawner spawner() { return spawner; } + public boolean useEntitlements() { + return useEntitlements; + } + void setSecureSettings(SecureSettings secureSettings) { this.secureSettings.set(secureSettings); } @@ -71,6 +82,14 @@ Environment environment() { return nodeEnv.get(); } + void setPluginsLoader(PluginsLoader pluginsLoader) { + this.pluginsLoader.set(pluginsLoader); + } + + PluginsLoader pluginsLoader() { + return pluginsLoader.get(); + } + void exitWithNodeValidationException(NodeValidationException e) { Logger logger = LogManager.getLogger(Elasticsearch.class); logger.error("node validation exception\n{}", e.getMessage()); diff --git a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java index 6a32959b7e5f7..e2644b41519c1 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.bootstrap; diff --git a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java index 84811362c08e6..0adfdc5d3a7ae 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.bootstrap; @@ -210,7 +211,6 @@ static List checks() { checks.add(new OnErrorCheck()); checks.add(new OnOutOfMemoryErrorCheck()); checks.add(new EarlyAccessCheck()); - checks.add(new AllPermissionCheck()); checks.add(new DiscoveryConfiguredCheck()); checks.add(new ByteOrderCheck()); return Collections.unmodifiableList(checks); @@ -411,12 +411,12 @@ static class MaxFileSizeCheck implements BootstrapCheck { @Override public BootstrapCheckResult check(BootstrapContext context) { - final long maxFileSize = getMaxFileSize(); + final long maxFileSize = getProcessLimits().maxFileSize(); if (maxFileSize != Long.MIN_VALUE && maxFileSize != ProcessLimits.UNLIMITED) { final String message = String.format( Locale.ROOT, "max file size [%d] for user [%s] is too low, increase to [unlimited]", - getMaxFileSize(), + maxFileSize, BootstrapInfo.getSystemProperties().get("user.name") ); return BootstrapCheckResult.failure(message); @@ -425,8 +425,8 @@ public BootstrapCheckResult check(BootstrapContext context) { } } - long getMaxFileSize() { - return NativeAccess.instance().getProcessLimits().maxVirtualMemorySize(); + protected ProcessLimits getProcessLimits() { + return NativeAccess.instance().getProcessLimits(); } @Override diff --git a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapContext.java b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapContext.java index 3b59164bc7313..9256225734118 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapContext.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapContext.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.bootstrap; diff --git a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapException.java b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapException.java index ec51981e2a076..98c73d6a07fd6 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapException.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.bootstrap; diff --git a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java index 005375bf38540..ffcbc839d8ac9 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.bootstrap; diff --git a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapSettings.java b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapSettings.java index 3569a8ee37537..fc644269634b1 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapSettings.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapSettings.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.bootstrap; diff --git a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapUtil.java b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapUtil.java index 7ff0bfeb4c76d..8f1537d917c15 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapUtil.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapUtil.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.bootstrap; diff --git a/server/src/main/java/org/elasticsearch/bootstrap/ConsoleCtrlHandler.java b/server/src/main/java/org/elasticsearch/bootstrap/ConsoleCtrlHandler.java index f17e6651f274f..1f40557190c38 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/ConsoleCtrlHandler.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/ConsoleCtrlHandler.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.bootstrap; diff --git a/server/src/main/java/org/elasticsearch/bootstrap/ConsoleLoader.java b/server/src/main/java/org/elasticsearch/bootstrap/ConsoleLoader.java index 955e4439d1f74..7b85b369b5dd5 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/ConsoleLoader.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/ConsoleLoader.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.bootstrap; diff --git a/server/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java b/server/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java index dbc94ad7812a7..845303abe6baf 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.bootstrap; @@ -15,6 +16,7 @@ import java.io.IOException; import java.net.SocketPermission; import java.net.URL; +import java.security.AllPermission; import java.security.CodeSource; import java.security.Permission; import java.security.PermissionCollection; @@ -38,6 +40,7 @@ final class ESPolicy extends Policy { static final String UNTRUSTED_RESOURCE = "untrusted.policy"; private static final String ALL_FILE_MASK = "read,readlink,write,delete,execute"; + private static final AllPermission ALL_PERMISSION = new AllPermission(); final Policy template; final Policy untrusted; @@ -123,7 +126,7 @@ public boolean implies(ProtectionDomain domain, Permission permission) { * It's helpful to use the infrastructure around FilePermission here to do the directory structure check with implies * so we use ALL_FILE_MASK mask to check if we can do something with this file, whatever the actual operation we're requesting */ - return canAccessSecuredFile(location, new FilePermission(permission.getName(), ALL_FILE_MASK)); + return canAccessSecuredFile(domain, new FilePermission(permission.getName(), ALL_FILE_MASK)); } if (location != null) { @@ -156,15 +159,24 @@ public boolean implies(ProtectionDomain domain, Permission permission) { } @SuppressForbidden(reason = "We get given an URL by the security infrastructure") - private boolean canAccessSecuredFile(URL location, FilePermission permission) { - if (location == null) { + private boolean canAccessSecuredFile(ProtectionDomain domain, FilePermission permission) { + if (domain == null || domain.getCodeSource() == null || domain.getCodeSource().getLocation() == null) { return false; } + // If the domain in question has AllPermission - only true of sources built into the JDK, as we prevent AllPermission from being + // configured in Elasticsearch - then it has access to this file. + + if (system.implies(domain, ALL_PERMISSION)) { + return true; + } + URL location = domain.getCodeSource().getLocation(); + // check the source Set accessibleSources = securedFiles.get(permission); if (accessibleSources != null) { // simple case - single-file referenced directly + return accessibleSources.contains(location); } else { // there's a directory reference in there somewhere diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java index be1220da6b1c4..216f22ddfd2bb 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.bootstrap; @@ -29,6 +30,7 @@ import org.elasticsearch.core.AbstractRefCounted; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.entitlement.bootstrap.EntitlementBootstrap; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.jdk.JarHell; @@ -39,6 +41,7 @@ import org.elasticsearch.nativeaccess.NativeAccess; import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeValidationException; +import org.elasticsearch.plugins.PluginsLoader; import java.io.IOException; import java.io.InputStream; @@ -48,10 +51,12 @@ import java.nio.file.Path; import java.security.Permission; import java.security.Security; +import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; import static org.elasticsearch.bootstrap.BootstrapSettings.SECURITY_FILTER_BAD_DEFAULTS_SETTING; import static org.elasticsearch.nativeaccess.WindowsFunctions.ConsoleCtrlHandler.CTRL_CLOSE_EVENT; @@ -101,6 +106,7 @@ private static Bootstrap initPhase1() { final PrintStream out = getStdout(); final PrintStream err = getStderr(); final ServerArgs args; + final boolean useEntitlements = Boolean.parseBoolean(System.getProperty("es.entitlements.enabled")); try { initSecurityProperties(); @@ -109,12 +115,14 @@ private static Bootstrap initPhase1() { * the presence of a security manager or lack thereof act as if there is a security manager present (e.g., DNS cache policy). * This forces such policies to take effect immediately. */ - org.elasticsearch.bootstrap.Security.setSecurityManager(new SecurityManager() { - @Override - public void checkPermission(Permission perm) { - // grant all permissions so that we can later set the security manager to the one that we want - } - }); + if (useEntitlements == false) { + org.elasticsearch.bootstrap.Security.setSecurityManager(new SecurityManager() { + @Override + public void checkPermission(Permission perm) { + // grant all permissions so that we can later set the security manager to the one that we want + } + }); + } LogConfigurator.registerErrorListener(); BootstrapInfo.init(); @@ -140,7 +148,7 @@ public void checkPermission(Permission perm) { return null; // unreachable, to satisfy compiler } - return new Bootstrap(out, err, args); + return new Bootstrap(out, err, args, useEntitlements); } /** @@ -197,12 +205,34 @@ private static void initPhase2(Bootstrap bootstrap) throws IOException { VectorUtil.class ); - // install SM after natives, shutdown hooks, etc. - org.elasticsearch.bootstrap.Security.configure( - nodeEnv, - SECURITY_FILTER_BAD_DEFAULTS_SETTING.get(args.nodeSettings()), - args.pidFile() - ); + // load the plugin Java modules and layers now for use in entitlements + var pluginsLoader = PluginsLoader.createPluginsLoader(nodeEnv.modulesFile(), nodeEnv.pluginsFile()); + bootstrap.setPluginsLoader(pluginsLoader); + + if (bootstrap.useEntitlements()) { + LogManager.getLogger(Elasticsearch.class).info("Bootstrapping Entitlements"); + + List pluginData = Stream.concat( + pluginsLoader.moduleBundles() + .stream() + .map(bundle -> new EntitlementBootstrap.PluginData(bundle.getDir(), bundle.pluginDescriptor().isModular(), false)), + pluginsLoader.pluginBundles() + .stream() + .map(bundle -> new EntitlementBootstrap.PluginData(bundle.getDir(), bundle.pluginDescriptor().isModular(), true)) + ).toList(); + + var pluginsResolver = PluginsResolver.create(pluginsLoader); + + EntitlementBootstrap.bootstrap(pluginData, pluginsResolver::resolveClassToPluginName); + } else { + // install SM after natives, shutdown hooks, etc. + LogManager.getLogger(Elasticsearch.class).info("Bootstrapping java SecurityManager"); + org.elasticsearch.bootstrap.Security.configure( + nodeEnv, + SECURITY_FILTER_BAD_DEFAULTS_SETTING.get(args.nodeSettings()), + args.pidFile() + ); + } } private static void ensureInitialized(Class... classes) { @@ -236,14 +266,18 @@ private static void ensureInitialized(Class... classes) { private static void initPhase3(Bootstrap bootstrap) throws IOException, NodeValidationException { checkLucene(); - Node node = new Node(bootstrap.environment()) { + Node node = new Node(bootstrap.environment(), bootstrap.pluginsLoader()) { @Override protected void validateNodeBeforeAcceptingRequests( final BootstrapContext context, final BoundTransportAddress boundTransportAddress, List checks ) throws NodeValidationException { - BootstrapChecks.check(context, boundTransportAddress, checks); + var additionalChecks = new ArrayList<>(checks); + if (bootstrap.useEntitlements() == false) { + additionalChecks.add(new BootstrapChecks.AllPermissionCheck()); + } + BootstrapChecks.check(context, boundTransportAddress, additionalChecks); } }; INSTANCE = new Elasticsearch(bootstrap.spawner(), node); diff --git a/server/src/main/java/org/elasticsearch/bootstrap/ElasticsearchProcess.java b/server/src/main/java/org/elasticsearch/bootstrap/ElasticsearchProcess.java index 7397bb98322f5..ec462d22b20b8 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/ElasticsearchProcess.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/ElasticsearchProcess.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.bootstrap; diff --git a/server/src/main/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandler.java b/server/src/main/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandler.java index 8fd5e40dab1d2..b2c1bcb1d544a 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandler.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandler.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.bootstrap; diff --git a/server/src/main/java/org/elasticsearch/bootstrap/FilePermissionUtils.java b/server/src/main/java/org/elasticsearch/bootstrap/FilePermissionUtils.java index b1553ca32122d..396318a2b2cf7 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/FilePermissionUtils.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/FilePermissionUtils.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.bootstrap; diff --git a/server/src/main/java/org/elasticsearch/bootstrap/PluginPolicyInfo.java b/server/src/main/java/org/elasticsearch/bootstrap/PluginPolicyInfo.java index a029acbb11da7..c5fb06a1bcba3 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/PluginPolicyInfo.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/PluginPolicyInfo.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.bootstrap; diff --git a/server/src/main/java/org/elasticsearch/bootstrap/PluginsResolver.java b/server/src/main/java/org/elasticsearch/bootstrap/PluginsResolver.java new file mode 100644 index 0000000000000..256e91cbee16d --- /dev/null +++ b/server/src/main/java/org/elasticsearch/bootstrap/PluginsResolver.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.bootstrap; + +import org.elasticsearch.plugins.PluginsLoader; + +import java.util.HashMap; +import java.util.Map; + +class PluginsResolver { + private final Map pluginNameByModule; + + private PluginsResolver(Map pluginNameByModule) { + this.pluginNameByModule = pluginNameByModule; + } + + public static PluginsResolver create(PluginsLoader pluginsLoader) { + Map pluginNameByModule = new HashMap<>(); + + pluginsLoader.pluginLayers().forEach(pluginLayer -> { + var pluginName = pluginLayer.pluginBundle().pluginDescriptor().getName(); + if (pluginLayer.pluginModuleLayer() != null && pluginLayer.pluginModuleLayer() != ModuleLayer.boot()) { + // This plugin is a Java Module + for (var module : pluginLayer.pluginModuleLayer().modules()) { + pluginNameByModule.put(module, pluginName); + } + } else { + // This plugin is not modularized + pluginNameByModule.put(pluginLayer.pluginClassLoader().getUnnamedModule(), pluginName); + } + }); + + return new PluginsResolver(pluginNameByModule); + } + + public String resolveClassToPluginName(Class clazz) { + var module = clazz.getModule(); + return pluginNameByModule.get(module); + } +} diff --git a/server/src/main/java/org/elasticsearch/bootstrap/PolicyUtil.java b/server/src/main/java/org/elasticsearch/bootstrap/PolicyUtil.java index 444ec49c70407..78cf0ee93a0e5 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/PolicyUtil.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/PolicyUtil.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.bootstrap; diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Security.java b/server/src/main/java/org/elasticsearch/bootstrap/Security.java index 12edf344c72a2..dc6de9a6b2c91 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Security.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Security.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.bootstrap; @@ -46,6 +47,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.function.Consumer; @@ -235,17 +237,25 @@ private static Map> readSecuredConfigFiles( for (Map.Entry> ps : settingPatterns) { if (ps.getKey().matcher(setting).matches()) { // add the setting value to the secured files for these codebase URLs - Path file = environment.configFile().resolve(environment.settings().get(setting)); - if (file.startsWith(environment.configFile()) == false) { - throw new IllegalStateException(ps.getValue() + " tried to grant access to file outside config directory " + file); - } - if (logger.isDebugEnabled()) { - ps.getValue() - .forEach( - url -> logger.debug("Jar {} securing access to config file {} through setting {}", url, file, setting) + String settingValue = environment.settings().get(setting); + // Some settings can also be an HTTPS URL in addition to a file path; if that's the case just skip this one. + // If the setting shouldn't be an HTTPS URL, that'll be caught by that setting's validation later in the process. + // HTTP (no S) URLs are not supported. + if (settingValue.toLowerCase(Locale.ROOT).startsWith("https://") == false) { + Path file = environment.configFile().resolve(settingValue); + if (file.startsWith(environment.configFile()) == false) { + throw new IllegalStateException( + ps.getValue() + " tried to grant access to file outside config directory " + file ); + } + if (logger.isDebugEnabled()) { + ps.getValue() + .forEach( + url -> logger.debug("Jar {} securing access to config file {} through setting {}", url, file, setting) + ); + } + securedConfigFiles.computeIfAbsent(file.toString(), k -> new HashSet<>()).addAll(ps.getValue()); } - securedConfigFiles.computeIfAbsent(file.toString(), k -> new HashSet<>()).addAll(ps.getValue()); } } } diff --git a/server/src/main/java/org/elasticsearch/bootstrap/ServerArgs.java b/server/src/main/java/org/elasticsearch/bootstrap/ServerArgs.java index f0b1f654150a9..6d6b6f95f2a06 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/ServerArgs.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/ServerArgs.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.bootstrap; diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Spawner.java b/server/src/main/java/org/elasticsearch/bootstrap/Spawner.java index 2d37da1d10245..6a4296d9b0478 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Spawner.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Spawner.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.bootstrap; diff --git a/server/src/main/java/org/elasticsearch/bootstrap/StartupException.java b/server/src/main/java/org/elasticsearch/bootstrap/StartupException.java index 14027a417e2f6..bb13c6959749d 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/StartupException.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/StartupException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.bootstrap; diff --git a/server/src/main/java/org/elasticsearch/client/internal/AdminClient.java b/server/src/main/java/org/elasticsearch/client/internal/AdminClient.java index 3804183818123..214ab4bae236f 100644 --- a/server/src/main/java/org/elasticsearch/client/internal/AdminClient.java +++ b/server/src/main/java/org/elasticsearch/client/internal/AdminClient.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.client.internal; diff --git a/server/src/main/java/org/elasticsearch/client/internal/Client.java b/server/src/main/java/org/elasticsearch/client/internal/Client.java index 6eb742fb5853e..4158bbfb27cda 100644 --- a/server/src/main/java/org/elasticsearch/client/internal/Client.java +++ b/server/src/main/java/org/elasticsearch/client/internal/Client.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.client.internal; diff --git a/server/src/main/java/org/elasticsearch/client/internal/ClusterAdminClient.java b/server/src/main/java/org/elasticsearch/client/internal/ClusterAdminClient.java index 1509e398fbffa..ad944759bb2f6 100644 --- a/server/src/main/java/org/elasticsearch/client/internal/ClusterAdminClient.java +++ b/server/src/main/java/org/elasticsearch/client/internal/ClusterAdminClient.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.client.internal; diff --git a/server/src/main/java/org/elasticsearch/client/internal/ElasticsearchClient.java b/server/src/main/java/org/elasticsearch/client/internal/ElasticsearchClient.java index d87ed2d7247d6..a301ee5d1e40f 100644 --- a/server/src/main/java/org/elasticsearch/client/internal/ElasticsearchClient.java +++ b/server/src/main/java/org/elasticsearch/client/internal/ElasticsearchClient.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.client.internal; diff --git a/server/src/main/java/org/elasticsearch/client/internal/FilterClient.java b/server/src/main/java/org/elasticsearch/client/internal/FilterClient.java index 3e31340ec535d..3588c2065b2b7 100644 --- a/server/src/main/java/org/elasticsearch/client/internal/FilterClient.java +++ b/server/src/main/java/org/elasticsearch/client/internal/FilterClient.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.client.internal; diff --git a/server/src/main/java/org/elasticsearch/client/internal/IndicesAdminClient.java b/server/src/main/java/org/elasticsearch/client/internal/IndicesAdminClient.java index 004eef1fecbef..7a5e027a850fe 100644 --- a/server/src/main/java/org/elasticsearch/client/internal/IndicesAdminClient.java +++ b/server/src/main/java/org/elasticsearch/client/internal/IndicesAdminClient.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.client.internal; diff --git a/server/src/main/java/org/elasticsearch/client/internal/OriginSettingClient.java b/server/src/main/java/org/elasticsearch/client/internal/OriginSettingClient.java index 9f081f63018e3..13d90725f6406 100644 --- a/server/src/main/java/org/elasticsearch/client/internal/OriginSettingClient.java +++ b/server/src/main/java/org/elasticsearch/client/internal/OriginSettingClient.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.client.internal; diff --git a/server/src/main/java/org/elasticsearch/client/internal/ParentTaskAssigningClient.java b/server/src/main/java/org/elasticsearch/client/internal/ParentTaskAssigningClient.java index 61528ccaa2427..014fef8bdfe56 100644 --- a/server/src/main/java/org/elasticsearch/client/internal/ParentTaskAssigningClient.java +++ b/server/src/main/java/org/elasticsearch/client/internal/ParentTaskAssigningClient.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.client.internal; @@ -17,6 +18,7 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.transport.RemoteClusterService; +import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportResponse; import java.util.concurrent.Executor; @@ -74,12 +76,18 @@ public RemoteClusterClient getRemoteClusterClient( return new RemoteClusterClient() { @Override public void execute( + Transport.Connection connection, RemoteClusterActionType action, Request request, ActionListener listener ) { request.setParentTask(parentTask); - delegate.execute(action, request, listener); + delegate.execute(connection, action, request, listener); + } + + @Override + public void getConnection(Request request, ActionListener listener) { + delegate.getConnection(request, listener); } }; } diff --git a/server/src/main/java/org/elasticsearch/client/internal/RemoteClusterClient.java b/server/src/main/java/org/elasticsearch/client/internal/RemoteClusterClient.java index 337286533f2ab..9e3497601fb57 100644 --- a/server/src/main/java/org/elasticsearch/client/internal/RemoteClusterClient.java +++ b/server/src/main/java/org/elasticsearch/client/internal/RemoteClusterClient.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.client.internal; @@ -12,6 +13,8 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.RemoteClusterActionType; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportResponse; /** @@ -20,16 +23,31 @@ public interface RemoteClusterClient { /** * Executes an action, denoted by an {@link ActionType}, on the remote cluster. - * - * @param action The action type to execute. - * @param request The action request. - * @param listener A listener for the response - * @param The request type. - * @param the response type. + */ + default void execute( + RemoteClusterActionType action, + Request request, + ActionListener listener + ) { + getConnection( + request, + listener.delegateFailureAndWrap((responseListener, connection) -> execute(connection, action, request, responseListener)) + ); + } + + /** + * Executes an action, denoted by an {@link ActionType}, using a connection to the remote cluster obtained using {@link #getConnection}. */ void execute( + Transport.Connection connection, RemoteClusterActionType action, Request request, ActionListener listener ); + + /** + * Obtain a connection to the remote cluster for use with the {@link #execute} override that allows to specify the connection. Useful + * for cases where you need to inspect {@link Transport.Connection#getVersion} before deciding the exact remote action to invoke. + */ + void getConnection(@Nullable Request request, ActionListener listener); } diff --git a/server/src/main/java/org/elasticsearch/client/internal/Requests.java b/server/src/main/java/org/elasticsearch/client/internal/Requests.java index acdc3b1b9f95b..a3eb547f21f6b 100644 --- a/server/src/main/java/org/elasticsearch/client/internal/Requests.java +++ b/server/src/main/java/org/elasticsearch/client/internal/Requests.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.client.internal; diff --git a/server/src/main/java/org/elasticsearch/client/internal/node/NodeClient.java b/server/src/main/java/org/elasticsearch/client/internal/node/NodeClient.java index f37adc2fb5e2d..7f5a7e20795b7 100644 --- a/server/src/main/java/org/elasticsearch/client/internal/node/NodeClient.java +++ b/server/src/main/java/org/elasticsearch/client/internal/node/NodeClient.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.client.internal.node; diff --git a/server/src/main/java/org/elasticsearch/client/internal/package-info.java b/server/src/main/java/org/elasticsearch/client/internal/package-info.java index ef91a791905e3..edd8d4829d2e6 100644 --- a/server/src/main/java/org/elasticsearch/client/internal/package-info.java +++ b/server/src/main/java/org/elasticsearch/client/internal/package-info.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ /** diff --git a/server/src/main/java/org/elasticsearch/client/internal/support/AbstractClient.java b/server/src/main/java/org/elasticsearch/client/internal/support/AbstractClient.java index c2268bfd9dc62..4a9d7f28dd0af 100644 --- a/server/src/main/java/org/elasticsearch/client/internal/support/AbstractClient.java +++ b/server/src/main/java/org/elasticsearch/client/internal/support/AbstractClient.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.client.internal.support; diff --git a/server/src/main/java/org/elasticsearch/client/internal/transport/NoNodeAvailableException.java b/server/src/main/java/org/elasticsearch/client/internal/transport/NoNodeAvailableException.java index 602596b2b39f9..44ad12b48bd68 100644 --- a/server/src/main/java/org/elasticsearch/client/internal/transport/NoNodeAvailableException.java +++ b/server/src/main/java/org/elasticsearch/client/internal/transport/NoNodeAvailableException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.client.internal.transport; diff --git a/server/src/main/java/org/elasticsearch/cluster/AbstractNamedDiffable.java b/server/src/main/java/org/elasticsearch/cluster/AbstractNamedDiffable.java index d2ccf3d6a2e0d..f4dbcdb5f3226 100644 --- a/server/src/main/java/org/elasticsearch/cluster/AbstractNamedDiffable.java +++ b/server/src/main/java/org/elasticsearch/cluster/AbstractNamedDiffable.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/AckedBatchedClusterStateUpdateTask.java b/server/src/main/java/org/elasticsearch/cluster/AckedBatchedClusterStateUpdateTask.java index 74965a4593d1a..56cb60aad826a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/AckedBatchedClusterStateUpdateTask.java +++ b/server/src/main/java/org/elasticsearch/cluster/AckedBatchedClusterStateUpdateTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/AckedClusterStateUpdateTask.java b/server/src/main/java/org/elasticsearch/cluster/AckedClusterStateUpdateTask.java index b81fde1156bd1..de42591c15d27 100644 --- a/server/src/main/java/org/elasticsearch/cluster/AckedClusterStateUpdateTask.java +++ b/server/src/main/java/org/elasticsearch/cluster/AckedClusterStateUpdateTask.java @@ -1,15 +1,16 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.cluster.ack.AckedRequest; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Priority; import org.elasticsearch.core.TimeValue; @@ -22,21 +23,38 @@ public abstract class AckedClusterStateUpdateTask extends ClusterStateUpdateTask implements ClusterStateAckListener { private final ActionListener listener; - private final AckedRequest request; + private final TimeValue ackTimeout; - protected AckedClusterStateUpdateTask(AckedRequest request, ActionListener listener) { - this(Priority.NORMAL, request, listener); + protected AckedClusterStateUpdateTask(AcknowledgedRequest request, ActionListener listener) { + this(Priority.NORMAL, request.masterNodeTimeout(), request.ackTimeout(), listener); + } + + protected AckedClusterStateUpdateTask( + TimeValue masterNodeTimeout, + TimeValue ackTimeout, + ActionListener listener + ) { + this(Priority.NORMAL, masterNodeTimeout, ackTimeout, listener); + } + + protected AckedClusterStateUpdateTask( + Priority priority, + AcknowledgedRequest request, + ActionListener listener + ) { + this(priority, request.masterNodeTimeout(), request.ackTimeout(), listener); } @SuppressWarnings("unchecked") protected AckedClusterStateUpdateTask( Priority priority, - AckedRequest request, + TimeValue masterNodeTimeout, + TimeValue ackTimeout, ActionListener listener ) { - super(priority, request.masterNodeTimeout()); + super(priority, masterNodeTimeout); this.listener = (ActionListener) listener; - this.request = request; + this.ackTimeout = ackTimeout; } /** @@ -80,6 +98,6 @@ public void onFailure(Exception e) { * Acknowledgement timeout, maximum time interval to wait for acknowledgements */ public final TimeValue ackTimeout() { - return request.ackTimeout(); + return ackTimeout; } } diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java b/server/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java index 0392ca2e6581a..e4a3003012912 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterFeatures.java b/server/src/main/java/org/elasticsearch/cluster/ClusterFeatures.java index bab68303e8de5..2cb4769dc4ddb 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterFeatures.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterFeatures.java @@ -1,16 +1,19 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; -import org.elasticsearch.common.collect.Iterators; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.common.xcontent.ChunkedToXContentObject; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.features.NodeFeature; @@ -91,6 +94,22 @@ public Set allNodeFeatures() { return allNodeFeatures; } + /** + * Returns {@code true} if {@code node} can have assumed features. + * @see org.elasticsearch.env.BuildVersion#canRemoveAssumedFeatures + */ + public static boolean featuresCanBeAssumedForNode(DiscoveryNode node) { + return node.getBuildVersion().canRemoveAssumedFeatures(); + } + + /** + * Returns {@code true} if one or more nodes in {@code nodes} can have assumed features. + * @see org.elasticsearch.env.BuildVersion#canRemoveAssumedFeatures + */ + public static boolean featuresCanBeAssumedForNodes(DiscoveryNodes nodes) { + return nodes.getAllNodes().stream().anyMatch(n -> n.getBuildVersion().canRemoveAssumedFeatures()); + } + /** * {@code true} if {@code feature} is present on all nodes in the cluster. *

    @@ -98,8 +117,32 @@ public Set allNodeFeatures() { * Please use {@link org.elasticsearch.features.FeatureService#clusterHasFeature} instead. */ @SuppressForbidden(reason = "directly reading cluster features") - public boolean clusterHasFeature(NodeFeature feature) { - return allNodeFeatures().contains(feature.id()); + public boolean clusterHasFeature(DiscoveryNodes nodes, NodeFeature feature) { + assert nodes.getNodes().keySet().equals(nodeFeatures.keySet()) + : "Cluster features nodes " + nodeFeatures.keySet() + " is different to discovery nodes " + nodes.getNodes().keySet(); + + // basic case + boolean allNodesHaveFeature = allNodeFeatures().contains(feature.id()); + if (allNodesHaveFeature) { + return true; + } + + // if the feature is assumed, check the versions more closely + // it's actually ok if the feature is assumed, and all nodes missing the feature can assume it + // TODO: do we need some kind of transient cache of this calculation? + if (feature.assumedAfterNextCompatibilityBoundary()) { + for (var nf : nodeFeatures.entrySet()) { + if (nf.getValue().contains(feature.id()) == false + && featuresCanBeAssumedForNode(nodes.getNodes().get(nf.getKey())) == false) { + return false; + } + } + + // all nodes missing the feature can assume it - so that's alright then + return true; + } + + return false; } /** @@ -243,15 +286,12 @@ public ClusterFeatures apply(ClusterFeatures part) { @Override public Iterator toXContentChunked(ToXContent.Params params) { - return Iterators.concat( - Iterators.single((builder, p) -> builder.startArray()), - nodeFeatures.entrySet().stream().sorted(Map.Entry.comparingByKey()).map(e -> (builder, p) -> { + return ChunkedToXContent.builder(params) + .array(nodeFeatures.entrySet().stream().sorted(Map.Entry.comparingByKey()).iterator(), e -> (builder, p) -> { String[] features = e.getValue().toArray(String[]::new); Arrays.sort(features); return builder.startObject().field("node_id", e.getKey()).array("features", features).endObject(); - }).iterator(), - Iterators.single((builder, p) -> builder.endArray()) - ); + }); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java index f817298e4e328..d60d0a1859caa 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; @@ -14,7 +15,6 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -35,9 +35,6 @@ import static org.elasticsearch.cluster.routing.ShardRouting.newUnassigned; import static org.elasticsearch.cluster.routing.UnassignedInfo.Reason.REINITIALIZED; -import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.endArray; -import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.singleChunk; -import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.startObject; /** * ClusterInfo is an object representing a map of nodes to {@link DiskUsage} @@ -141,7 +138,7 @@ private static ShardRouting createFakeShardRoutingFromNodeAndShard(NodeAndShard @Override public Iterator toXContentChunked(ToXContent.Params params) { - return Iterators.concat(startObject("nodes"), Iterators.map(leastAvailableSpaceUsage.entrySet().iterator(), c -> (builder, p) -> { + return ChunkedToXContent.builder(params).object("nodes", leastAvailableSpaceUsage.entrySet().iterator(), c -> (builder, p) -> { builder.startObject(c.getKey()); { // node builder.field("node_name", c.getValue().nodeName()); @@ -159,52 +156,32 @@ public Iterator toXContentChunked(ToXContent.Params params } builder.endObject(); // end "most_available" } - builder.endObject(); // end $nodename - return builder; - }), - singleChunk( - (builder, p) -> builder.endObject() // end "nodes" - .startObject("shard_sizes") - ), - - Iterators.map( + return builder.endObject(); // end $nodename + }) + .object( + "shard_sizes", shardSizes.entrySet().iterator(), c -> (builder, p) -> builder.humanReadableField(c.getKey() + "_bytes", c.getKey(), ByteSizeValue.ofBytes(c.getValue())) - ), - singleChunk( - (builder, p) -> builder.endObject() // end "shard_sizes" - .startObject("shard_data_set_sizes") - ), - Iterators.map( + ) + .object( + "shard_data_set_sizes", shardDataSetSizes.entrySet().iterator(), c -> (builder, p) -> builder.humanReadableField( c.getKey() + "_bytes", c.getKey().toString(), ByteSizeValue.ofBytes(c.getValue()) ) - ), - singleChunk( - (builder, p) -> builder.endObject() // end "shard_data_set_sizes" - .startObject("shard_paths") - ), - Iterators.map(dataPath.entrySet().iterator(), c -> (builder, p) -> builder.field(c.getKey().toString(), c.getValue())), - singleChunk( - (builder, p) -> builder.endObject() // end "shard_paths" - .startArray("reserved_sizes") - ), - Iterators.map(reservedSpace.entrySet().iterator(), c -> (builder, p) -> { + ) + .object("shard_paths", dataPath.entrySet().iterator(), (xb, c) -> xb.field(c.getKey().toString(), c.getValue())) + .array("reserved_sizes", reservedSpace.entrySet().iterator(), c -> (builder, p) -> { builder.startObject(); { builder.field("node_id", c.getKey().nodeId); builder.field("path", c.getKey().path); - c.getValue().toXContent(builder, params); + c.getValue().toXContent(builder, p); } return builder.endObject(); // NodeAndPath - }), - - endArray() // end "reserved_sizes" - - ); + }); } /** @@ -313,7 +290,7 @@ public String toString() { // exposed for tests, computed here rather than exposing all the collections separately int getChunkCount() { - return leastAvailableSpaceUsage.size() + shardSizes.size() + shardDataSetSizes.size() + dataPath.size() + reservedSpace.size() + 6; + return leastAvailableSpaceUsage.size() + shardSizes.size() + shardDataSetSizes.size() + dataPath.size() + reservedSpace.size() + 10; } public record NodeAndShard(String nodeId, ShardId shardId) implements Writeable { diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterInfoService.java b/server/src/main/java/org/elasticsearch/cluster/ClusterInfoService.java index 14ceda19b634e..496ff42637731 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterInfoService.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterInfoService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterInfoSimulator.java b/server/src/main/java/org/elasticsearch/cluster/ClusterInfoSimulator.java index 593bb251d3f5e..f1048c7939ef9 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterInfoSimulator.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterInfoSimulator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java index e399e739da047..5b87267b69527 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterName.java b/server/src/main/java/org/elasticsearch/cluster/ClusterName.java index dd4194b60e6ac..8ee90fe713565 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterName.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterName.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterSnapshotStats.java b/server/src/main/java/org/elasticsearch/cluster/ClusterSnapshotStats.java index c62eeeab3e479..ac96a2d55bc71 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterSnapshotStats.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterSnapshotStats.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; @@ -227,7 +228,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); builder.endObject(); - builder.timeField("oldest_start_time_millis", "oldest_start_time", firstStartTimeMillis); + builder.timestampFieldsFromUnixEpochMillis("oldest_start_time_millis", "oldest_start_time", firstStartTimeMillis); return builder.endObject(); } diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java index 02d5bdfdbebc0..f7cad013554c6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; @@ -240,9 +241,7 @@ public ClusterState( } private boolean assertEventIngestedIsUnknownInMixedClusters(Metadata metadata, CompatibilityVersions compatibilityVersions) { - if (compatibilityVersions.transportVersion().before(TransportVersions.EVENT_INGESTED_RANGE_IN_CLUSTER_STATE) - && metadata != null - && metadata.indices() != null) { + if (compatibilityVersions.transportVersion().before(TransportVersions.V_8_15_0) && metadata != null && metadata.indices() != null) { for (IndexMetadata indexMetadata : metadata.indices().values()) { assert indexMetadata.getEventIngestedRange() == IndexLongFieldRange.UNKNOWN : "event.ingested range should be UNKNOWN but is " @@ -758,10 +757,8 @@ public Iterator toXContentChunked(ToXContent.Params outerP // customs metrics.contains(Metric.CUSTOMS) - ? Iterators.flatMap( - customs.entrySet().iterator(), - cursor -> ChunkedToXContentHelper.wrapWithObject(cursor.getKey(), cursor.getValue().toXContentChunked(outerParams)) - ) + ? ChunkedToXContent.builder(outerParams) + .forEach(customs.entrySet().iterator(), (b, e) -> b.xContentObject(e.getKey(), e.getValue())) : Collections.emptyIterator() ); } diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterStateAckListener.java b/server/src/main/java/org/elasticsearch/cluster/ClusterStateAckListener.java index c30665b6d3bca..8f57a71658604 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterStateAckListener.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterStateAckListener.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterStateApplier.java b/server/src/main/java/org/elasticsearch/cluster/ClusterStateApplier.java index 7b5f5cd6ca789..8eff2fab7091a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterStateApplier.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterStateApplier.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterStateListener.java b/server/src/main/java/org/elasticsearch/cluster/ClusterStateListener.java index 65a0487a4cded..bb2f2caddbeb1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterStateListener.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterStateListener.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterStateObserver.java b/server/src/main/java/org/elasticsearch/cluster/ClusterStateObserver.java index 52e2c2399397e..4d3ed67d0c527 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterStateObserver.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterStateObserver.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterStatePublicationEvent.java b/server/src/main/java/org/elasticsearch/cluster/ClusterStatePublicationEvent.java index bae96e912da26..c6cfefdd10c72 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterStatePublicationEvent.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterStatePublicationEvent.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java b/server/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java index 081f8150d8c8e..36d1947af73ba 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterStateTaskListener.java b/server/src/main/java/org/elasticsearch/cluster/ClusterStateTaskListener.java index 85305529b6a4d..b14e2233a9a5b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterStateTaskListener.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterStateTaskListener.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java b/server/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java index 4f038132e7dfa..3979e5c7a4683 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/Diff.java b/server/src/main/java/org/elasticsearch/cluster/Diff.java index 37605da53a649..1b4f33e4d15ee 100644 --- a/server/src/main/java/org/elasticsearch/cluster/Diff.java +++ b/server/src/main/java/org/elasticsearch/cluster/Diff.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/Diffable.java b/server/src/main/java/org/elasticsearch/cluster/Diffable.java index ce940f806b093..61b0c0cb39c12 100644 --- a/server/src/main/java/org/elasticsearch/cluster/Diffable.java +++ b/server/src/main/java/org/elasticsearch/cluster/Diffable.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/DiffableUtils.java b/server/src/main/java/org/elasticsearch/cluster/DiffableUtils.java index 061da87b00f69..0926dfdb4931c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/DiffableUtils.java +++ b/server/src/main/java/org/elasticsearch/cluster/DiffableUtils.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/DiskUsage.java b/server/src/main/java/org/elasticsearch/cluster/DiskUsage.java index 3bc6f889e5778..d06797372d9f2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/DiskUsage.java +++ b/server/src/main/java/org/elasticsearch/cluster/DiskUsage.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/EmptyClusterInfoService.java b/server/src/main/java/org/elasticsearch/cluster/EmptyClusterInfoService.java index 05cb928e63974..259975ee1fecf 100644 --- a/server/src/main/java/org/elasticsearch/cluster/EmptyClusterInfoService.java +++ b/server/src/main/java/org/elasticsearch/cluster/EmptyClusterInfoService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/IncompatibleClusterStateVersionException.java b/server/src/main/java/org/elasticsearch/cluster/IncompatibleClusterStateVersionException.java index 3d6b44bd33b8b..89a31dc69122e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/IncompatibleClusterStateVersionException.java +++ b/server/src/main/java/org/elasticsearch/cluster/IncompatibleClusterStateVersionException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java index f4440688ea307..2ec19c73d4a20 100644 --- a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java +++ b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/LocalMasterServiceTask.java b/server/src/main/java/org/elasticsearch/cluster/LocalMasterServiceTask.java index 0fff944925abd..3a1878485a79f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/LocalMasterServiceTask.java +++ b/server/src/main/java/org/elasticsearch/cluster/LocalMasterServiceTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/LocalNodeMasterListener.java b/server/src/main/java/org/elasticsearch/cluster/LocalNodeMasterListener.java index ff37613d8cb05..b9ba50507ea2c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/LocalNodeMasterListener.java +++ b/server/src/main/java/org/elasticsearch/cluster/LocalNodeMasterListener.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/NamedDiff.java b/server/src/main/java/org/elasticsearch/cluster/NamedDiff.java index bda313b8bb6f3..cae73963dde96 100644 --- a/server/src/main/java/org/elasticsearch/cluster/NamedDiff.java +++ b/server/src/main/java/org/elasticsearch/cluster/NamedDiff.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/NamedDiffable.java b/server/src/main/java/org/elasticsearch/cluster/NamedDiffable.java index 25cd2ff177900..bebbaa0e4b3c8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/NamedDiffable.java +++ b/server/src/main/java/org/elasticsearch/cluster/NamedDiffable.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/NamedDiffableValueSerializer.java b/server/src/main/java/org/elasticsearch/cluster/NamedDiffableValueSerializer.java index d77afcd86a190..0600ec9b95d8b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/NamedDiffableValueSerializer.java +++ b/server/src/main/java/org/elasticsearch/cluster/NamedDiffableValueSerializer.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java b/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java index d765755b5d250..d26fb7adb56a9 100644 --- a/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java +++ b/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/NotMasterException.java b/server/src/main/java/org/elasticsearch/cluster/NotMasterException.java index d9616b0dd6c32..43c43bda9aa8c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/NotMasterException.java +++ b/server/src/main/java/org/elasticsearch/cluster/NotMasterException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/RepositoryCleanupInProgress.java b/server/src/main/java/org/elasticsearch/cluster/RepositoryCleanupInProgress.java index cc5e71b38ecb2..06ec8abf60ff4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/RepositoryCleanupInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/RepositoryCleanupInProgress.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java b/server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java index 2b618aa53a354..a084276de9112 100644 --- a/server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/SimpleBatchedAckListenerTaskExecutor.java b/server/src/main/java/org/elasticsearch/cluster/SimpleBatchedAckListenerTaskExecutor.java index 4e1c84a8e3c1d..fb746028e1f2a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SimpleBatchedAckListenerTaskExecutor.java +++ b/server/src/main/java/org/elasticsearch/cluster/SimpleBatchedAckListenerTaskExecutor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/SimpleBatchedExecutor.java b/server/src/main/java/org/elasticsearch/cluster/SimpleBatchedExecutor.java index 7fd772f51771e..98d9a04ac2d59 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SimpleBatchedExecutor.java +++ b/server/src/main/java/org/elasticsearch/cluster/SimpleBatchedExecutor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/SimpleDiffable.java b/server/src/main/java/org/elasticsearch/cluster/SimpleDiffable.java index 303d792263386..e17877cb61925 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SimpleDiffable.java +++ b/server/src/main/java/org/elasticsearch/cluster/SimpleDiffable.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java index 914bf2d0cdb3e..fe144135d42bd 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; @@ -179,7 +180,7 @@ public Iterator toXContentChunked(ToXContent.Params ignore builder.value(snapshot.getName()); } builder.endArray(); - builder.timeField("start_time_millis", "start_time", entry.startTime); + builder.timestampFieldsFromUnixEpochMillis("start_time_millis", "start_time", entry.startTime); builder.field("repository_state_id", entry.repositoryStateId); builder.field("state", entry.state); } diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java index 7b0ab346501f3..d82a31720d6d4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; @@ -1403,7 +1404,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } } builder.endArray(); - builder.timeField("start_time_millis", "start_time", startTime); + builder.timestampFieldsFromUnixEpochMillis("start_time_millis", "start_time", startTime); builder.field("repository_state_id", repositoryStateId); builder.startArray("shards"); { diff --git a/server/src/main/java/org/elasticsearch/cluster/TimeoutClusterStateListener.java b/server/src/main/java/org/elasticsearch/cluster/TimeoutClusterStateListener.java index 39ba62c8d0078..a8a37061172b8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/TimeoutClusterStateListener.java +++ b/server/src/main/java/org/elasticsearch/cluster/TimeoutClusterStateListener.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster; diff --git a/server/src/main/java/org/elasticsearch/cluster/ack/AckedRequest.java b/server/src/main/java/org/elasticsearch/cluster/ack/AckedRequest.java deleted file mode 100644 index 9c84fc2b544e7..0000000000000 --- a/server/src/main/java/org/elasticsearch/cluster/ack/AckedRequest.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.cluster.ack; - -import org.elasticsearch.core.TimeValue; - -/** - * Identifies a cluster state update request with acknowledgement support - */ -public interface AckedRequest { - - /** - * Returns the acknowledgement timeout - */ - TimeValue ackTimeout(); - - /** - * Returns the timeout for the request to be completed on the master node - */ - TimeValue masterNodeTimeout(); -} diff --git a/server/src/main/java/org/elasticsearch/cluster/ack/ClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/cluster/ack/ClusterStateUpdateRequest.java deleted file mode 100644 index 9c132cb2a5718..0000000000000 --- a/server/src/main/java/org/elasticsearch/cluster/ack/ClusterStateUpdateRequest.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.cluster.ack; - -import org.elasticsearch.core.TimeValue; - -/** - * Base class to be used when needing to update the cluster state - * Contains the basic fields that are always needed - */ -public abstract class ClusterStateUpdateRequest> implements AckedRequest { - - private TimeValue ackTimeout; - private TimeValue masterNodeTimeout; - - /** - * Returns the maximum time interval to wait for acknowledgements - */ - @Override - public TimeValue ackTimeout() { - return ackTimeout; - } - - /** - * Sets the acknowledgement timeout - */ - @SuppressWarnings("unchecked") - public T ackTimeout(TimeValue ackTimeout) { - this.ackTimeout = ackTimeout; - return (T) this; - } - - /** - * Returns the maximum time interval to wait for the request to - * be completed on the master node - */ - @Override - public TimeValue masterNodeTimeout() { - return masterNodeTimeout; - } - - /** - * Sets the master node timeout - */ - @SuppressWarnings("unchecked") - public T masterNodeTimeout(TimeValue masterNodeTimeout) { - this.masterNodeTimeout = masterNodeTimeout; - return (T) this; - } -} diff --git a/server/src/main/java/org/elasticsearch/cluster/ack/IndicesClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/cluster/ack/IndicesClusterStateUpdateRequest.java deleted file mode 100644 index 72c73f5f6326d..0000000000000 --- a/server/src/main/java/org/elasticsearch/cluster/ack/IndicesClusterStateUpdateRequest.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -package org.elasticsearch.cluster.ack; - -import org.elasticsearch.index.Index; - -/** - * Base cluster state update request that allows to execute update against multiple indices - */ -public abstract class IndicesClusterStateUpdateRequest> extends ClusterStateUpdateRequest { - - private Index[] indices; - - /** - * Returns the indices the operation needs to be executed on - */ - public Index[] indices() { - return indices; - } - - /** - * Sets the indices the operation needs to be executed on - */ - @SuppressWarnings("unchecked") - public T indices(Index[] indices) { - this.indices = indices; - return (T) this; - } -} diff --git a/server/src/main/java/org/elasticsearch/cluster/action/index/MappingUpdatedAction.java b/server/src/main/java/org/elasticsearch/cluster/action/index/MappingUpdatedAction.java index aa8ba17a0f05f..4152bd422b159 100644 --- a/server/src/main/java/org/elasticsearch/cluster/action/index/MappingUpdatedAction.java +++ b/server/src/main/java/org/elasticsearch/cluster/action/index/MappingUpdatedAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.action.index; diff --git a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java index ca5fab1087cdc..bd438d66549aa 100644 --- a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java +++ b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.action.shard; @@ -721,7 +722,7 @@ public ClusterState execute(BatchExecutionContext batchE */ IndexLongFieldRange newEventIngestedMillisRange = IndexLongFieldRange.UNKNOWN; TransportVersion minTransportVersion = batchExecutionContext.initialState().getMinTransportVersion(); - if (minTransportVersion.onOrAfter(TransportVersions.EVENT_INGESTED_RANGE_IN_CLUSTER_STATE)) { + if (minTransportVersion.onOrAfter(TransportVersions.V_8_15_0)) { newEventIngestedMillisRange = currentEventIngestedMillisRange.extendWithShardRange( startedShardEntry.shardId.id(), indexMetadata.getNumberOfShards(), @@ -826,7 +827,7 @@ public static class StartedShardEntry extends TransportRequest { primaryTerm = in.readVLong(); this.message = in.readString(); this.timestampRange = ShardLongFieldRange.readFrom(in); - if (in.getTransportVersion().onOrAfter(TransportVersions.EVENT_INGESTED_RANGE_IN_CLUSTER_STATE)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { this.eventIngestedRange = ShardLongFieldRange.readFrom(in); } else { this.eventIngestedRange = ShardLongFieldRange.UNKNOWN; @@ -857,7 +858,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(primaryTerm); out.writeString(message); timestampRange.writeTo(out); - if (out.getTransportVersion().onOrAfter(TransportVersions.EVENT_INGESTED_RANGE_IN_CLUSTER_STATE)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { eventIngestedRange.writeTo(out); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/block/ClusterBlock.java b/server/src/main/java/org/elasticsearch/cluster/block/ClusterBlock.java index 463bc4d9c396c..25c6a1ff5b67f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/block/ClusterBlock.java +++ b/server/src/main/java/org/elasticsearch/cluster/block/ClusterBlock.java @@ -1,13 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.block; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -20,6 +22,7 @@ import java.util.EnumSet; import java.util.Locale; import java.util.Objects; +import java.util.function.Predicate; public class ClusterBlock implements Writeable, ToXContentFragment { @@ -141,7 +144,12 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVInt(id); out.writeOptionalString(uuid); out.writeString(description); - out.writeEnumSet(levels); + if (out.getTransportVersion().onOrAfter(TransportVersions.NEW_REFRESH_CLUSTER_BLOCK)) { + out.writeEnumSet(levels); + } else { + // do not send ClusterBlockLevel.REFRESH to old nodes + out.writeEnumSet(filterLevels(levels, level -> ClusterBlockLevel.REFRESH.equals(level) == false)); + } out.writeBoolean(retryable); out.writeBoolean(disableStatePersistence); RestStatus.writeTo(out, status); @@ -184,4 +192,19 @@ public int hashCode() { public boolean isAllowReleaseResources() { return allowReleaseResources; } + + static EnumSet filterLevels(EnumSet levels, Predicate predicate) { + assert levels != null; + int size = levels.size(); + if (size == 0 || (size == 1 && predicate.test(levels.iterator().next()))) { + return levels; + } + var filteredLevels = EnumSet.noneOf(ClusterBlockLevel.class); + for (ClusterBlockLevel level : levels) { + if (predicate.test(level)) { + filteredLevels.add(level); + } + } + return filteredLevels; + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/block/ClusterBlockException.java b/server/src/main/java/org/elasticsearch/cluster/block/ClusterBlockException.java index c496ccccd9c10..33faa5e01b6ca 100644 --- a/server/src/main/java/org/elasticsearch/cluster/block/ClusterBlockException.java +++ b/server/src/main/java/org/elasticsearch/cluster/block/ClusterBlockException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.block; diff --git a/server/src/main/java/org/elasticsearch/cluster/block/ClusterBlockLevel.java b/server/src/main/java/org/elasticsearch/cluster/block/ClusterBlockLevel.java index dae78a054ce55..262044b091ac7 100644 --- a/server/src/main/java/org/elasticsearch/cluster/block/ClusterBlockLevel.java +++ b/server/src/main/java/org/elasticsearch/cluster/block/ClusterBlockLevel.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.block; @@ -14,7 +15,8 @@ public enum ClusterBlockLevel { READ, WRITE, METADATA_READ, - METADATA_WRITE; + METADATA_WRITE, + REFRESH; public static final EnumSet ALL = EnumSet.allOf(ClusterBlockLevel.class); public static final EnumSet READ_WRITE = EnumSet.of(READ, WRITE); diff --git a/server/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java b/server/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java index 079fac7faaaa0..aa3a6a201eac4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java +++ b/server/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.block; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/ApplyCommitRequest.java b/server/src/main/java/org/elasticsearch/cluster/coordination/ApplyCommitRequest.java index 8957f27decd2c..59a53eab4a8bc 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/ApplyCommitRequest.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/ApplyCommitRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/CleanableResponseHandler.java b/server/src/main/java/org/elasticsearch/cluster/coordination/CleanableResponseHandler.java index a7c032440fb36..e909bc958b29a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/CleanableResponseHandler.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/CleanableResponseHandler.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterBootstrapService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterBootstrapService.java index ac0e079ef4c5e..5e351eca8af12 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterBootstrapService.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterBootstrapService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelper.java b/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelper.java index b46b79754be7a..57df379be02fd 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelper.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterStatePublisher.java b/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterStatePublisher.java index baa20dc0920da..81c12e3e8f000 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterStatePublisher.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterStatePublisher.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterStateSerializationStats.java b/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterStateSerializationStats.java index 8f515c96a2801..8299155b05564 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterStateSerializationStats.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterStateSerializationStats.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationDiagnosticsService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationDiagnosticsService.java index d16f4ee27e8a1..84f9c42d27ece 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationDiagnosticsService.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationDiagnosticsService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; @@ -29,7 +30,6 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.util.concurrent.ListenableFuture; -import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; @@ -193,9 +193,7 @@ public void start() { * system context. */ if (clusterService.localNode().isMasterNode() == false) { - final ThreadContext threadContext = transportService.getThreadPool().getThreadContext(); - try (ThreadContext.StoredContext ignored = threadContext.stashContext()) { - threadContext.markAsSystemContext(); + try (var ignored = transportService.getThreadPool().getThreadContext().newEmptySystemContext()) { beginPollingRemoteMasterStabilityDiagnostic(); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationMetadata.java b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationMetadata.java index 0a859159c7843..b667b1a098d24 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationMetadata.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java index 402e170f1ea53..62d3f5e5866bc 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; @@ -466,7 +467,7 @@ public void handleCommit(ApplyCommitRequest applyCommit) { logger.debug( "handleCommit: ignored commit request due to term mismatch " + "(expected: [term {} version {}], actual: [term {} version {}])", - getLastAcceptedTerm(), + getCurrentTerm(), getLastAcceptedVersion(), applyCommit.getTerm(), applyCommit.getVersion() diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationStateRejectedException.java b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationStateRejectedException.java index f55f095031f7d..f3b03bc6bceee 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationStateRejectedException.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationStateRejectedException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java index e922d130d7f83..8a4464f194fc5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/DetachClusterCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/DetachClusterCommand.java index 9f05d93f7a9fb..7f636708ef943 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/DetachClusterCommand.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/DetachClusterCommand.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java index 82abf4b4c7d5d..35177b0981b89 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/ElectionSchedulerFactory.java b/server/src/main/java/org/elasticsearch/cluster/coordination/ElectionSchedulerFactory.java index d0b80a9739f7b..71f9e4da7106b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/ElectionSchedulerFactory.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/ElectionSchedulerFactory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/ElectionStrategy.java b/server/src/main/java/org/elasticsearch/cluster/coordination/ElectionStrategy.java index 2bf6e10a9855a..f8ab14f540312 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/ElectionStrategy.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/ElectionStrategy.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/FailedToCommitClusterStateException.java b/server/src/main/java/org/elasticsearch/cluster/coordination/FailedToCommitClusterStateException.java index 2157bca90d7c7..2731bc2973321 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/FailedToCommitClusterStateException.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/FailedToCommitClusterStateException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java b/server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java index 6fc15c81bfe51..6ab4ae82ced9b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/InMemoryPersistedState.java b/server/src/main/java/org/elasticsearch/cluster/coordination/InMemoryPersistedState.java index befda82b5d080..ca0918dba7568 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/InMemoryPersistedState.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/InMemoryPersistedState.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Join.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Join.java index d1fe472278f12..94d5e5be36d84 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Join.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Join.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java index 05dbc66c95971..46ffb7cc2b9c9 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinReason.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinReason.java index 356a17f1e1101..c8f16d532b34e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinReason.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinReason.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinReasonService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinReasonService.java index cf5ff1ff44c6e..91938e7e0f3e4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinReasonService.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinReasonService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinRequest.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinRequest.java index 1817497d0912c..2247d7a92cf41 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinRequest.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinStatus.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinStatus.java index 6e0e7d8dda5a5..89083848d488d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinStatus.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinStatus.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; @@ -24,7 +25,7 @@ public JoinStatus(StreamInput in) throws IOException { new DiscoveryNode(in), in.readLong(), in.readString(), - in.getTransportVersion().onOrAfter(TransportVersions.JOIN_STATUS_AGE_SERIALIZATION) + in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0) ? in.readTimeValue() : new TimeValue(in.readLong(), TimeUnit.valueOf(in.readString())) ); @@ -35,7 +36,7 @@ public void writeTo(StreamOutput out) throws IOException { remoteNode.writeTo(out); out.writeLong(term); out.writeString(message); - if (out.getTransportVersion().onOrAfter(TransportVersions.JOIN_STATUS_AGE_SERIALIZATION)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { out.writeTimeValue(age); } else { out.writeLong(age.duration()); diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTask.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTask.java index 198912646cd65..440f86bafdd6b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTask.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTask.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java index c20a3d64b5543..3918f1c2f1637 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/LagDetector.java b/server/src/main/java/org/elasticsearch/cluster/coordination/LagDetector.java index 08a857de9ca4d..124c17d705378 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/LagDetector.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/LagDetector.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; @@ -23,7 +24,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.PrioritizedThrottledTaskRunner; -import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.core.TimeValue; @@ -270,9 +270,7 @@ public void onFailure(Exception e) { @Override public void onResponse(Releasable releasable) { boolean success = false; - final ThreadContext threadContext = transportService.getThreadPool().getThreadContext(); - try (ThreadContext.StoredContext ignored = threadContext.stashContext()) { - threadContext.markAsSystemContext(); + try (var ignored = transportService.getThreadPool().getThreadContext().newEmptySystemContext()) { client.execute( TransportNodesHotThreadsAction.TYPE, new NodesHotThreadsRequest( diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java b/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java index 1963a9bfe643e..ec2af973d4a4c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderHeartbeatService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderHeartbeatService.java index 70010be1258e1..a14156c43b0c9 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderHeartbeatService.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderHeartbeatService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/MasterHistory.java b/server/src/main/java/org/elasticsearch/cluster/coordination/MasterHistory.java index 720510e50c48d..25a8ac01096b2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/MasterHistory.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/MasterHistory.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/MasterHistoryService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/MasterHistoryService.java index 1b2bf81bc85c3..bf450a5f76de8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/MasterHistoryService.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/MasterHistoryService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/NoMasterBlockService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/NoMasterBlockService.java index 316daf44202c2..9a420b49d5490 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/NoMasterBlockService.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/NoMasterBlockService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeHealthCheckFailureException.java b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeHealthCheckFailureException.java index 779b388aa7a17..f1f8ec4c17b4d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeHealthCheckFailureException.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeHealthCheckFailureException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java index 9223e02fc946c..9678fa051e9ec 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; @@ -28,6 +29,7 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; import org.elasticsearch.features.FeatureService; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; @@ -38,6 +40,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; @@ -46,6 +49,7 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static org.elasticsearch.cluster.metadata.IndexMetadataVerifier.isReadOnlySupportedVersion; import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; public class NodeJoinExecutor implements ClusterStateTaskExecutor { @@ -136,8 +140,8 @@ public ClusterState execute(BatchExecutionContext batchExecutionContex DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(newState.nodes()); Map compatibilityVersionsMap = new HashMap<>(newState.compatibilityVersions()); - Map> nodeFeatures = new HashMap<>(newState.nodeFeatures()); - Set allNodesFeatures = ClusterFeatures.calculateAllNodeFeatures(nodeFeatures.values()); + Map> nodeFeatures = new HashMap<>(newState.nodeFeatures()); // as present in cluster state + Set effectiveClusterFeatures = calculateEffectiveClusterFeatures(newState.nodes(), nodeFeatures); assert nodesBuilder.isLocalNodeElectedMaster(); @@ -173,14 +177,21 @@ public ClusterState execute(BatchExecutionContext batchExecutionContex } blockForbiddenVersions(compatibilityVersions.transportVersion()); ensureNodesCompatibility(node.getVersion(), minClusterNodeVersion, maxClusterNodeVersion); - enforceNodeFeatureBarrier(node.getId(), allNodesFeatures, features); + Set newNodeEffectiveFeatures = enforceNodeFeatureBarrier(node, effectiveClusterFeatures, features); // we do this validation quite late to prevent race conditions between nodes joining and importing dangling indices // we have to reject nodes that don't support all indices we have in this cluster - ensureIndexCompatibility(node.getMinIndexVersion(), node.getMaxIndexVersion(), initialState.getMetadata()); + ensureIndexCompatibility( + node.getMinIndexVersion(), + node.getMinReadOnlyIndexVersion(), + node.getMaxIndexVersion(), + initialState.getMetadata() + ); nodesBuilder.add(node); compatibilityVersionsMap.put(node.getId(), compatibilityVersions); + // store the actual node features here, not including assumed features, as this is persisted in cluster state nodeFeatures.put(node.getId(), features); - allNodesFeatures.retainAll(features); + effectiveClusterFeatures.retainAll(newNodeEffectiveFeatures); + nodesChanged = true; minClusterNodeVersion = Version.min(minClusterNodeVersion, node.getVersion()); maxClusterNodeVersion = Version.max(maxClusterNodeVersion, node.getVersion()); @@ -354,14 +365,49 @@ private static void blockForbiddenVersions(TransportVersion joiningTransportVers } } + /** + * Calculate the cluster's effective features. This includes all features that are assumed on any nodes in the cluster, + * that are also present across the whole cluster as a result. + */ + private Set calculateEffectiveClusterFeatures(DiscoveryNodes nodes, Map> nodeFeatures) { + if (featureService.featuresCanBeAssumedForNodes(nodes)) { + Set assumedFeatures = featureService.getNodeFeatures() + .values() + .stream() + .filter(NodeFeature::assumedAfterNextCompatibilityBoundary) + .map(NodeFeature::id) + .collect(Collectors.toSet()); + + // add all assumed features to the featureset of all nodes of the next major version + nodeFeatures = new HashMap<>(nodeFeatures); + for (var node : nodes.getNodes().entrySet()) { + if (featureService.featuresCanBeAssumedForNode(node.getValue())) { + assert nodeFeatures.containsKey(node.getKey()) : "Node " + node.getKey() + " does not have any features"; + nodeFeatures.computeIfPresent(node.getKey(), (k, v) -> { + var newFeatures = new HashSet<>(v); + return newFeatures.addAll(assumedFeatures) ? newFeatures : v; + }); + } + } + } + + return ClusterFeatures.calculateAllNodeFeatures(nodeFeatures.values()); + } + /** * Ensures that all indices are compatible with the given index version. This will ensure that all indices in the given metadata * will not be created with a newer version of elasticsearch as well as that all indices are newer or equal to the minimum index * compatibility version. * @see IndexVersions#MINIMUM_COMPATIBLE + * @see IndexVersions#MINIMUM_READONLY_COMPATIBLE * @throws IllegalStateException if any index is incompatible with the given version */ - public static void ensureIndexCompatibility(IndexVersion minSupportedVersion, IndexVersion maxSupportedVersion, Metadata metadata) { + public static void ensureIndexCompatibility( + IndexVersion minSupportedVersion, + IndexVersion minReadOnlySupportedVersion, + IndexVersion maxSupportedVersion, + Metadata metadata + ) { // we ensure that all indices in the cluster we join are compatible with us no matter if they are // closed or not we can't read mappings of these indices so we need to reject the join... for (IndexMetadata idxMetadata : metadata) { @@ -376,14 +422,17 @@ public static void ensureIndexCompatibility(IndexVersion minSupportedVersion, In ); } if (idxMetadata.getCompatibilityVersion().before(minSupportedVersion)) { - throw new IllegalStateException( - "index " - + idxMetadata.getIndex() - + " version not supported: " - + idxMetadata.getCompatibilityVersion().toReleaseVersion() - + " minimum compatible index version is: " - + minSupportedVersion.toReleaseVersion() - ); + boolean isReadOnlySupported = isReadOnlySupportedVersion(idxMetadata, minSupportedVersion, minReadOnlySupportedVersion); + if (isReadOnlySupported == false) { + throw new IllegalStateException( + "index " + + idxMetadata.getIndex() + + " version not supported: " + + idxMetadata.getCompatibilityVersion().toReleaseVersion() + + " minimum compatible index version is: " + + minSupportedVersion.toReleaseVersion() + ); + } } } } @@ -460,13 +509,44 @@ public static void ensureVersionBarrier(Version joiningNodeVersion, Version minC } } - private void enforceNodeFeatureBarrier(String nodeId, Set existingNodesFeatures, Set newNodeFeatures) { + /** + * Enforces the feature join barrier - a joining node should have all features already present in all existing nodes in the cluster + * + * @return The set of features that this node has (including assumed features) + */ + private Set enforceNodeFeatureBarrier(DiscoveryNode node, Set effectiveClusterFeatures, Set newNodeFeatures) { // prevent join if it does not have one or more features that all other nodes have - Set missingFeatures = new HashSet<>(existingNodesFeatures); + Set missingFeatures = new HashSet<>(effectiveClusterFeatures); missingFeatures.removeAll(newNodeFeatures); - if (missingFeatures.isEmpty() == false) { - throw new IllegalStateException("Node " + nodeId + " is missing required features " + missingFeatures); + if (missingFeatures.isEmpty()) { + // nothing missing - all ok + return newNodeFeatures; + } + + if (featureService.featuresCanBeAssumedForNode(node)) { + // it might still be ok for this node to join if this node can have assumed features, + // and all the missing features are assumed + // we can get the NodeFeature object direct from this node's registered features + // as all existing nodes in the cluster have the features present in existingNodesFeatures, including this one + newNodeFeatures = new HashSet<>(newNodeFeatures); + for (Iterator it = missingFeatures.iterator(); it.hasNext();) { + String feature = it.next(); + NodeFeature nf = featureService.getNodeFeatures().get(feature); + if (nf.assumedAfterNextCompatibilityBoundary()) { + // its ok for this feature to be missing from this node + it.remove(); + // and it should be assumed to still be in the cluster + newNodeFeatures.add(feature); + } + // even if we don't remove it, still continue, so the exception message below is accurate + } + } + + if (missingFeatures.isEmpty()) { + return newNodeFeatures; + } else { + throw new IllegalStateException("Node " + node.getId() + " is missing required features " + missingFeatures); } } @@ -476,7 +556,12 @@ public static Collection> addBuiltInJoin final Collection> validators = new ArrayList<>(); validators.add((node, state) -> { ensureNodesCompatibility(node.getVersion(), state.getNodes()); - ensureIndexCompatibility(node.getMinIndexVersion(), node.getMaxIndexVersion(), state.getMetadata()); + ensureIndexCompatibility( + node.getMinIndexVersion(), + node.getMinReadOnlyIndexVersion(), + node.getMaxIndexVersion(), + state.getMetadata() + ); }); validators.addAll(onJoinValidators); return Collections.unmodifiableCollection(validators); diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeLeftExecutor.java b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeLeftExecutor.java index 5cbe742aec628..c74c9fd63b2ba 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeLeftExecutor.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeLeftExecutor.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeToolCli.java b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeToolCli.java index 81044e8e3ad51..70aa8313db155 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeToolCli.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeToolCli.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeToolCliProvider.java b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeToolCliProvider.java index fa3dca5bc44d4..67c8b7aa65752 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeToolCliProvider.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeToolCliProvider.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/PeersResponse.java b/server/src/main/java/org/elasticsearch/cluster/coordination/PeersResponse.java index e00edafba44a5..c366dd5bcef39 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/PeersResponse.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/PeersResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/PendingClusterStateStats.java b/server/src/main/java/org/elasticsearch/cluster/coordination/PendingClusterStateStats.java index b0a62e8fa90b8..2a6172cc08f44 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/PendingClusterStateStats.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/PendingClusterStateStats.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/PreVoteCollector.java b/server/src/main/java/org/elasticsearch/cluster/coordination/PreVoteCollector.java index b17eb731cfc6a..e8bc469737ab4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/PreVoteCollector.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/PreVoteCollector.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/PreVoteRequest.java b/server/src/main/java/org/elasticsearch/cluster/coordination/PreVoteRequest.java index 161d459cad455..630bf4abe18bd 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/PreVoteRequest.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/PreVoteRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/PreVoteResponse.java b/server/src/main/java/org/elasticsearch/cluster/coordination/PreVoteResponse.java index df4d81eeb88f5..1f9729e1cc13d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/PreVoteResponse.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/PreVoteResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Publication.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Publication.java index 6afb85bdf629e..9c716e9fe37da 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Publication.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Publication.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java index 227e78ed37114..af3fdc317c8a7 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/PublishClusterStateStats.java b/server/src/main/java/org/elasticsearch/cluster/coordination/PublishClusterStateStats.java index 34a7f7557b55a..7d7d417f4ce35 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/PublishClusterStateStats.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/PublishClusterStateStats.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/PublishRequest.java b/server/src/main/java/org/elasticsearch/cluster/coordination/PublishRequest.java index 9541df0dbcf18..417651e375b14 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/PublishRequest.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/PublishRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/PublishResponse.java b/server/src/main/java/org/elasticsearch/cluster/coordination/PublishResponse.java index 604c2de29489a..fad913dd6db5d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/PublishResponse.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/PublishResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/PublishWithJoinResponse.java b/server/src/main/java/org/elasticsearch/cluster/coordination/PublishWithJoinResponse.java index 59bc1c747cc33..7aef4bb06218f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/PublishWithJoinResponse.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/PublishWithJoinResponse.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Reconfigurator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Reconfigurator.java index 2378678ec04db..dce939225d0d8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Reconfigurator.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Reconfigurator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/RemoveCustomsCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/RemoveCustomsCommand.java index 77ed4a5d52ef1..996d2c71858e7 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/RemoveCustomsCommand.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/RemoveCustomsCommand.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/RemoveIndexSettingsCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/RemoveIndexSettingsCommand.java index c6514f9cb4a0b..a2354268779e4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/RemoveIndexSettingsCommand.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/RemoveIndexSettingsCommand.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/RemoveSettingsCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/RemoveSettingsCommand.java index 1f4600538291f..4c69a2d90d45c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/RemoveSettingsCommand.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/RemoveSettingsCommand.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/StableMasterHealthIndicatorService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/StableMasterHealthIndicatorService.java index 6a15e0327d669..4919f93f0ed6b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/StableMasterHealthIndicatorService.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/StableMasterHealthIndicatorService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/StartJoinRequest.java b/server/src/main/java/org/elasticsearch/cluster/coordination/StartJoinRequest.java index cb492f39f9337..68e694ca63723 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/StartJoinRequest.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/StartJoinRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/StatefulPreVoteCollector.java b/server/src/main/java/org/elasticsearch/cluster/coordination/StatefulPreVoteCollector.java index bf33f97f2ad42..3d33cb30cad7b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/StatefulPreVoteCollector.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/StatefulPreVoteCollector.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/TermVersionRequest.java b/server/src/main/java/org/elasticsearch/cluster/coordination/TermVersionRequest.java index 3decb9ca2a84c..878abd66abfa2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/TermVersionRequest.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/TermVersionRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java index a094fbee07e1f..20e02657d2b13 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/ValidateJoinRequest.java b/server/src/main/java/org/elasticsearch/cluster/coordination/ValidateJoinRequest.java index 20c1139884b24..1d99f28e62582 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/ValidateJoinRequest.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/ValidateJoinRequest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/stateless/AtomicRegisterPreVoteCollector.java b/server/src/main/java/org/elasticsearch/cluster/coordination/stateless/AtomicRegisterPreVoteCollector.java index e9659bde065d7..aaf158349f10e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/stateless/AtomicRegisterPreVoteCollector.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/stateless/AtomicRegisterPreVoteCollector.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination.stateless; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/stateless/Heartbeat.java b/server/src/main/java/org/elasticsearch/cluster/coordination/stateless/Heartbeat.java index 8aff66f4889c2..853f83cf02172 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/stateless/Heartbeat.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/stateless/Heartbeat.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination.stateless; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/stateless/HeartbeatStore.java b/server/src/main/java/org/elasticsearch/cluster/coordination/stateless/HeartbeatStore.java index 38204b846d65e..cdbef68135e29 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/stateless/HeartbeatStore.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/stateless/HeartbeatStore.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination.stateless; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/stateless/SingleNodeReconfigurator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/stateless/SingleNodeReconfigurator.java index ec6b34b81beeb..e7b40882a9be0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/stateless/SingleNodeReconfigurator.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/stateless/SingleNodeReconfigurator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination.stateless; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/stateless/StoreHeartbeatService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/stateless/StoreHeartbeatService.java index 5bdff09c2b8b3..a3cd727421aa7 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/stateless/StoreHeartbeatService.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/stateless/StoreHeartbeatService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.coordination.stateless; diff --git a/server/src/main/java/org/elasticsearch/cluster/desirednodes/VersionConflictException.java b/server/src/main/java/org/elasticsearch/cluster/desirednodes/VersionConflictException.java index 10fbc9b0a0232..a21e565750dd4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/desirednodes/VersionConflictException.java +++ b/server/src/main/java/org/elasticsearch/cluster/desirednodes/VersionConflictException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.desirednodes; diff --git a/server/src/main/java/org/elasticsearch/cluster/features/NodeFeaturesFixupListener.java b/server/src/main/java/org/elasticsearch/cluster/features/NodeFeaturesFixupListener.java index c8b2555c0f15d..759821019152f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/features/NodeFeaturesFixupListener.java +++ b/server/src/main/java/org/elasticsearch/cluster/features/NodeFeaturesFixupListener.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.features; diff --git a/server/src/main/java/org/elasticsearch/cluster/health/ClusterHealthStatus.java b/server/src/main/java/org/elasticsearch/cluster/health/ClusterHealthStatus.java index c53395b5d76c1..13bd4e8c3f260 100644 --- a/server/src/main/java/org/elasticsearch/cluster/health/ClusterHealthStatus.java +++ b/server/src/main/java/org/elasticsearch/cluster/health/ClusterHealthStatus.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.health; diff --git a/server/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java b/server/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java index 46273e92b84b2..9cf567c219660 100644 --- a/server/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java +++ b/server/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.health; @@ -110,7 +111,7 @@ public ClusterIndexHealth(final StreamInput in) throws IOException { unassignedShards = in.readVInt(); status = ClusterHealthStatus.readFrom(in); shards = in.readMapValues(ClusterShardHealth::new, ClusterShardHealth::getShardId); - if (in.getTransportVersion().onOrAfter(TransportVersions.UNASSIGNED_PRIMARY_COUNT_ON_CLUSTER_HEALTH)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { unassignedPrimaryShards = in.readVInt(); } else { unassignedPrimaryShards = 0; @@ -202,7 +203,7 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeVInt(unassignedShards); out.writeByte(status.value()); out.writeMapValues(shards); - if (out.getTransportVersion().onOrAfter(TransportVersions.UNASSIGNED_PRIMARY_COUNT_ON_CLUSTER_HEALTH)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeVInt(unassignedPrimaryShards); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java b/server/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java index 7ebd90050bff3..f512acb6e04d0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java +++ b/server/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.health; @@ -95,7 +96,7 @@ public ClusterShardHealth(final StreamInput in) throws IOException { initializingShards = in.readVInt(); unassignedShards = in.readVInt(); primaryActive = in.readBoolean(); - if (in.getTransportVersion().onOrAfter(TransportVersions.UNASSIGNED_PRIMARY_COUNT_ON_CLUSTER_HEALTH)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { unassignedPrimaryShards = in.readVInt(); } else { unassignedPrimaryShards = 0; @@ -166,7 +167,7 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeVInt(initializingShards); out.writeVInt(unassignedShards); out.writeBoolean(primaryActive); - if (out.getTransportVersion().onOrAfter(TransportVersions.UNASSIGNED_PRIMARY_COUNT_ON_CLUSTER_HEALTH)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeVInt(unassignedPrimaryShards); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java b/server/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java index 3f27d8d7c2c63..31f275e29c368 100644 --- a/server/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java +++ b/server/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.health; @@ -119,7 +120,7 @@ public ClusterStateHealth(final StreamInput in) throws IOException { status = ClusterHealthStatus.readFrom(in); indices = in.readMapValues(ClusterIndexHealth::new, ClusterIndexHealth::getIndex); activeShardsPercent = in.readDouble(); - if (in.getTransportVersion().onOrAfter(TransportVersions.UNASSIGNED_PRIMARY_COUNT_ON_CLUSTER_HEALTH)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { unassignedPrimaryShards = in.readVInt(); } else { unassignedPrimaryShards = 0; @@ -211,7 +212,7 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeByte(status.value()); out.writeMapValues(indices); out.writeDouble(activeShardsPercent); - if (out.getTransportVersion().onOrAfter(TransportVersions.UNASSIGNED_PRIMARY_COUNT_ON_CLUSTER_HEALTH)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { out.writeVInt(unassignedPrimaryShards); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasAction.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasAction.java index 533ae3a3ad50d..a106ef937e156 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasAction.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasInfo.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasInfo.java index 3c6db4fb39b1b..3de7eb5611a59 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasInfo.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasInfo.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java index ff412d629b3b1..92d886edda5e5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java index 6147712a5d70a..890a1ab2b1151 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AutoExpandReplicas.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AutoExpandReplicas.java index 9fb44e0106dfe..ef28a46d423da 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AutoExpandReplicas.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AutoExpandReplicas.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ClusterNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ClusterNameExpressionResolver.java index eddbedaf310bf..7d7306db786a6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ClusterNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ClusterNameExpressionResolver.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java index d3d758e110ff3..016fec60f06be 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java index 71966deb076b6..1151a99a24403 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; @@ -16,7 +17,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContent; @@ -101,8 +102,8 @@ public static ComponentTemplateMetadata fromXContent(XContentParser parser) thro } @Override - public Iterator toXContentChunked(ToXContent.Params ignored) { - return ChunkedToXContentHelper.xContentValuesMap(COMPONENT_TEMPLATE.getPreferredName(), componentTemplates); + public Iterator toXContentChunked(ToXContent.Params params) { + return ChunkedToXContent.builder(params).xContentObjectFields(COMPONENT_TEMPLATE.getPreferredName(), componentTemplates); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index fd1019efd7b78..3dd50d4e386d3 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; @@ -27,6 +28,8 @@ import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; @@ -188,9 +191,14 @@ public List getRequiredComponentTemplates() { if (ignoreMissingComponentTemplates == null) { return componentTemplates; } - return componentTemplates.stream() - .filter(componentTemplate -> ignoreMissingComponentTemplates.contains(componentTemplate) == false) - .toList(); + // note: this loop is unrolled rather than streaming-style because it's hot enough to show up in a flamegraph + List required = new ArrayList<>(componentTemplates.size()); + for (String template : componentTemplates) { + if (ignoreMissingComponentTemplates.contains(template) == false) { + required.add(template); + } + } + return Collections.unmodifiableList(required); } @Nullable @@ -364,16 +372,17 @@ public static class DataStreamTemplate implements Writeable, ToXContentObject { private static final ParseField HIDDEN = new ParseField("hidden"); private static final ParseField ALLOW_CUSTOM_ROUTING = new ParseField("allow_custom_routing"); + + /** + * Use the {@link DataStreamFailureStore.Template#enabled()} instead + */ + @Deprecated(since = "8.18") private static final ParseField FAILURE_STORE = new ParseField("failure_store"); public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "data_stream_template", false, - args -> new DataStreamTemplate( - args[0] != null && (boolean) args[0], - args[1] != null && (boolean) args[1], - DataStream.isFailureStoreFeatureFlagEnabled() && args[2] != null && (boolean) args[2] - ) + args -> new DataStreamTemplate(args[0] != null && (boolean) args[0], args[1] != null && (boolean) args[1]) ); static { @@ -386,20 +395,14 @@ public static class DataStreamTemplate implements Writeable, ToXContentObject { private final boolean hidden; private final boolean allowCustomRouting; - private final boolean failureStore; public DataStreamTemplate() { - this(false, false, false); + this(false, false); } public DataStreamTemplate(boolean hidden, boolean allowCustomRouting) { - this(hidden, allowCustomRouting, false); - } - - public DataStreamTemplate(boolean hidden, boolean allowCustomRouting, boolean failureStore) { this.hidden = hidden; this.allowCustomRouting = allowCustomRouting; - this.failureStore = failureStore; } DataStreamTemplate(StreamInput in) throws IOException { @@ -417,10 +420,9 @@ public DataStreamTemplate(boolean hidden, boolean allowCustomRouting, boolean fa boolean value = in.readBoolean(); assert value == false : "expected false, because this used to be an optional enum that never got set"; } - if (in.getTransportVersion().onOrAfter(DataStream.ADDED_FAILURE_STORE_TRANSPORT_VERSION)) { - failureStore = in.readBoolean(); - } else { - failureStore = false; + if (in.getTransportVersion() + .between(DataStream.ADDED_FAILURE_STORE_TRANSPORT_VERSION, TransportVersions.ADD_DATA_STREAM_OPTIONS_TO_TEMPLATES)) { + in.readBoolean(); } } @@ -450,10 +452,6 @@ public boolean isAllowCustomRouting() { return allowCustomRouting; } - public boolean hasFailureStore() { - return failureStore; - } - @Override public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(hidden); @@ -464,8 +462,11 @@ public void writeTo(StreamOutput out) throws IOException { // See comment in constructor. out.writeBoolean(false); } - if (out.getTransportVersion().onOrAfter(DataStream.ADDED_FAILURE_STORE_TRANSPORT_VERSION)) { - out.writeBoolean(failureStore); + if (out.getTransportVersion() + .between(DataStream.ADDED_FAILURE_STORE_TRANSPORT_VERSION, TransportVersions.ADD_DATA_STREAM_OPTIONS_TO_TEMPLATES)) { + // Previous versions expect the failure store to be configured via the DataStreamTemplate. We add it here, so we don't break + // the serialisation, but we do not care to preserve the value because this feature is still behind a feature flag. + out.writeBoolean(false); } } @@ -474,9 +475,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); builder.field("hidden", hidden); builder.field(ALLOW_CUSTOM_ROUTING.getPreferredName(), allowCustomRouting); - if (DataStream.isFailureStoreFeatureFlagEnabled()) { - builder.field(FAILURE_STORE.getPreferredName(), failureStore); - } builder.endObject(); return builder; } @@ -486,12 +484,12 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; DataStreamTemplate that = (DataStreamTemplate) o; - return hidden == that.hidden && allowCustomRouting == that.allowCustomRouting && failureStore == that.failureStore; + return hidden == that.hidden && allowCustomRouting == that.allowCustomRouting; } @Override public int hashCode() { - return Objects.hash(hidden, allowCustomRouting, failureStore); + return Objects.hash(hidden, allowCustomRouting); } } @@ -536,6 +534,11 @@ public Builder template(Template template) { return this; } + public Builder template(Template.Builder template) { + this.template = template.build(); + return this; + } + public Builder componentTemplates(List componentTemplates) { this.componentTemplates = componentTemplates; return this; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java index 8fd606048d539..e798b0f6add4f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; @@ -16,7 +17,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContent; @@ -102,8 +103,8 @@ public void writeTo(StreamOutput out) throws IOException { } @Override - public Iterator toXContentChunked(ToXContent.Params ignored) { - return ChunkedToXContentHelper.xContentValuesMap(INDEX_TEMPLATE.getPreferredName(), indexTemplates); + public Iterator toXContentChunked(ToXContent.Params params) { + return ChunkedToXContent.builder(params).xContentObjectFields(INDEX_TEMPLATE.getPreferredName(), indexTemplates); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index 8acee4f6be821..c1b015dc3700b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; @@ -36,6 +37,7 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContentObject; @@ -48,7 +50,6 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; -import java.util.Collection; import java.util.Comparator; import java.util.HashMap; import java.util.List; @@ -70,6 +71,7 @@ public final class DataStream implements SimpleDiffable, ToXContentO public static final FeatureFlag FAILURE_STORE_FEATURE_FLAG = new FeatureFlag("failure_store"); public static final TransportVersion ADDED_FAILURE_STORE_TRANSPORT_VERSION = TransportVersions.V_8_12_0; public static final TransportVersion ADDED_AUTO_SHARDING_EVENT_VERSION = TransportVersions.V_8_14_0; + public static final TransportVersion ADD_DATA_STREAM_OPTIONS_VERSION = TransportVersions.V_8_16_0; public static boolean isFailureStoreFeatureFlagEnabled() { return FAILURE_STORE_FEATURE_FLAG.isEnabled(); @@ -111,11 +113,12 @@ public static boolean isFailureStoreFeatureFlagEnabled() { private final IndexMode indexMode; @Nullable private final DataStreamLifecycle lifecycle; - private final boolean failureStoreEnabled; + private final DataStreamOptions dataStreamOptions; private final DataStreamIndices backingIndices; private final DataStreamIndices failureIndices; + // visible for testing public DataStream( String name, List indices, @@ -127,7 +130,7 @@ public DataStream( boolean allowCustomRouting, IndexMode indexMode, DataStreamLifecycle lifecycle, - boolean failureStoreEnabled, + @Nullable DataStreamOptions dataStreamOptions, List failureIndices, boolean rolloverOnWrite, @Nullable DataStreamAutoShardingEvent autoShardingEvent @@ -143,13 +146,12 @@ public DataStream( allowCustomRouting, indexMode, lifecycle, - failureStoreEnabled, + dataStreamOptions, new DataStreamIndices(BACKING_INDEX_PREFIX, List.copyOf(indices), rolloverOnWrite, autoShardingEvent), new DataStreamIndices(FAILURE_STORE_PREFIX, List.copyOf(failureIndices), false, null) ); } - // visible for testing DataStream( String name, long generation, @@ -161,7 +163,7 @@ public DataStream( boolean allowCustomRouting, IndexMode indexMode, DataStreamLifecycle lifecycle, - boolean failureStoreEnabled, + DataStreamOptions dataStreamOptions, DataStreamIndices backingIndices, DataStreamIndices failureIndices ) { @@ -176,7 +178,7 @@ public DataStream( this.allowCustomRouting = allowCustomRouting; this.indexMode = indexMode; this.lifecycle = lifecycle; - this.failureStoreEnabled = failureStoreEnabled; + this.dataStreamOptions = dataStreamOptions == null ? DataStreamOptions.EMPTY : dataStreamOptions; assert backingIndices.indices.isEmpty() == false; assert replicated == false || (backingIndices.rolloverOnWrite == false && failureIndices.rolloverOnWrite == false) : "replicated data streams cannot be marked for lazy rollover"; @@ -197,9 +199,9 @@ public static DataStream read(StreamInput in) throws IOException { var lifecycle = in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X) ? in.readOptionalWriteable(DataStreamLifecycle::new) : null; - var failureStoreEnabled = in.getTransportVersion().onOrAfter(DataStream.ADDED_FAILURE_STORE_TRANSPORT_VERSION) - ? in.readBoolean() - : false; + // This boolean flag has been moved in data stream options + var failureStoreEnabled = in.getTransportVersion() + .between(DataStream.ADDED_FAILURE_STORE_TRANSPORT_VERSION, TransportVersions.V_8_16_0) ? in.readBoolean() : false; var failureIndices = in.getTransportVersion().onOrAfter(DataStream.ADDED_FAILURE_STORE_TRANSPORT_VERSION) ? readIndices(in) : List.of(); @@ -208,10 +210,18 @@ public static DataStream read(StreamInput in) throws IOException { if (in.getTransportVersion().onOrAfter(DataStream.ADDED_AUTO_SHARDING_EVENT_VERSION)) { backingIndicesBuilder.setAutoShardingEvent(in.readOptionalWriteable(DataStreamAutoShardingEvent::new)); } - if (in.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_FIELD_PARITY)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { failureIndicesBuilder.setRolloverOnWrite(in.readBoolean()) .setAutoShardingEvent(in.readOptionalWriteable(DataStreamAutoShardingEvent::new)); } + DataStreamOptions dataStreamOptions; + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { + dataStreamOptions = in.readOptionalWriteable(DataStreamOptions::read); + } else { + // We cannot distinguish if failure store was explicitly disabled or not. Given that failure store + // is still behind a feature flag in previous version we use the default value instead of explicitly disabling it. + dataStreamOptions = failureStoreEnabled ? DataStreamOptions.FAILURE_STORE_ENABLED : null; + } return new DataStream( name, generation, @@ -223,7 +233,7 @@ public static DataStream read(StreamInput in) throws IOException { allowCustomRouting, indexMode, lifecycle, - failureStoreEnabled, + dataStreamOptions, backingIndicesBuilder.build(), failureIndicesBuilder.build() ); @@ -273,6 +283,10 @@ public boolean isFailureStoreIndex(String indexName) { return failureIndices.containsIndex(indexName); } + public DataStreamOptions getDataStreamOptions() { + return dataStreamOptions; + } + public boolean rolloverOnWrite() { return backingIndices.rolloverOnWrite; } @@ -286,7 +300,15 @@ public boolean rolloverOnWrite() { * @return true if it's a system index or has a dot-prefixed name. */ public boolean isInternal() { - return isSystem() || name.charAt(0) == '.'; + return isSystem() || isDotPrefixName(name); + } + + private static boolean isInternalName(String name, SystemIndices systemIndices) { + return isDotPrefixName(name) || systemIndices.isSystemDataStream(name); + } + + private static boolean isDotPrefixName(String name) { + return name.charAt(0) == '.'; } /** @@ -405,13 +427,55 @@ public boolean isAllowCustomRouting() { } /** - * Determines if this data stream should persist ingest pipeline and mapping failures from bulk requests to a locally - * configured failure store. + * Determines whether this data stream has its failure store enabled explicitly in its metadata. + */ + public boolean isFailureStoreExplicitlyEnabled() { + return dataStreamOptions.failureStore() != null && Boolean.TRUE.equals(dataStreamOptions.failureStore().enabled()); + } + + /** + * Returns whether this data stream has its failure store enabled, either explicitly in its metadata or implicitly via settings. + * + *

    If the failure store is either explicitly enabled or explicitly disabled in its options metadata, that value is returned. If not, + * it checks whether its name matches one of the patterns in the settings, and that the data stream is not internal (i.e. neither a + * dot-prefixed nor a system data stream). + * + * @param dataStreamFailureStoreSettings The settings to use to determine whether the failure store should be implicitly enabled + */ + public boolean isFailureStoreEffectivelyEnabled(DataStreamFailureStoreSettings dataStreamFailureStoreSettings) { + return isFailureStoreEffectivelyEnabled(dataStreamOptions, dataStreamFailureStoreSettings, name, isInternal()); + } + + /** + * Returns whether a data stream has its failure store enabled, either explicitly in its metadata or implicitly via settings, based + * on the given parameters. The logic is equivalent to that in + * {@link #isFailureStoreEffectivelyEnabled(DataStreamFailureStoreSettings)}. * - * @return Whether this data stream should store ingestion failures. + * @param options The {@link DataStreamOptions} for the data stream (which may be null) + * @param dataStreamFailureStoreSettings The settings to use to determine whether the failure store should be implicitly enabled + * @param name The name of the data stream + * @param systemIndices The {@link SystemIndices} instance to use to determine whether this is a system data stream */ - public boolean isFailureStoreEnabled() { - return failureStoreEnabled; + public static boolean isFailureStoreEffectivelyEnabled( + @Nullable DataStreamOptions options, + DataStreamFailureStoreSettings dataStreamFailureStoreSettings, + String name, + SystemIndices systemIndices + ) { + return isFailureStoreEffectivelyEnabled(options, dataStreamFailureStoreSettings, name, isInternalName(name, systemIndices)); + } + + private static boolean isFailureStoreEffectivelyEnabled( + DataStreamOptions options, + DataStreamFailureStoreSettings dataStreamFailureStoreSettings, + String name, + boolean isInternal + ) { + if (options != null && options.failureStore() != null && options.failureStore().enabled() != null) { + return options.failureStore().enabled(); + } else { + return (isInternal == false) && dataStreamFailureStoreSettings.failureStoreEnabledForDataStreamName(name); + } } @Nullable @@ -447,43 +511,52 @@ public DataStreamIndices getDataStreamIndices(boolean failureStore) { * Performs a rollover on a {@code DataStream} instance and returns a new instance containing * the updated list of backing indices and incremented generation. * - * @param writeIndex new write index - * @param generation new generation - * @param timeSeries whether the template that created this data stream is in time series mode - * @param autoShardingEvent the auto sharding event this rollover operation is applying - * + * @param writeIndex new write index + * @param generation new generation + * @param indexModeFromTemplate the index mode that originates from the template that created this data stream + * @param autoShardingEvent the auto sharding event this rollover operation is applying * @return new {@code DataStream} instance with the rollover operation applied */ public DataStream rollover( Index writeIndex, long generation, - boolean timeSeries, + IndexMode indexModeFromTemplate, @Nullable DataStreamAutoShardingEvent autoShardingEvent ) { ensureNotReplicated(); - return unsafeRollover(writeIndex, generation, timeSeries, autoShardingEvent); + return unsafeRollover(writeIndex, generation, indexModeFromTemplate, autoShardingEvent); } /** - * Like {@link #rollover(Index, long, boolean, DataStreamAutoShardingEvent)}, but does no validation, use with care only. + * Like {@link #rollover(Index, long, IndexMode, DataStreamAutoShardingEvent)}, but does no validation, use with care only. */ - public DataStream unsafeRollover(Index writeIndex, long generation, boolean timeSeries, DataStreamAutoShardingEvent autoShardingEvent) { - IndexMode indexMode = this.indexMode; - if ((indexMode == null || indexMode == IndexMode.STANDARD) && timeSeries) { + public DataStream unsafeRollover( + Index writeIndex, + long generation, + IndexMode indexModeFromTemplate, + DataStreamAutoShardingEvent autoShardingEvent + ) { + IndexMode dsIndexMode = this.indexMode; + if ((dsIndexMode == null || dsIndexMode == IndexMode.STANDARD) && indexModeFromTemplate == IndexMode.TIME_SERIES) { // This allows for migrating a data stream to be a tsdb data stream: // (only if index_mode=null|standard then allow it to be set to time_series) - indexMode = IndexMode.TIME_SERIES; - } else if (indexMode == IndexMode.TIME_SERIES && timeSeries == false) { + dsIndexMode = IndexMode.TIME_SERIES; + } else if (dsIndexMode == IndexMode.TIME_SERIES && (indexModeFromTemplate == null || indexModeFromTemplate == IndexMode.STANDARD)) { // Allow downgrading a time series data stream to a regular data stream - indexMode = null; + dsIndexMode = null; + } else if ((dsIndexMode == null || dsIndexMode == IndexMode.STANDARD) && indexModeFromTemplate == IndexMode.LOGSDB) { + dsIndexMode = IndexMode.LOGSDB; + } else if (dsIndexMode == IndexMode.LOGSDB && (indexModeFromTemplate == null || indexModeFromTemplate == IndexMode.STANDARD)) { + // Allow downgrading a time series data stream to a regular data stream + dsIndexMode = null; } List backingIndices = new ArrayList<>(this.backingIndices.indices); backingIndices.add(writeIndex); return copy().setBackingIndices( this.backingIndices.copy().setIndices(backingIndices).setAutoShardingEvent(autoShardingEvent).setRolloverOnWrite(false).build() - ).setGeneration(generation).setIndexMode(indexMode).build(); + ).setGeneration(generation).setIndexMode(dsIndexMode).build(); } /** @@ -772,27 +845,57 @@ public DataStream promoteDataStream() { /** * Reconciles this data stream with a list of indices available in a snapshot. Allows snapshots to store accurate data - * stream definitions that do not reference backing indices not contained in the snapshot. + * stream definitions that do not reference backing indices and failure indices not contained in the snapshot. * * @param indicesInSnapshot List of indices in the snapshot + * @param snapshotMetadataBuilder a metadata builder with the current view of the snapshot metadata * @return Reconciled {@link DataStream} instance or {@code null} if no reconciled version of this data stream could be built from the * given indices */ @Nullable - public DataStream snapshot(Collection indicesInSnapshot) { + public DataStream snapshot(Set indicesInSnapshot, Metadata.Builder snapshotMetadataBuilder) { + boolean backingIndicesChanged = false; + boolean failureIndicesChanged = false; + // do not include indices not available in the snapshot - List reconciledIndices = new ArrayList<>(this.backingIndices.indices); - if (reconciledIndices.removeIf(x -> indicesInSnapshot.contains(x.getName()) == false) == false) { + List reconciledBackingIndices = this.backingIndices.indices; + if (isAnyIndexMissing(this.backingIndices.getIndices(), snapshotMetadataBuilder, indicesInSnapshot)) { + reconciledBackingIndices = new ArrayList<>(this.backingIndices.indices); + backingIndicesChanged = reconciledBackingIndices.removeIf(x -> indicesInSnapshot.contains(x.getName()) == false); + if (reconciledBackingIndices.isEmpty()) { + return null; + } + } + + List reconciledFailureIndices = this.failureIndices.indices; + if (DataStream.isFailureStoreFeatureFlagEnabled() + && isAnyIndexMissing(failureIndices.indices, snapshotMetadataBuilder, indicesInSnapshot)) { + reconciledFailureIndices = new ArrayList<>(this.failureIndices.indices); + failureIndicesChanged = reconciledFailureIndices.removeIf(x -> indicesInSnapshot.contains(x.getName()) == false); + } + + if (backingIndicesChanged == false && failureIndicesChanged == false) { return this; } - if (reconciledIndices.size() == 0) { - return null; + Builder builder = copy(); + if (backingIndicesChanged) { + builder.setBackingIndices(backingIndices.copy().setIndices(reconciledBackingIndices).build()); } + if (failureIndicesChanged) { + builder.setFailureIndices(failureIndices.copy().setIndices(reconciledFailureIndices).build()); + } + return builder.setMetadata(metadata == null ? null : new HashMap<>(metadata)).build(); + } - return copy().setBackingIndices(backingIndices.copy().setIndices(reconciledIndices).build()) - .setMetadata(metadata == null ? null : new HashMap<>(metadata)) - .build(); + private static boolean isAnyIndexMissing(List indices, Metadata.Builder builder, Set indicesInSnapshot) { + for (Index index : indices) { + final String indexName = index.getName(); + if (builder.get(indexName) == null || indicesInSnapshot.contains(indexName) == false) { + return true; + } + } + return false; } /** @@ -1053,8 +1156,11 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { out.writeOptionalWriteable(lifecycle); } + if (out.getTransportVersion() + .between(DataStream.ADDED_FAILURE_STORE_TRANSPORT_VERSION, DataStream.ADD_DATA_STREAM_OPTIONS_VERSION)) { + out.writeBoolean(isFailureStoreExplicitlyEnabled()); + } if (out.getTransportVersion().onOrAfter(DataStream.ADDED_FAILURE_STORE_TRANSPORT_VERSION)) { - out.writeBoolean(failureStoreEnabled); out.writeCollection(failureIndices.indices); } if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) { @@ -1063,10 +1169,13 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(DataStream.ADDED_AUTO_SHARDING_EVENT_VERSION)) { out.writeOptionalWriteable(backingIndices.autoShardingEvent); } - if (out.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_FIELD_PARITY)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { out.writeBoolean(failureIndices.rolloverOnWrite); out.writeOptionalWriteable(failureIndices.autoShardingEvent); } + if (out.getTransportVersion().onOrAfter(DataStream.ADD_DATA_STREAM_OPTIONS_VERSION)) { + out.writeOptionalWriteable(dataStreamOptions.isEmpty() ? null : dataStreamOptions); + } } public static final ParseField NAME_FIELD = new ParseField("name"); @@ -1086,6 +1195,7 @@ public void writeTo(StreamOutput out) throws IOException { public static final ParseField AUTO_SHARDING_FIELD = new ParseField("auto_sharding"); public static final ParseField FAILURE_ROLLOVER_ON_WRITE_FIELD = new ParseField("failure_rollover_on_write"); public static final ParseField FAILURE_AUTO_SHARDING_FIELD = new ParseField("failure_auto_sharding"); + public static final ParseField DATA_STREAM_OPTIONS_FIELD = new ParseField("options"); @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("data_stream", args -> { @@ -1100,6 +1210,16 @@ public void writeTo(StreamOutput out) throws IOException { (DataStreamAutoShardingEvent) args[15] ) : new DataStreamIndices(FAILURE_STORE_PREFIX, List.of(), false, null); + // We cannot distinguish if failure store was explicitly disabled or not. Given that failure store + // is still behind a feature flag in previous version we use the default value instead of explicitly disabling it. + DataStreamOptions dataStreamOptions = DataStreamOptions.EMPTY; + if (DataStream.isFailureStoreFeatureFlagEnabled()) { + if (args[16] != null) { + dataStreamOptions = (DataStreamOptions) args[16]; + } else if (failureStoreEnabled) { + dataStreamOptions = DataStreamOptions.FAILURE_STORE_ENABLED; + } + } return new DataStream( (String) args[0], (Long) args[2], @@ -1111,7 +1231,7 @@ public void writeTo(StreamOutput out) throws IOException { args[7] != null && (boolean) args[7], args[8] != null ? IndexMode.fromString((String) args[8]) : null, (DataStreamLifecycle) args[9], - failureStoreEnabled, + dataStreamOptions, new DataStreamIndices( BACKING_INDEX_PREFIX, (List) args[1], @@ -1149,6 +1269,7 @@ public void writeTo(StreamOutput out) throws IOException { ); // The fields behind the feature flag should always be last. if (DataStream.isFailureStoreFeatureFlagEnabled()) { + // Should be removed after backport PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), FAILURE_STORE_FIELD); PARSER.declareObjectArray( ConstructingObjectParser.optionalConstructorArg(), @@ -1161,6 +1282,11 @@ public void writeTo(StreamOutput out) throws IOException { (p, c) -> DataStreamAutoShardingEvent.fromXContent(p), FAILURE_AUTO_SHARDING_FIELD ); + PARSER.declareObject( + ConstructingObjectParser.optionalConstructorArg(), + (p, c) -> DataStreamOptions.fromXContent(p), + DATA_STREAM_OPTIONS_FIELD + ); } } @@ -1198,7 +1324,6 @@ public XContentBuilder toXContent( builder.field(SYSTEM_FIELD.getPreferredName(), system); builder.field(ALLOW_CUSTOM_ROUTING.getPreferredName(), allowCustomRouting); if (DataStream.isFailureStoreFeatureFlagEnabled()) { - builder.field(FAILURE_STORE_FIELD.getPreferredName(), failureStoreEnabled); if (failureIndices.indices.isEmpty() == false) { builder.xContentList(FAILURE_INDICES_FIELD.getPreferredName(), failureIndices.indices); } @@ -1208,6 +1333,10 @@ public XContentBuilder toXContent( failureIndices.autoShardingEvent.toXContent(builder, params); builder.endObject(); } + if (dataStreamOptions.isEmpty() == false) { + builder.field(DATA_STREAM_OPTIONS_FIELD.getPreferredName()); + dataStreamOptions.toXContent(builder, params); + } } if (indexMode != null) { builder.field(INDEX_MODE.getPreferredName(), indexMode); @@ -1240,7 +1369,7 @@ public boolean equals(Object o) { && allowCustomRouting == that.allowCustomRouting && indexMode == that.indexMode && Objects.equals(lifecycle, that.lifecycle) - && failureStoreEnabled == that.failureStoreEnabled + && Objects.equals(dataStreamOptions, that.dataStreamOptions) && Objects.equals(backingIndices, that.backingIndices) && Objects.equals(failureIndices, that.failureIndices); } @@ -1257,7 +1386,7 @@ public int hashCode() { allowCustomRouting, indexMode, lifecycle, - failureStoreEnabled, + dataStreamOptions, backingIndices, failureIndices ); @@ -1295,7 +1424,7 @@ public Index getWriteIndex(IndexRequest request, Metadata metadata) { + "]" ) .collect(Collectors.joining()); - throw new IllegalArgumentException( + throw new TimestampError( "the document timestamp [" + timestampAsString + "] is outside of ranges of currently writable indices [" @@ -1313,6 +1442,7 @@ public DataStream getParentDataStream() { } public static final XContentParserConfiguration TS_EXTRACT_CONFIG = XContentParserConfiguration.EMPTY.withFiltering( + null, Set.of(TIMESTAMP_FIELD_NAME), null, false @@ -1357,10 +1487,10 @@ private static Instant getTimeStampFromRaw(Object rawTimestamp) { } else if (rawTimestamp instanceof String sTimestamp) { return DateFormatters.from(TIMESTAMP_FORMATTER.parse(sTimestamp), TIMESTAMP_FORMATTER.locale()).toInstant(); } else { - throw new IllegalArgumentException("timestamp [" + rawTimestamp + "] type [" + rawTimestamp.getClass() + "] error"); + throw new TimestampError("timestamp [" + rawTimestamp + "] type [" + rawTimestamp.getClass() + "] error"); } } catch (Exception e) { - throw new IllegalArgumentException("Error get data stream timestamp field: " + e.getMessage(), e); + throw new TimestampError("Error get data stream timestamp field: " + e.getMessage(), e); } } @@ -1384,7 +1514,7 @@ private static Instant getTimestampFromParser(BytesReference source, XContentTyp ); }; } catch (Exception e) { - throw new IllegalArgumentException("Error extracting data stream timestamp field: " + e.getMessage(), e); + throw new TimestampError("Error extracting data stream timestamp field: " + e.getMessage(), e); } } @@ -1570,7 +1700,7 @@ public static class Builder { private IndexMode indexMode = null; @Nullable private DataStreamLifecycle lifecycle = null; - private boolean failureStoreEnabled = false; + private DataStreamOptions dataStreamOptions = DataStreamOptions.EMPTY; private DataStreamIndices backingIndices; private DataStreamIndices failureIndices = DataStreamIndices.failureIndicesBuilder(List.of()).build(); @@ -1595,7 +1725,7 @@ private Builder(DataStream dataStream) { allowCustomRouting = dataStream.allowCustomRouting; indexMode = dataStream.indexMode; lifecycle = dataStream.lifecycle; - failureStoreEnabled = dataStream.failureStoreEnabled; + dataStreamOptions = dataStream.dataStreamOptions; backingIndices = dataStream.backingIndices; failureIndices = dataStream.failureIndices; } @@ -1650,8 +1780,8 @@ public Builder setLifecycle(DataStreamLifecycle lifecycle) { return this; } - public Builder setFailureStoreEnabled(boolean failureStoreEnabled) { - this.failureStoreEnabled = failureStoreEnabled; + public Builder setDataStreamOptions(DataStreamOptions dataStreamOptions) { + this.dataStreamOptions = dataStreamOptions; return this; } @@ -1687,10 +1817,26 @@ public DataStream build() { allowCustomRouting, indexMode, lifecycle, - failureStoreEnabled, + dataStreamOptions, backingIndices, failureIndices ); } } + + /** + * This is a specialised error to capture that a document does not have a valid timestamp + * to index a document. It is mainly applicable for TSDS data streams because they need the timestamp + * to determine the write index. + */ + public static class TimestampError extends IllegalArgumentException { + + public TimestampError(String message, Exception cause) { + super(message, cause); + } + + public TimestampError(String message) { + super(message); + } + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAction.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAction.java index 32bf46ce45919..545f949f962c1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAlias.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAlias.java index ba5e662cca5f9..f38ca54622a68 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAlias.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAlias.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAutoShardingEvent.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAutoShardingEvent.java index ff143681827ca..6fec15e0fbfe8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAutoShardingEvent.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAutoShardingEvent.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFactoryRetention.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFactoryRetention.java deleted file mode 100644 index be42916b07956..0000000000000 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFactoryRetention.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.cluster.metadata; - -import org.elasticsearch.common.settings.ClusterSettings; -import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.plugins.PluginsService; - -/** - * Holds the factory retention configuration. Factory retention is the global retention configuration meant to be - * used if a user hasn't provided other retention configuration via {@link DataStreamGlobalRetention} metadata in the - * cluster state. - * @deprecated This interface is deprecated, please use {@link DataStreamGlobalRetentionSettings}. - */ -@Deprecated -public interface DataStreamFactoryRetention { - - @Nullable - TimeValue getMaxRetention(); - - @Nullable - TimeValue getDefaultRetention(); - - /** - * @return true, if at least one of the two settings is not null, false otherwise. - */ - default boolean isDefined() { - return getMaxRetention() != null || getDefaultRetention() != null; - } - - /** - * Applies any post constructor initialisation, for example, listening to cluster setting changes. - */ - void init(ClusterSettings clusterSettings); - - /** - * Loads a single instance of a DataStreamFactoryRetention from the {@link PluginsService} and finalises the - * initialisation by calling {@link DataStreamFactoryRetention#init(ClusterSettings)} - */ - static DataStreamFactoryRetention load(PluginsService pluginsService, ClusterSettings clusterSettings) { - DataStreamFactoryRetention factoryRetention = pluginsService.loadSingletonServiceProvider( - DataStreamFactoryRetention.class, - DataStreamFactoryRetention::emptyFactoryRetention - ); - factoryRetention.init(clusterSettings); - return factoryRetention; - } - - /** - * Returns empty factory global retention settings. - */ - static DataStreamFactoryRetention emptyFactoryRetention() { - return new DataStreamFactoryRetention() { - - @Override - public TimeValue getMaxRetention() { - return null; - } - - @Override - public TimeValue getDefaultRetention() { - return null; - } - - @Override - public void init(ClusterSettings clusterSettings) { - - } - }; - } -} diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStore.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStore.java index d647956e752a3..5a6217eea8f7b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStore.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStore.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; @@ -13,7 +14,9 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -23,29 +26,37 @@ /** * Holds the data stream failure store metadata that enable or disable the failure store of a data stream. Currently, it - * supports the following configurations: - * - enabled + * supports the following configurations only explicitly enabling or disabling the failure store */ -public record DataStreamFailureStore(boolean enabled) implements SimpleDiffable, ToXContentObject { +public record DataStreamFailureStore(Boolean enabled) implements SimpleDiffable, ToXContentObject { + public static final String FAILURE_STORE = "failure_store"; + public static final String ENABLED = "enabled"; - public static final ParseField ENABLED_FIELD = new ParseField("enabled"); + public static final ParseField ENABLED_FIELD = new ParseField(ENABLED); public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "failure_store", + FAILURE_STORE, false, - (args, unused) -> new DataStreamFailureStore(args[0] == null || (Boolean) args[0]) + (args, unused) -> new DataStreamFailureStore((Boolean) args[0]) ); static { - PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), ENABLED_FIELD); + PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), ENABLED_FIELD); } - public DataStreamFailureStore() { - this(true); + /** + * @param enabled, true when the failure is enabled, false when it's disabled, null when it depends on other configuration. Currently, + * null value is not supported because there are no other arguments + * @throws IllegalArgumentException when all the constructor arguments are null + */ + public DataStreamFailureStore { + if (enabled == null) { + throw new IllegalArgumentException("Failure store configuration should have at least one non-null configuration value."); + } } public DataStreamFailureStore(StreamInput in) throws IOException { - this(in.readBoolean()); + this(in.readOptionalBoolean()); } public static Diff readDiffFrom(StreamInput in) throws IOException { @@ -54,7 +65,7 @@ public static Diff readDiffFrom(StreamInput in) throws I @Override public void writeTo(StreamOutput out) throws IOException { - out.writeBoolean(enabled); + out.writeOptionalBoolean(enabled); } @Override @@ -65,7 +76,9 @@ public String toString() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(ENABLED_FIELD.getPreferredName(), enabled); + if (enabled != null) { + builder.field(ENABLED_FIELD.getPreferredName(), enabled); + } builder.endObject(); return builder; } @@ -73,4 +86,80 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public static DataStreamFailureStore fromXContent(XContentParser parser) throws IOException { return PARSER.parse(parser, null); } + + /** + * This class is only used in template configuration. It wraps the fields of {@link DataStreamFailureStore} with {@link ResettableValue} + * to allow a user to signal when they want to reset any previously encountered values during template composition. Furthermore, it + * provides the method {@link #merge(Template, Template)} that dictates how two templates can be composed. + */ + public record Template(ResettableValue enabled) implements Writeable, ToXContentObject { + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "failure_store_template", + false, + (args, unused) -> new Template(args[0] == null ? ResettableValue.undefined() : (ResettableValue) args[0]) + ); + + static { + PARSER.declareField( + ConstructingObjectParser.optionalConstructorArg(), + (p, c) -> p.currentToken() == XContentParser.Token.VALUE_NULL + ? ResettableValue.reset() + : ResettableValue.create(p.booleanValue()), + ENABLED_FIELD, + ObjectParser.ValueType.BOOLEAN_OR_NULL + ); + } + + public Template { + if (enabled.get() == null) { + throw new IllegalArgumentException("Failure store configuration should have at least one non-null configuration value."); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + ResettableValue.write(out, enabled, StreamOutput::writeBoolean); + } + + public static Template read(StreamInput in) throws IOException { + ResettableValue enabled = ResettableValue.read(in, StreamInput::readBoolean); + return new Template(enabled); + } + + /** + * Converts the template to XContent, depending on the XContent.Params set by {@link ResettableValue#hideResetValues(Params)} + * it may or may not display any explicit nulls when the value is to be reset. + */ + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + enabled.toXContent(builder, params, ENABLED_FIELD.getPreferredName()); + builder.endObject(); + return builder; + } + + public static Template fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + /** + * Returns a template which has the value of the initial template updated with the values of the update. + * Note: for now it's a trivial composition because we have only one non-null field. + * @return the composed template + */ + public static Template merge(Template ignored, Template update) { + return update; + } + + public DataStreamFailureStore toFailureStore() { + return new DataStreamFailureStore(enabled.get()); + } + + @Override + public String toString() { + return Strings.toString(this, true, true); + } + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreDefinition.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreDefinition.java index 0927c3d91a71f..7315e9f7a51d3 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreDefinition.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreDefinition.java @@ -1,13 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; +import org.elasticsearch.cluster.routing.allocation.DataTier; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; @@ -18,6 +20,8 @@ import org.elasticsearch.index.mapper.RoutingFieldMapper; import java.io.IOException; +import java.util.HashSet; +import java.util.Set; /** * A utility class that contains the mappings and settings logic for failure store indices that are a part of data streams. @@ -25,12 +29,30 @@ public class DataStreamFailureStoreDefinition { public static final String FAILURE_STORE_REFRESH_INTERVAL_SETTING_NAME = "data_streams.failure_store.refresh_interval"; + public static final String INDEX_FAILURE_STORE_VERSION_SETTING_NAME = "index.failure_store.version"; public static final Settings DATA_STREAM_FAILURE_STORE_SETTINGS; + // Only a subset of user configurable settings is applicable for a failure index. Here we have an + // allowlist that will filter all other settings out. + public static final Set SUPPORTED_USER_SETTINGS = Set.of( + DataTier.TIER_PREFERENCE, + IndexMetadata.SETTING_INDEX_HIDDEN, + INDEX_FAILURE_STORE_VERSION_SETTING_NAME, + IndexMetadata.SETTING_NUMBER_OF_SHARDS, + IndexMetadata.SETTING_NUMBER_OF_REPLICAS, + IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, + IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), + IndexMetadata.LIFECYCLE_NAME + ); + public static final Set SUPPORTED_USER_SETTINGS_PREFIXES = Set.of( + IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + ".", + IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_PREFIX + ".", + IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_PREFIX + "." + ); public static final CompressedXContent DATA_STREAM_FAILURE_STORE_MAPPING; public static final int FAILURE_STORE_DEFINITION_VERSION = 1; public static final Setting FAILURE_STORE_DEFINITION_VERSION_SETTING = Setting.intSetting( - "index.failure_store.version", + INDEX_FAILURE_STORE_VERSION_SETTING_NAME, 0, Setting.Property.IndexScope ); @@ -39,11 +61,6 @@ public class DataStreamFailureStoreDefinition { DATA_STREAM_FAILURE_STORE_SETTINGS = Settings.builder() // Always start with the hidden settings for a backing index. .put(IndexMetadata.SETTING_INDEX_HIDDEN, true) - // Override any pipeline settings on the failure store to not use any - // specified by the data stream template. Default pipelines are very much - // meant for the backing indices only. - .putNull(IndexSettings.DEFAULT_PIPELINE.getKey()) - .putNull(IndexSettings.FINAL_PIPELINE.getKey()) .put(FAILURE_STORE_DEFINITION_VERSION_SETTING.getKey(), FAILURE_STORE_DEFINITION_VERSION) .build(); @@ -198,4 +215,23 @@ public static Settings.Builder applyFailureStoreSettings(Settings nodeSettings, } return builder; } + + /** + * Removes the unsupported by the failure store settings from the settings provided. + * ATTENTION: This method should be applied BEFORE we set the necessary settings for an index + * @param builder the settings builder that is going to be updated + * @return the original settings builder, with the unsupported settings removed. + */ + public static Settings.Builder filterUserDefinedSettings(Settings.Builder builder) { + if (builder.keys().isEmpty() == false) { + Set existingKeys = new HashSet<>(builder.keys()); + for (String setting : existingKeys) { + if (SUPPORTED_USER_SETTINGS.contains(setting) == false + && SUPPORTED_USER_SETTINGS_PREFIXES.stream().anyMatch(setting::startsWith) == false) { + builder.remove(setting); + } + } + } + return builder; + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreSettings.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreSettings.java new file mode 100644 index 0000000000000..c5076d01eabb0 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreSettings.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.cluster.metadata; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Setting; + +import java.util.List; +import java.util.function.Predicate; + +import static org.elasticsearch.core.Predicates.never; + +/** + * Holder for the data stream global settings relating to the data stream failure store. This defines, validates, and monitors the settings. + */ +public class DataStreamFailureStoreSettings { + + private static final Logger logger = LogManager.getLogger(DataStreamFailureStoreSettings.class); + + public static final Setting> DATA_STREAM_FAILURE_STORED_ENABLED_SETTING = Setting.stringListSetting( + "data_streams.failure_store.enabled", + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); + + private Predicate failureStoreEnabledByName; + + private DataStreamFailureStoreSettings() { + failureStoreEnabledByName = never(); + } + + /** + * Creates an instance and initialises the cluster settings listeners. + * + * @param clusterSettings The cluster settings to initialize the instance from and to watch for updates to + */ + public static DataStreamFailureStoreSettings create(ClusterSettings clusterSettings) { + DataStreamFailureStoreSettings dataStreamFailureStoreSettings = new DataStreamFailureStoreSettings(); + if (DataStream.isFailureStoreFeatureFlagEnabled()) { + clusterSettings.initializeAndWatch( + DATA_STREAM_FAILURE_STORED_ENABLED_SETTING, + dataStreamFailureStoreSettings::setEnabledByNamePatterns + ); + } + return dataStreamFailureStoreSettings; + } + + /** + * Returns whether the settings indicate that the failure store should be enabled by the cluster settings for the given name. + * + * @param name The data stream name + */ + public boolean failureStoreEnabledForDataStreamName(String name) { + assert DataStream.isFailureStoreFeatureFlagEnabled() : "Testing whether failure store is enabled should be behind by feature flag"; + return failureStoreEnabledByName.test(name); + } + + private void setEnabledByNamePatterns(List patterns) { + failureStoreEnabledByName = Regex.simpleMatcher(patterns.toArray(String[]::new)); + logger.info("Updated data stream name patterns for enabling failure store to [{}]", patterns); + } +} diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetention.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetention.java index 185f625f6f91f..af807e87a848e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetention.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetention.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetentionSettings.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetentionSettings.java index a1fcf56a92726..9e7256d6818bb 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetentionSettings.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetentionSettings.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; @@ -25,8 +26,6 @@ * The global retention settings apply to non-system data streams that are managed by the data stream lifecycle. They consist of: * - The default retention which applies to data streams that do not have a retention defined. * - The max retention which applies to all data streams that do not have retention or their retention has exceeded this value. - *

    - * Temporarily, we fall back to {@link DataStreamFactoryRetention} to facilitate a smooth transition to these settings. */ public class DataStreamGlobalRetentionSettings { @@ -83,42 +82,35 @@ public Iterator> settings() { Setting.Property.Dynamic ); - private final DataStreamFactoryRetention factoryRetention; - @Nullable private volatile TimeValue defaultRetention; @Nullable private volatile TimeValue maxRetention; - private DataStreamGlobalRetentionSettings(DataStreamFactoryRetention factoryRetention) { - this.factoryRetention = factoryRetention; + private DataStreamGlobalRetentionSettings() { + } @Nullable public TimeValue getMaxRetention() { - return shouldFallbackToFactorySettings() ? factoryRetention.getMaxRetention() : maxRetention; + return maxRetention; } @Nullable public TimeValue getDefaultRetention() { - return shouldFallbackToFactorySettings() ? factoryRetention.getDefaultRetention() : defaultRetention; + return defaultRetention; } public boolean areDefined() { return getDefaultRetention() != null || getMaxRetention() != null; } - private boolean shouldFallbackToFactorySettings() { - return defaultRetention == null && maxRetention == null; - } - /** * Creates an instance and initialises the cluster settings listeners * @param clusterSettings it will register the cluster settings listeners to monitor for changes - * @param factoryRetention for migration purposes, it will be removed shortly */ - public static DataStreamGlobalRetentionSettings create(ClusterSettings clusterSettings, DataStreamFactoryRetention factoryRetention) { - DataStreamGlobalRetentionSettings dataStreamGlobalRetentionSettings = new DataStreamGlobalRetentionSettings(factoryRetention); + public static DataStreamGlobalRetentionSettings create(ClusterSettings clusterSettings) { + DataStreamGlobalRetentionSettings dataStreamGlobalRetentionSettings = new DataStreamGlobalRetentionSettings(); clusterSettings.initializeAndWatch(DATA_STREAMS_DEFAULT_RETENTION_SETTING, dataStreamGlobalRetentionSettings::setDefaultRetention); clusterSettings.initializeAndWatch(DATA_STREAMS_MAX_RETENTION_SETTING, dataStreamGlobalRetentionSettings::setMaxRetention); return dataStreamGlobalRetentionSettings; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java index bd9a65735be05..fbee25d0f5453 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java index c65f83eca0aa2..3f5e7a2e0c4aa 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; @@ -16,10 +17,9 @@ import org.elasticsearch.cluster.NamedDiff; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ParseField; @@ -231,13 +231,10 @@ public static DataStreamMetadata fromXContent(XContentParser parser) throws IOEx } @Override - public Iterator toXContentChunked(ToXContent.Params ignored) { - return Iterators.concat( - ChunkedToXContentHelper.xContentValuesMap(DATA_STREAM.getPreferredName(), dataStreams), - ChunkedToXContentHelper.startObject(DATA_STREAM_ALIASES.getPreferredName()), - dataStreamAliases.values().iterator(), - ChunkedToXContentHelper.endObject() - ); + public Iterator toXContentChunked(ToXContent.Params params) { + return ChunkedToXContent.builder(params) + .xContentObjectFields(DATA_STREAM.getPreferredName(), dataStreams) + .xContentObject(DATA_STREAM_ALIASES.getPreferredName(), dataStreamAliases.values().iterator()); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamOptions.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamOptions.java index 9c7d2a986fa48..423b698442581 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamOptions.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamOptions.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; @@ -13,9 +14,9 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.Nullable; import org.elasticsearch.xcontent.ConstructingObjectParser; -import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -23,6 +24,8 @@ import java.io.IOException; +import static org.elasticsearch.cluster.metadata.DataStreamFailureStore.FAILURE_STORE; + /** * Holds data stream dedicated configuration options such as failure store, (in the future lifecycle). Currently, it * supports the following configurations: @@ -33,7 +36,10 @@ public record DataStreamOptions(@Nullable DataStreamFailureStore failureStore) SimpleDiffable, ToXContentObject { - public static final ParseField FAILURE_STORE_FIELD = new ParseField("failure_store"); + public static final ParseField FAILURE_STORE_FIELD = new ParseField(FAILURE_STORE); + public static final DataStreamOptions FAILURE_STORE_ENABLED = new DataStreamOptions(new DataStreamFailureStore(true)); + public static final DataStreamOptions FAILURE_STORE_DISABLED = new DataStreamOptions(new DataStreamFailureStore(false)); + public static final DataStreamOptions EMPTY = new DataStreamOptions(null); public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "options", @@ -42,31 +48,28 @@ public record DataStreamOptions(@Nullable DataStreamFailureStore failureStore) ); static { - PARSER.declareField( + PARSER.declareObject( ConstructingObjectParser.optionalConstructorArg(), (p, c) -> DataStreamFailureStore.fromXContent(p), - FAILURE_STORE_FIELD, - ObjectParser.ValueType.OBJECT_OR_NULL + FAILURE_STORE_FIELD ); } - public DataStreamOptions() { - this(null); - } - public static DataStreamOptions read(StreamInput in) throws IOException { return new DataStreamOptions(in.readOptionalWriteable(DataStreamFailureStore::new)); } - @Nullable - public DataStreamFailureStore getFailureStore() { - return failureStore; - } - public static Diff readDiffFrom(StreamInput in) throws IOException { return SimpleDiffable.readDiffFrom(DataStreamOptions::read, in); } + /** + * @return true if none of the options are defined + */ + public boolean isEmpty() { + return failureStore == null; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(failureStore); @@ -90,4 +93,100 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public static DataStreamOptions fromXContent(XContentParser parser) throws IOException { return PARSER.parse(parser, null); } + + /** + * This class is only used in template configuration. It wraps the fields of {@link DataStreamOptions} with {@link ResettableValue} + * to allow a user to signal when they want to reset any previously encountered values during template composition. Furthermore, it + * provides the {@link Template.Builder} that dictates how two templates can be composed. + */ + public record Template(ResettableValue failureStore) implements Writeable, ToXContentObject { + public static final Template EMPTY = new Template(ResettableValue.undefined()); + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "data_stream_options_template", + false, + (args, unused) -> new Template( + args[0] == null ? ResettableValue.undefined() : (ResettableValue) args[0] + ) + ); + + static { + PARSER.declareObjectOrNull( + ConstructingObjectParser.optionalConstructorArg(), + (p, s) -> ResettableValue.create(DataStreamFailureStore.Template.fromXContent(p)), + ResettableValue.reset(), + FAILURE_STORE_FIELD + ); + } + + public Template { + assert failureStore != null : "Template does not accept null values, please use Resettable.undefined()"; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + ResettableValue.write(out, failureStore, (o, v) -> v.writeTo(o)); + } + + public static Template read(StreamInput in) throws IOException { + ResettableValue failureStore = ResettableValue.read(in, DataStreamFailureStore.Template::read); + return new Template(failureStore); + } + + public static Template fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + /** + * Converts the template to XContent, depending on the {@param params} set by {@link ResettableValue#hideResetValues(Params)} + * it may or may not display any explicit nulls when the value is to be reset. + */ + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + failureStore.toXContent(builder, params, FAILURE_STORE_FIELD.getPreferredName()); + builder.endObject(); + return builder; + } + + public DataStreamOptions toDataStreamOptions() { + return new DataStreamOptions(failureStore.mapAndGet(DataStreamFailureStore.Template::toFailureStore)); + } + + public static Builder builder(Template template) { + return new Builder(template); + } + + /** + * Builds and composes a data stream template. + */ + public static class Builder { + private ResettableValue failureStore = ResettableValue.undefined(); + + public Builder(Template template) { + if (template != null) { + failureStore = template.failureStore(); + } + } + + /** + * Updates the current failure store configuration with the provided value. This is not a replacement necessarily, if both + * instance contain data the configurations are merged. + */ + public Builder updateFailureStore(ResettableValue newFailureStore) { + failureStore = ResettableValue.merge(failureStore, newFailureStore, DataStreamFailureStore.Template::merge); + return this; + } + + public Template build() { + return new Template(failureStore); + } + } + + @Override + public String toString() { + return Strings.toString(this, true, true); + } + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNode.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNode.java index 403b4b85e664b..eab17fa93536d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNode.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNode.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodeWithStatus.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodeWithStatus.java index 92b370dec3a8b..7b89406be9aa0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodeWithStatus.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodeWithStatus.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodes.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodes.java index e572f20557c79..c7a8113d6d49c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodes.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodes.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodesMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodesMetadata.java index ab5c324150f2b..6bc7fffbeaa54 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodesMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodesMetadata.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DiffableStringMap.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DiffableStringMap.java index 51a66ab094591..d931675547b2a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DiffableStringMap.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DiffableStringMap.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java index 511f3f528fb65..0585263c45e99 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java index e9405f061e99e..aa0b1c045be5c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java index 140eeb2e240ff..62867b4260bfd 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; @@ -18,7 +19,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.index.Index; import org.elasticsearch.xcontent.ContextParser; import org.elasticsearch.xcontent.ObjectParser; @@ -127,8 +128,8 @@ public boolean containsIndex(final Index index) { } @Override - public Iterator toXContentChunked(ToXContent.Params ignored) { - return ChunkedToXContentHelper.array(TOMBSTONES_FIELD.getPreferredName(), tombstones.iterator()); + public Iterator toXContentChunked(ToXContent.Params params) { + return ChunkedToXContent.builder(params).array(TOMBSTONES_FIELD.getPreferredName(), tombstones.iterator()); } public static IndexGraveyard fromXContent(final XContentParser parser) throws IOException { @@ -433,7 +434,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.startObject(); builder.field(INDEX_KEY); index.toXContent(builder, params); - builder.timeField(DELETE_DATE_IN_MILLIS_KEY, DELETE_DATE_KEY, deleteDateInMillis); + builder.timestampFieldsFromUnixEpochMillis(DELETE_DATE_IN_MILLIS_KEY, DELETE_DATE_KEY, deleteDateInMillis); return builder.endObject(); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 611640f4a3b0f..ae4b16b08862f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; @@ -40,6 +41,7 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.gateway.MetadataStateFormat; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; @@ -138,6 +140,15 @@ public class IndexMetadata implements Diffable, ToXContentFragmen RestStatus.TOO_MANY_REQUESTS, EnumSet.of(ClusterBlockLevel.WRITE) ); + public static final ClusterBlock INDEX_REFRESH_BLOCK = new ClusterBlock( + 14, + "index refresh blocked, waiting for shard(s) to be started", + true, + false, + false, + RestStatus.REQUEST_TIMEOUT, + EnumSet.of(ClusterBlockLevel.REFRESH) + ); // 'event.ingested' (part of Elastic Common Schema) range is tracked in cluster state, along with @timestamp public static final String EVENT_INGESTED_FIELD_NAME = "event.ingested"; @@ -946,9 +957,9 @@ public IndexMetadata withTimestampRanges( if (timestampRange.equals(this.timestampRange) && eventIngestedRange.equals(this.eventIngestedRange)) { return this; } + @UpdateForV9 // remove this check when 8.15 is no longer communicable IndexLongFieldRange allowedEventIngestedRange = eventIngestedRange; - // remove this check when the EVENT_INGESTED_RANGE_IN_CLUSTER_STATE version is removed - if (minClusterTransportVersion.before(TransportVersions.EVENT_INGESTED_RANGE_IN_CLUSTER_STATE)) { + if (minClusterTransportVersion.before(TransportVersions.V_8_15_0)) { allowedEventIngestedRange = IndexLongFieldRange.UNKNOWN; } return new IndexMetadata( @@ -1646,7 +1657,7 @@ private static class IndexMetadataDiff implements Diff { DiffableUtils.getStringKeySerializer(), ROLLOVER_INFO_DIFF_VALUE_READER ); - if (in.getTransportVersion().onOrAfter(TransportVersions.INDEX_METADATA_MAPPINGS_UPDATED_VERSION)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { mappingsUpdatedVersion = IndexVersion.readVersion(in); } else { mappingsUpdatedVersion = IndexVersions.ZERO; @@ -1666,7 +1677,7 @@ private static class IndexMetadataDiff implements Diff { indexWriteLoadForecast = null; shardSizeInBytesForecast = null; } - if (in.getTransportVersion().onOrAfter(TransportVersions.EVENT_INGESTED_RANGE_IN_CLUSTER_STATE)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { eventIngestedRange = IndexLongFieldRange.readFrom(in); } else { eventIngestedRange = IndexLongFieldRange.UNKNOWN; @@ -1700,7 +1711,7 @@ public void writeTo(StreamOutput out) throws IOException { customData.writeTo(out); inSyncAllocationIds.writeTo(out); rolloverInfos.writeTo(out); - if (out.getTransportVersion().onOrAfter(TransportVersions.INDEX_METADATA_MAPPINGS_UPDATED_VERSION)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { IndexVersion.writeVersion(mappingsUpdatedVersion, out); } if (out.getTransportVersion().onOrAfter(SYSTEM_INDEX_FLAG_ADDED)) { @@ -1712,7 +1723,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalDouble(indexWriteLoadForecast); out.writeOptionalLong(shardSizeInBytesForecast); } - if (out.getTransportVersion().onOrAfter(TransportVersions.EVENT_INGESTED_RANGE_IN_CLUSTER_STATE)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { eventIngestedRange.writeTo(out); } else { assert eventIngestedRange == IndexLongFieldRange.UNKNOWN @@ -1811,7 +1822,7 @@ public static IndexMetadata readFrom(StreamInput in, @Nullable Function DiffableUtils.StringSetValueSerializer.getInstance().write(v, o) ); out.writeCollection(rolloverInfos.values()); - if (out.getTransportVersion().onOrAfter(TransportVersions.INDEX_METADATA_MAPPINGS_UPDATED_VERSION)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { IndexVersion.writeVersion(mappingsUpdatedVersion, out); } if (out.getTransportVersion().onOrAfter(SYSTEM_INDEX_FLAG_ADDED)) { @@ -1881,7 +1892,7 @@ public void writeTo(StreamOutput out, boolean mappingsAsHash) throws IOException out.writeOptionalDouble(writeLoadForecast); out.writeOptionalLong(shardSizeInBytesForecast); } - if (out.getTransportVersion().onOrAfter(TransportVersions.EVENT_INGESTED_RANGE_IN_CLUSTER_STATE)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { eventIngestedRange.writeTo(out); } else { assert eventIngestedRange == IndexLongFieldRange.UNKNOWN @@ -2207,8 +2218,7 @@ public Builder eventIngestedRange(IndexLongFieldRange eventIngestedRange, Transp + minClusterTransportVersion + "; eventIngestedRange = " + eventIngestedRange; - if (minClusterTransportVersion != null - && minClusterTransportVersion.before(TransportVersions.EVENT_INGESTED_RANGE_IN_CLUSTER_STATE)) { + if (minClusterTransportVersion != null && minClusterTransportVersion.before(TransportVersions.V_8_15_0)) { this.eventIngestedRange = IndexLongFieldRange.UNKNOWN; } else { this.eventIngestedRange = eventIngestedRange; @@ -2267,7 +2277,7 @@ IndexMetadata build(boolean repair) { "routing partition size [" + routingPartitionSize + "] should be a positive number" - + " less than the number of shards [" + + " less than the number of routing shards [" + getRoutingNumShards() + "] for [" + index diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataStats.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataStats.java index 7e5d7672a0e3b..201f4aab6eb85 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataStats.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataStats.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java index e774d7e4d552d..be2563c4732b7 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; @@ -87,8 +88,12 @@ public IndexMetadataVerifier( * If the index does not need upgrade it returns the index metadata unchanged, otherwise it returns a modified index metadata. If index * cannot be updated the method throws an exception. */ - public IndexMetadata verifyIndexMetadata(IndexMetadata indexMetadata, IndexVersion minimumIndexCompatibilityVersion) { - checkSupportedVersion(indexMetadata, minimumIndexCompatibilityVersion); + public IndexMetadata verifyIndexMetadata( + IndexMetadata indexMetadata, + IndexVersion minimumIndexCompatibilityVersion, + IndexVersion minimumReadOnlyIndexCompatibilityVersion + ) { + checkSupportedVersion(indexMetadata, minimumIndexCompatibilityVersion, minimumReadOnlyIndexCompatibilityVersion); // First convert any shared_cache searchable snapshot indices to only use _tier_preference: data_frozen IndexMetadata newMetadata = convertSharedCacheTierPreference(indexMetadata); @@ -104,26 +109,81 @@ public IndexMetadata verifyIndexMetadata(IndexMetadata indexMetadata, IndexVersi } /** - * Check that the index version is compatible. Elasticsearch does not support indices created before the - * previous major version. + * Check that the index version is compatible. Elasticsearch supports reading and writing indices created in the current version ("N") + + as well as the previous major version ("N-1"). Elasticsearch only supports reading indices created down to the penultimate version + + ("N-2") and does not support reading nor writing any version below that. */ - private static void checkSupportedVersion(IndexMetadata indexMetadata, IndexVersion minimumIndexCompatibilityVersion) { - boolean isSupportedVersion = indexMetadata.getCompatibilityVersion().onOrAfter(minimumIndexCompatibilityVersion); - if (isSupportedVersion == false) { - throw new IllegalStateException( - "The index " - + indexMetadata.getIndex() - + " has current compatibility version [" - + indexMetadata.getCompatibilityVersion().toReleaseVersion() - + "] but the minimum compatible version is [" - + minimumIndexCompatibilityVersion.toReleaseVersion() - + "]. It should be re-indexed in Elasticsearch " - + (Version.CURRENT.major - 1) - + ".x before upgrading to " - + Build.current().version() - + "." - ); + private static void checkSupportedVersion( + IndexMetadata indexMetadata, + IndexVersion minimumIndexCompatibilityVersion, + IndexVersion minimumReadOnlyIndexCompatibilityVersion + ) { + if (isFullySupportedVersion(indexMetadata, minimumIndexCompatibilityVersion)) { + return; + } + if (isReadOnlySupportedVersion(indexMetadata, minimumIndexCompatibilityVersion, minimumReadOnlyIndexCompatibilityVersion)) { + return; + } + throw new IllegalStateException( + "The index " + + indexMetadata.getIndex() + + " has current compatibility version [" + + indexMetadata.getCompatibilityVersion().toReleaseVersion() + + "] but the minimum compatible version is [" + + minimumIndexCompatibilityVersion.toReleaseVersion() + + "]. It should be re-indexed in Elasticsearch " + + (Version.CURRENT.major - 1) + + ".x before upgrading to " + + Build.current().version() + + "." + ); + } + + private static boolean isFullySupportedVersion(IndexMetadata indexMetadata, IndexVersion minimumIndexCompatibilityVersion) { + return indexMetadata.getCompatibilityVersion().onOrAfter(minimumIndexCompatibilityVersion); + } + + /** + * Returns {@code true} if the index version is compatible in read-only mode. As of today, only searchable snapshots and archive indices + * in version N-2 with a write block are read-only compatible. This method throws an {@link IllegalStateException} if the index is + * either a searchable snapshot or an archive index with a read-only compatible version but is missing the write block. + * + * @param indexMetadata the index metadata + * @param minimumIndexCompatibilityVersion the min. index compatible version for reading and writing indices (used in assertion) + * @param minReadOnlyIndexCompatibilityVersion the min. index compatible version for only reading indices + * + * @return {@code true} if the index version is compatible in read-only mode, {@code false} otherwise. + * @throws IllegalStateException if the index is read-only compatible but has no write block in place. + */ + public static boolean isReadOnlySupportedVersion( + IndexMetadata indexMetadata, + IndexVersion minimumIndexCompatibilityVersion, + IndexVersion minReadOnlyIndexCompatibilityVersion + ) { + boolean isReadOnlySupportedVersion = indexMetadata.getCompatibilityVersion().onOrAfter(minReadOnlyIndexCompatibilityVersion); + assert isFullySupportedVersion(indexMetadata, minimumIndexCompatibilityVersion) == false; + + if (isReadOnlySupportedVersion + && (indexMetadata.isSearchableSnapshot() || indexMetadata.getCreationVersion().isLegacyIndexVersion())) { + boolean isReadOnly = IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.get(indexMetadata.getSettings()); + if (isReadOnly == false) { + throw new IllegalStateException( + "The index " + + indexMetadata.getIndex() + + " created in version [" + + indexMetadata.getCreationVersion() + + "] with current compatibility version [" + + indexMetadata.getCompatibilityVersion().toReleaseVersion() + + "] must be marked as read-only using the setting [" + + IndexMetadata.SETTING_BLOCKS_WRITE + + "] set to [true] before upgrading to " + + Build.current().version() + + '.' + ); + } + return true; } + return false; } /** diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index b945fe7e510f6..a2211cf8ea893 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; @@ -36,6 +37,7 @@ import org.elasticsearch.indices.InvalidIndexNameException; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel; +import org.elasticsearch.transport.RemoteClusterAware; import java.time.Instant; import java.time.ZoneId; @@ -46,18 +48,25 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.SortedMap; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.LongSupplier; import java.util.function.Predicate; -import java.util.stream.Collectors; -import java.util.stream.Stream; +/** + * This class main focus is to resolve multi-syntax target expressions to resources or concrete indices. This resolution is influenced + * by IndicesOptions and other flags passed through the method call. Examples of the functionality it provides: + * - Resolve expressions to concrete indices + * - Resolve expressions to data stream names + * - Resolve expressions to resources (meaning indices, data streams and aliases) + * Note: This class is performance sensitive, so we pay extra attention on the data structure usage and we avoid streams and iterators + * when possible in favor of the classic for-i loops. + */ +@SuppressWarnings("ForLoopReplaceableByForEach") public class IndexNameExpressionResolver { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(IndexNameExpressionResolver.class); @@ -72,6 +81,13 @@ public IndexNameExpressionResolver(ThreadContext threadContext, SystemIndices sy this.systemIndices = Objects.requireNonNull(systemIndices, "System Indices must not be null"); } + /** + * This represents a resolved expression in the form of the name of a resource in the cluster. + * Soon it will facilitate an index component selector, which will define which part of the resource the expression is targeting. + * @param resource the name of a resource that an expression refers to. + */ + public record ResolvedExpression(String resource) {} + /** * Same as {@link #concreteIndexNames(ClusterState, IndicesOptions, String...)}, but the index expressions and options * are encapsulated in the specified request. @@ -189,8 +205,9 @@ public List dataStreamNames(ClusterState state, IndicesOptions options, getSystemIndexAccessPredicate(), getNetNewSystemIndexPredicate() ); - final Collection expressions = resolveExpressions(context, indexExpressions); + final Collection expressions = resolveExpressionsToResources(context, indexExpressions); return expressions.stream() + .map(ResolvedExpression::resource) .map(x -> state.metadata().getIndicesLookup().get(x)) .filter(Objects::nonNull) .filter(ia -> ia.getType() == Type.DATA_STREAM) @@ -219,10 +236,11 @@ public IndexAbstraction resolveWriteIndexAbstraction(ClusterState state, DocWrit getNetNewSystemIndexPredicate() ); - final Collection expressions = resolveExpressions(context, request.index()); + final Collection expressions = resolveExpressionsToResources(context, request.index()); if (expressions.size() == 1) { - IndexAbstraction ia = state.metadata().getIndicesLookup().get(expressions.iterator().next()); + ResolvedExpression resolvedExpression = expressions.iterator().next(); + IndexAbstraction ia = state.metadata().getIndicesLookup().get(resolvedExpression.resource()); if (ia.getType() == Type.ALIAS) { Index writeIndex = ia.getWriteIndex(); if (writeIndex == null) { @@ -235,7 +253,7 @@ public IndexAbstraction resolveWriteIndexAbstraction(ClusterState state, DocWrit ); } } - checkSystemIndexAccess(context, Set.of(ia.getWriteIndex())); + SystemResourceAccess.checkSystemIndexAccess(context, threadContext, ia.getWriteIndex()); return ia; } else { throw new IllegalArgumentException( @@ -244,27 +262,110 @@ public IndexAbstraction resolveWriteIndexAbstraction(ClusterState state, DocWrit } } - protected static Collection resolveExpressions(Context context, String... expressions) { - if (context.getOptions().expandWildcardExpressions() == false) { + /** + * Resolve the expression to the set of indices, aliases, and, optionally, data streams that the expression matches. + * If {@param preserveDataStreams} is {@code true}, data streams that are covered by the wildcards from the + * {@param expressions} are returned as-is, without expanding them further to their respective backing indices. + */ + protected static Collection resolveExpressionsToResources(Context context, String... expressions) { + // If we do not expand wildcards, then empty or _all expression result in an empty list + boolean expandWildcards = context.getOptions().expandWildcardExpressions(); + if (expandWildcards == false) { if (expressions == null || expressions.length == 0 || expressions.length == 1 && Metadata.ALL.equals(expressions[0])) { return List.of(); - } else { - return ExplicitResourceNameFilter.filterUnavailable( - context, - DateMathExpressionResolver.resolve(context, List.of(expressions)) - ); } } else { if (expressions == null || expressions.length == 0 || expressions.length == 1 && (Metadata.ALL.equals(expressions[0]) || Regex.isMatchAllPattern(expressions[0]))) { return WildcardExpressionResolver.resolveAll(context); + } else if (isNoneExpression(expressions)) { + return List.of(); + } + } + + // Using ArrayList when we know we do not have wildcards is an optimisation, given that one expression result in 0 or 1 resources. + Collection resources = expandWildcards && WildcardExpressionResolver.hasWildcards(expressions) + ? new LinkedHashSet<>() + : new ArrayList<>(expressions.length); + boolean wildcardSeen = false; + for (int i = 0, n = expressions.length; i < n; i++) { + String originalExpression = expressions[i]; + + // Resolve exclusion, a `-` prefixed expression is an exclusion only if it succeeds a wildcard. + boolean isExclusion = wildcardSeen && originalExpression.startsWith("-"); + String baseExpression = isExclusion ? originalExpression.substring(1) : originalExpression; + + // Resolve date math + baseExpression = DateMathExpressionResolver.resolveExpression(baseExpression, context::getStartTime); + + // Validate base expression + validateResourceExpression(context, baseExpression, expressions); + + // Check if it's wildcard + boolean isWildcard = expandWildcards && WildcardExpressionResolver.isWildcard(originalExpression); + wildcardSeen |= isWildcard; + + if (isWildcard) { + Set matchingResources = WildcardExpressionResolver.matchWildcardToResources(context, baseExpression); + + if (context.getOptions().allowNoIndices() == false && matchingResources.isEmpty()) { + throw notFoundException(baseExpression); + } + + if (isExclusion) { + resources.removeAll(matchingResources); + } else { + resources.addAll(matchingResources); + } } else { - return WildcardExpressionResolver.resolve( - context, - ExplicitResourceNameFilter.filterUnavailable(context, DateMathExpressionResolver.resolve(context, List.of(expressions))) - ); + if (isExclusion) { + resources.remove(new ResolvedExpression(baseExpression)); + } else if (ensureAliasOrIndexExists(context, baseExpression)) { + resources.add(new ResolvedExpression(baseExpression)); + } + } + } + return resources; + } + + /** + * Validates the requested expression by performing the following checks: + * - Ensure it's not empty + * - Ensure it doesn't start with `_` + * - Ensure it's not a remote expression unless the allow unavailable targets is enabled. + */ + private static void validateResourceExpression(Context context, String current, String[] expressions) { + if (Strings.isEmpty(current)) { + throw notFoundException(current); + } + // Expressions can not start with an underscore. This is reserved for APIs. If the check gets here, the API + // does not exist and the path is interpreted as an expression. If the expression begins with an underscore, + // throw a specific error that is different from the [[IndexNotFoundException]], which is typically thrown + // if the expression can't be found. + if (current.charAt(0) == '_') { + throw new InvalidIndexNameException(current, "must not start with '_'."); + } + ensureRemoteExpressionRequireIgnoreUnavailable(context.getOptions(), current, expressions); + } + + /** + * Throws an exception if the expression is a remote expression and we do not allow unavailable targets + */ + private static void ensureRemoteExpressionRequireIgnoreUnavailable(IndicesOptions options, String current, String[] expressions) { + if (options.ignoreUnavailable()) { + return; + } + if (RemoteClusterAware.isRemoteIndexName(current)) { + List crossClusterIndices = new ArrayList<>(); + for (int i = 0; i < expressions.length; i++) { + if (RemoteClusterAware.isRemoteIndexName(expressions[i])) { + crossClusterIndices.add(expressions[i]); + } } + throw new IllegalArgumentException( + "Cross-cluster calls are not supported in this context but remote indices were requested: " + crossClusterIndices + ); } } @@ -337,12 +438,12 @@ String[] concreteIndexNames(Context context, String... indexExpressions) { } Index[] concreteIndices(Context context, String... indexExpressions) { - final Collection expressions = resolveExpressions(context, indexExpressions); + final Collection expressions = resolveExpressionsToResources(context, indexExpressions); final Set concreteIndicesResult = Sets.newLinkedHashSetWithExpectedSize(expressions.size()); final Map indicesLookup = context.getState().metadata().getIndicesLookup(); - for (String expression : expressions) { - final IndexAbstraction indexAbstraction = indicesLookup.get(expression); + for (ResolvedExpression expression : expressions) { + final IndexAbstraction indexAbstraction = indicesLookup.get(expression.resource()); assert indexAbstraction != null; if (indexAbstraction.getType() == Type.ALIAS && context.isResolveToWriteIndex()) { Index writeIndex = indexAbstraction.getWriteIndex(); @@ -376,7 +477,7 @@ Index[] concreteIndices(Context context, String... indexExpressions) { throw new IllegalArgumentException( indexAbstraction.getType().getDisplayName() + " [" - + expression + + expression.resource() + "] has more than one index associated with it " + Arrays.toString(indexNames) + ", can't execute a single index op" @@ -391,15 +492,19 @@ Index[] concreteIndices(Context context, String... indexExpressions) { && context.getOptions().includeFailureIndices()) { // Collect the data streams involved Set aliasDataStreams = new HashSet<>(); - for (Index index : indexAbstraction.getIndices()) { + List indices = indexAbstraction.getIndices(); + for (int i = 0, n = indices.size(); i < n; i++) { + Index index = indices.get(i); aliasDataStreams.add(indicesLookup.get(index.getName()).getParentDataStream()); } for (DataStream dataStream : aliasDataStreams) { resolveIndicesForDataStream(context, dataStream, concreteIndicesResult); } } else { - for (Index index : indexAbstraction.getIndices()) { - if (shouldTrackConcreteIndex(context, context.getOptions(), index)) { + List indices = indexAbstraction.getIndices(); + for (int i = 0, n = indices.size(); i < n; i++) { + Index index = indices.get(i); + if (shouldTrackConcreteIndex(context, index)) { concreteIndicesResult.add(index); } } @@ -410,14 +515,17 @@ Index[] concreteIndices(Context context, String... indexExpressions) { if (context.getOptions().allowNoIndices() == false && concreteIndicesResult.isEmpty()) { throw notFoundException(indexExpressions); } - checkSystemIndexAccess(context, concreteIndicesResult); - return concreteIndicesResult.toArray(Index.EMPTY_ARRAY); + Index[] resultArray = concreteIndicesResult.toArray(Index.EMPTY_ARRAY); + SystemResourceAccess.checkSystemIndexAccess(context, threadContext, resultArray); + return resultArray; } private static void resolveIndicesForDataStream(Context context, DataStream dataStream, Set concreteIndicesResult) { if (shouldIncludeRegularIndices(context.getOptions())) { - for (Index index : dataStream.getIndices()) { - if (shouldTrackConcreteIndex(context, context.getOptions(), index)) { + List indices = dataStream.getIndices(); + for (int i = 0, n = indices.size(); i < n; i++) { + Index index = indices.get(i); + if (shouldTrackConcreteIndex(context, index)) { concreteIndicesResult.add(index); } } @@ -425,8 +533,10 @@ private static void resolveIndicesForDataStream(Context context, DataStream data if (shouldIncludeFailureIndices(context.getOptions())) { // We short-circuit here, if failure indices are not allowed and they can be skipped if (context.getOptions().allowFailureIndices() || context.getOptions().ignoreUnavailable() == false) { - for (Index index : dataStream.getFailureIndices().getIndices()) { - if (shouldTrackConcreteIndex(context, context.getOptions(), index)) { + List failureIndices = dataStream.getFailureIndices().getIndices(); + for (int i = 0, n = failureIndices.size(); i < n; i++) { + Index index = failureIndices.get(i); + if (shouldTrackConcreteIndex(context, index)) { concreteIndicesResult.add(index); } } @@ -476,64 +586,6 @@ private static boolean resolvesToMoreThanOneIndex(IndexAbstraction indexAbstract return indexAbstraction.getIndices().size() > 1; } - private void checkSystemIndexAccess(Context context, Set concreteIndices) { - final Predicate systemIndexAccessPredicate = context.getSystemIndexAccessPredicate(); - if (systemIndexAccessPredicate == Predicates.always()) { - return; - } - doCheckSystemIndexAccess(context, concreteIndices, systemIndexAccessPredicate); - } - - private void doCheckSystemIndexAccess(Context context, Set concreteIndices, Predicate systemIndexAccessPredicate) { - final Metadata metadata = context.getState().metadata(); - final List resolvedSystemIndices = new ArrayList<>(); - final List resolvedNetNewSystemIndices = new ArrayList<>(); - final Set resolvedSystemDataStreams = new HashSet<>(); - final SortedMap indicesLookup = metadata.getIndicesLookup(); - boolean matchedIndex = false; - for (Index concreteIndex : concreteIndices) { - IndexMetadata idxMetadata = metadata.index(concreteIndex); - String name = concreteIndex.getName(); - if (idxMetadata.isSystem() && systemIndexAccessPredicate.test(name) == false) { - matchedIndex = true; - IndexAbstraction indexAbstraction = indicesLookup.get(name); - if (indexAbstraction.getParentDataStream() != null) { - resolvedSystemDataStreams.add(indexAbstraction.getParentDataStream().getName()); - } else if (systemIndices.isNetNewSystemIndex(name)) { - resolvedNetNewSystemIndices.add(name); - } else { - resolvedSystemIndices.add(name); - } - } - } - if (matchedIndex) { - handleMatchedSystemIndices(resolvedSystemIndices, resolvedSystemDataStreams, resolvedNetNewSystemIndices); - } - } - - private void handleMatchedSystemIndices( - List resolvedSystemIndices, - Set resolvedSystemDataStreams, - List resolvedNetNewSystemIndices - ) { - if (resolvedSystemIndices.isEmpty() == false) { - Collections.sort(resolvedSystemIndices); - deprecationLogger.warn( - DeprecationCategory.API, - "open_system_index_access", - "this request accesses system indices: {}, but in a future major version, direct access to system " - + "indices will be prevented by default", - resolvedSystemIndices - ); - } - if (resolvedSystemDataStreams.isEmpty() == false) { - throw SystemIndices.dataStreamAccessException(threadContext, resolvedSystemDataStreams); - } - if (resolvedNetNewSystemIndices.isEmpty() == false) { - throw SystemIndices.netNewSystemIndexAccessException(threadContext, resolvedNetNewSystemIndices); - } - } - private static IndexNotFoundException notFoundException(String... indexExpressions) { final IndexNotFoundException infe; if (indexExpressions == null @@ -561,21 +613,19 @@ private static IndexNotFoundException notFoundException(String... indexExpressio return infe; } - private static boolean shouldTrackConcreteIndex(Context context, IndicesOptions options, Index index) { - if (context.systemIndexAccessLevel == SystemIndexAccessLevel.BACKWARDS_COMPATIBLE_ONLY - && context.netNewSystemIndexPredicate.test(index.getName())) { + private static boolean shouldTrackConcreteIndex(Context context, Index index) { + if (SystemResourceAccess.isNetNewInBackwardCompatibleMode(context, index)) { // Exclude this one as it's a net-new system index, and we explicitly don't want those. return false; } + IndicesOptions options = context.getOptions(); if (DataStream.isFailureStoreFeatureFlagEnabled() && context.options.allowFailureIndices() == false) { DataStream parentDataStream = context.getState().metadata().getIndicesLookup().get(index.getName()).getParentDataStream(); - if (parentDataStream != null && parentDataStream.isFailureStoreEnabled()) { - if (parentDataStream.isFailureStoreIndex(index.getName())) { - if (options.ignoreUnavailable()) { - return false; - } else { - throw new FailureIndexNotSupportedException(index); - } + if (parentDataStream != null && parentDataStream.isFailureStoreIndex(index.getName())) { + if (options.ignoreUnavailable()) { + return false; + } else { + throw new FailureIndexNotSupportedException(index); } } } @@ -640,7 +690,7 @@ public Index concreteSingleIndex(ClusterState state, IndicesRequest request) { * Utility method that allows to resolve an index expression to its corresponding single write index. * * @param state the cluster state containing all the data to resolve to expression to a concrete index - * @param request The request that defines how the an alias or an index need to be resolved to a concrete index + * @param request The request that defines how an alias or an index need to be resolved to a concrete index * and the expression that can be resolved to an alias or an index name. * @throws IllegalArgumentException if the index resolution does not lead to an index, or leads to more than one index * @return the write index obtained as a result of the index resolution @@ -714,25 +764,10 @@ public boolean hasIndexAbstraction(String indexAbstraction, ClusterState state) return state.metadata().hasIndexAbstraction(resolvedAliasOrIndex); } - /** - * @return If the specified string is data math expression then this method returns the resolved expression. - */ - public static String resolveDateMathExpression(String dateExpression) { - return DateMathExpressionResolver.resolveExpression(dateExpression); - } - - /** - * @param time instant to consider when parsing the expression - * @return If the specified string is data math expression then this method returns the resolved expression. - */ - public static String resolveDateMathExpression(String dateExpression, long time) { - return DateMathExpressionResolver.resolveExpression(dateExpression, () -> time); - } - /** * Resolve an array of expressions to the set of indices and aliases that these expressions match. */ - public Set resolveExpressions(ClusterState state, String... expressions) { + public Set resolveExpressions(ClusterState state, String... expressions) { return resolveExpressions(state, IndicesOptions.lenientExpandOpen(), false, expressions); } @@ -741,7 +776,7 @@ public Set resolveExpressions(ClusterState state, String... expressions) * If {@param preserveDataStreams} is {@code true}, datastreams that are covered by the wildcards from the * {@param expressions} are returned as-is, without expanding them further to their respective backing indices. */ - public Set resolveExpressions( + public Set resolveExpressions( ClusterState state, IndicesOptions indicesOptions, boolean preserveDataStreams, @@ -758,10 +793,11 @@ public Set resolveExpressions( getSystemIndexAccessPredicate(), getNetNewSystemIndexPredicate() ); - Collection resolved = resolveExpressions(context, expressions); - if (resolved instanceof Set) { + // unmodifiable without creating a new collection as it might contain many items + Collection resolved = resolveExpressionsToResources(context, expressions); + if (resolved instanceof Set) { // unmodifiable without creating a new collection as it might contain many items - return Collections.unmodifiableSet((Set) resolved); + return Collections.unmodifiableSet((Set) resolved); } else { return Set.copyOf(resolved); } @@ -772,9 +808,9 @@ public Set resolveExpressions( * given index. *

    Only aliases with filters are returned. If the indices list contains a non-filtering reference to * the index itself - null is returned. Returns {@code null} if no filtering is required. - * NOTE: The provided expressions must have been resolved already via {@link #resolveExpressions}. + * NOTE: The provided expressions must have been resolved already via {@link #resolveExpressionsToResources(Context, String...)}. */ - public String[] filteringAliases(ClusterState state, String index, Set resolvedExpressions) { + public String[] filteringAliases(ClusterState state, String index, Set resolvedExpressions) { return indexAliases(state, index, AliasMetadata::filteringRequired, DataStreamAlias::filteringRequired, false, resolvedExpressions); } @@ -792,7 +828,8 @@ boolean iterateIndexAliases(int indexAliasesSize, int resolvedExpressionsSize) { * Iterates through the list of indices and selects the effective list of required aliases for the given index. *

    Only aliases where the given predicate tests successfully are returned. If the indices list contains a non-required reference to * the index itself - null is returned. Returns {@code null} if no filtering is required. - *

    NOTE: the provided expressions must have been resolved already via {@link #resolveExpressions}. + *

    NOTE: the provided expressions must have been resolved already via + * {@link #resolveExpressionsToResources(Context, String...)}. */ public String[] indexAliases( ClusterState state, @@ -800,26 +837,25 @@ public String[] indexAliases( Predicate requiredAlias, Predicate requiredDataStreamAlias, boolean skipIdentity, - Set resolvedExpressions + Set resolvedExpressions ) { - if (isAllIndices(resolvedExpressions)) { + if (isAllIndicesExpression(resolvedExpressions)) { return null; } - final IndexMetadata indexMetadata = state.metadata().getIndices().get(index); if (indexMetadata == null) { // Shouldn't happen throw new IndexNotFoundException(index); } - if (skipIdentity == false && resolvedExpressions.contains(index)) { + if (skipIdentity == false && resolvedExpressions.contains(new ResolvedExpression(index))) { return null; } IndexAbstraction ia = state.metadata().getIndicesLookup().get(index); DataStream dataStream = ia.getParentDataStream(); if (dataStream != null) { - if (skipIdentity == false && resolvedExpressions.contains(dataStream.getName())) { + if (skipIdentity == false && resolvedExpressions.contains(new ResolvedExpression(dataStream.getName()))) { // skip the filters when the request targets the data stream name return null; } @@ -828,11 +864,12 @@ public String[] indexAliases( if (iterateIndexAliases(dataStreamAliases.size(), resolvedExpressions.size())) { aliasesForDataStream = dataStreamAliases.values() .stream() - .filter(dataStreamAlias -> resolvedExpressions.contains(dataStreamAlias.getName())) + .filter(dataStreamAlias -> resolvedExpressions.contains(new ResolvedExpression(dataStreamAlias.getName()))) .filter(dataStreamAlias -> dataStreamAlias.getDataStreams().contains(dataStream.getName())) .toList(); } else { aliasesForDataStream = resolvedExpressions.stream() + .map(ResolvedExpression::resource) .map(dataStreamAliases::get) .filter(dataStreamAlias -> dataStreamAlias != null && dataStreamAlias.getDataStreams().contains(dataStream.getName())) .toList(); @@ -861,17 +898,19 @@ public String[] indexAliases( // faster to iterate indexAliases aliasCandidates = indexAliases.values() .stream() - .filter(aliasMetadata -> resolvedExpressions.contains(aliasMetadata.alias())) + .filter(aliasMetadata -> resolvedExpressions.contains(new ResolvedExpression(aliasMetadata.alias()))) .toArray(AliasMetadata[]::new); } else { // faster to iterate resolvedExpressions aliasCandidates = resolvedExpressions.stream() + .map(ResolvedExpression::resource) .map(indexAliases::get) .filter(Objects::nonNull) .toArray(AliasMetadata[]::new); } List aliases = null; - for (AliasMetadata aliasMetadata : aliasCandidates) { + for (int i = 0; i < aliasCandidates.length; i++) { + AliasMetadata aliasMetadata = aliasCandidates[i]; if (requiredAlias.test(aliasMetadata)) { // If required - add it to the list of aliases if (aliases == null) { @@ -907,12 +946,7 @@ public Map> resolveSearchRouting(ClusterState state, @Nullab getSystemIndexAccessPredicate(), getNetNewSystemIndexPredicate() ); - final Collection resolvedExpressions = resolveExpressions(context, expressions); - - // TODO: it appears that this can never be true? - if (isAllIndices(resolvedExpressions)) { - return resolveSearchRoutingAllIndices(state.metadata(), routing); - } + final Collection resolvedExpressions = resolveExpressionsToResources(context, expressions); Map> routings = null; Set paramRouting = null; @@ -922,10 +956,11 @@ public Map> resolveSearchRouting(ClusterState state, @Nullab paramRouting = Sets.newHashSet(Strings.splitStringByCommaToArray(routing)); } - for (String expression : resolvedExpressions) { - IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(expression); + for (ResolvedExpression resolvedExpression : resolvedExpressions) { + IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(resolvedExpression.resource()); if (indexAbstraction != null && indexAbstraction.getType() == Type.ALIAS) { - for (Index index : indexAbstraction.getIndices()) { + for (int i = 0, n = indexAbstraction.getIndices().size(); i < n; i++) { + Index index = indexAbstraction.getIndices().get(i); String concreteIndex = index.getName(); if (norouting.contains(concreteIndex) == false) { AliasMetadata aliasMetadata = state.metadata().index(concreteIndex).getAliases().get(indexAbstraction.getName()); @@ -954,14 +989,15 @@ public Map> resolveSearchRouting(ClusterState state, @Nullab continue; } if (dataStream.getIndices() != null) { - for (Index index : dataStream.getIndices()) { + for (int i = 0, n = dataStream.getIndices().size(); i < n; i++) { + Index index = dataStream.getIndices().get(i); String concreteIndex = index.getName(); routings = collectRoutings(routings, paramRouting, norouting, concreteIndex); } } } else { // Index - routings = collectRoutings(routings, paramRouting, norouting, expression); + routings = collectRoutings(routings, paramRouting, norouting, resolvedExpression.resource()); } } @@ -999,14 +1035,38 @@ public static Map> resolveSearchRoutingAllIndices(Metadata m Set r = Sets.newHashSet(Strings.splitStringByCommaToArray(routing)); Map> routings = new HashMap<>(); String[] concreteIndices = metadata.getConcreteAllIndices(); - for (String index : concreteIndices) { - routings.put(index, r); + for (int i = 0; i < concreteIndices.length; i++) { + routings.put(concreteIndices[i], r); } return routings; } return null; } + /** + * Identifies whether the array containing index names given as argument refers to all indices + * The empty or null array identifies all indices + * + * @param aliasesOrIndices the array containing index names + * @return true if the provided array maps to all indices, false otherwise + */ + public static boolean isAllIndicesExpression(Collection aliasesOrIndices) { + return aliasesOrIndices == null || aliasesOrIndices.isEmpty() || isExplicitAllPatternExpression(aliasesOrIndices); + } + + /** + * Identifies whether the array containing index names given as argument explicitly refers to all indices + * The empty or null array doesn't explicitly map to all indices + * + * @param aliasesOrIndices the array containing index names + * @return true if the provided array explicitly maps to all indices, false otherwise + */ + static boolean isExplicitAllPatternExpression(Collection aliasesOrIndices) { + return aliasesOrIndices != null + && aliasesOrIndices.size() == 1 + && Metadata.ALL.equals(aliasesOrIndices.iterator().next().resource()); + } + /** * Identifies whether the array containing index names given as argument refers to all indices * The empty or null array identifies all indices @@ -1029,6 +1089,16 @@ static boolean isExplicitAllPattern(Collection aliasesOrIndices) { return aliasesOrIndices != null && aliasesOrIndices.size() == 1 && Metadata.ALL.equals(aliasesOrIndices.iterator().next()); } + /** + * Identifies if this expression list is *,-* which effectively means a request that requests no indices. + */ + static boolean isNoneExpression(String[] expressions) { + return expressions.length == 2 && "*".equals(expressions[0]) && "-*".equals(expressions[1]); + } + + /** + * @return the system access level that will be applied in this resolution. See {@link SystemIndexAccessLevel} for details. + */ public SystemIndexAccessLevel getSystemIndexAccessLevel() { final SystemIndexAccessLevel accessLevel = SystemIndices.getSystemIndexAccessLevel(threadContext); assert accessLevel != SystemIndexAccessLevel.BACKWARDS_COMPATIBLE_ONLY @@ -1036,6 +1106,14 @@ public SystemIndexAccessLevel getSystemIndexAccessLevel() { return accessLevel; } + /** + * Determines the right predicate based on the {@link IndexNameExpressionResolver#getSystemIndexAccessLevel()}. Specifically: + * - NONE implies no access to net-new system indices and data streams + * - BACKWARDS_COMPATIBLE_ONLY allows access also to net-new system resources + * - ALL allows access to everything + * - otherwise we fall back to {@link SystemIndices#getProductSystemIndexNamePredicate(ThreadContext)} + * @return the predicate that defines the access to system indices. + */ public Predicate getSystemIndexAccessPredicate() { final SystemIndexAccessLevel systemIndexAccessLevel = getSystemIndexAccessLevel(); final Predicate systemIndexAccessLevelPredicate; @@ -1060,6 +1138,43 @@ public Predicate getNetNewSystemIndexPredicate() { return systemIndices::isNetNewSystemIndex; } + /** + * This returns `true` if the given {@param name} is of a resource that exists. + * Otherwise, it returns `false` if the `ignore_unvailable` option is `true`, or, if `false`, it throws a "not found" type of + * exception. + */ + @Nullable + private static boolean ensureAliasOrIndexExists(Context context, String name) { + boolean ignoreUnavailable = context.getOptions().ignoreUnavailable(); + IndexAbstraction indexAbstraction = context.getState().getMetadata().getIndicesLookup().get(name); + if (indexAbstraction == null) { + if (ignoreUnavailable) { + return false; + } else { + throw notFoundException(name); + } + } + // treat aliases as unavailable indices when ignoreAliases is set to true (e.g. delete index and update aliases api) + if (indexAbstraction.getType() == Type.ALIAS && context.getOptions().ignoreAliases()) { + if (ignoreUnavailable) { + return false; + } else { + throw aliasesNotSupportedException(name); + } + } + if (indexAbstraction.isDataStreamRelated() && context.includeDataStreams() == false) { + if (ignoreUnavailable) { + return false; + } else { + IndexNotFoundException infe = notFoundException(name); + // Allows callers to handle IndexNotFoundException differently based on whether data streams were excluded. + infe.addMetadata(EXCLUDED_DATA_STREAMS_KEY, "true"); + throw infe; + } + } + return true; + } + public static class Context { private final ClusterState state; @@ -1235,7 +1350,7 @@ public Predicate getSystemIndexAccessPredicate() { } /** - * Resolves alias/index name expressions with wildcards into the corresponding concrete indices/aliases + * Resolves name expressions with wildcards into the corresponding concrete indices/aliases/data streams */ static final class WildcardExpressionResolver { @@ -1244,26 +1359,27 @@ private WildcardExpressionResolver() { } /** - * Returns all the indices, datastreams, and aliases, considering the open/closed, system, and hidden context parameters. - * Depending on the context, returns the names of the datastreams themselves or their backing indices. + * Returns all the indices, data streams, and aliases, considering the open/closed, system, and hidden context parameters. + * Depending on the context, returns the names of the data streams themselves or their backing indices. */ - public static Collection resolveAll(Context context) { - List concreteIndices = resolveEmptyOrTrivialWildcard(context); + public static Collection resolveAll(Context context) { + List concreteIndices = resolveEmptyOrTrivialWildcard(context); if (context.includeDataStreams() == false && context.getOptions().ignoreAliases()) { return concreteIndices; } - Stream ias = context.getState() + Set resolved = new HashSet<>(concreteIndices.size()); + context.getState() .metadata() .getIndicesLookup() .values() .stream() .filter(ia -> context.getOptions().expandWildcardsHidden() || ia.isHidden() == false) .filter(ia -> shouldIncludeIfDataStream(ia, context) || shouldIncludeIfAlias(ia, context)) - .filter(ia -> ia.isSystem() == false || context.systemIndexAccessPredicate.test(ia.getName())); + .filter(ia -> ia.isSystem() == false || context.systemIndexAccessPredicate.test(ia.getName())) + .forEach(ia -> resolved.addAll(expandToOpenClosed(context, ia))); - Set resolved = expandToOpenClosed(context, ias).collect(Collectors.toSet()); resolved.addAll(concreteIndices); return resolved; } @@ -1276,56 +1392,6 @@ private static boolean shouldIncludeIfAlias(IndexAbstraction ia, IndexNameExpres return context.getOptions().ignoreAliases() == false && ia.getType() == Type.ALIAS; } - /** - * Returns all the existing resource (index, alias and datastream) names that the {@param expressions} list resolves to. - * The passed-in {@param expressions} can contain wildcards and exclusions, as well as plain resource names. - *
    - * The return is a {@code Collection} (usually a {@code Set} but can also be a {@code List}, for performance reasons) of plain - * resource names only. All the returned resources are "accessible", in the given context, i.e. the resources exist - * and are not an alias or a datastream if the context does not permit it. - * Wildcard expressions, depending on the context: - *

      - *
    1. might throw an exception if they don't resolve to anything
    2. - *
    3. might not resolve to hidden or system resources (but plain names can refer to hidden or system resources)
    4. - *
    5. might resolve to aliases and datastreams, and it could be (depending on the context) that their backing indices are what's - * ultimately returned, instead of the alias or datastream name
    6. - *
    - */ - public static Collection resolve(Context context, List expressions) { - ExpressionList expressionList = new ExpressionList(context, expressions); - // fast exit if there are no wildcards to evaluate - if (expressionList.hasWildcard() == false) { - return expressions; - } - Set result = new HashSet<>(); - for (ExpressionList.Expression expression : expressionList) { - if (expression.isWildcard()) { - Stream matchingResources = matchResourcesToWildcard(context, expression.get()); - Stream matchingOpenClosedNames = expandToOpenClosed(context, matchingResources); - AtomicBoolean emptyWildcardExpansion = new AtomicBoolean(false); - if (context.getOptions().allowNoIndices() == false) { - emptyWildcardExpansion.set(true); - matchingOpenClosedNames = matchingOpenClosedNames.peek(x -> emptyWildcardExpansion.set(false)); - } - if (expression.isExclusion()) { - matchingOpenClosedNames.forEachOrdered(result::remove); - } else { - matchingOpenClosedNames.forEachOrdered(result::add); - } - if (emptyWildcardExpansion.get()) { - throw notFoundException(expression.get()); - } - } else { - if (expression.isExclusion()) { - result.remove(expression.get()); - } else { - result.add(expression.get()); - } - } - } - return result; - } - private static IndexMetadata.State excludeState(IndicesOptions options) { final IndexMetadata.State excludeState; if (options.expandWildcardsOpen() && options.expandWildcardsClosed()) { @@ -1342,55 +1408,82 @@ private static IndexMetadata.State excludeState(IndicesOptions options) { } /** - * Given a single wildcard {@param expression}, return the {@code Stream} that contains all the resources (i.e. indices, aliases, - * and datastreams), that exist in the cluster at this moment in time, and that the wildcard "resolves" to (i.e. the resource's + * Given a single wildcard {@param expression}, return a {@code Set} that contains all the resources (i.e. indices, aliases, + * and data streams), that exist in the cluster at this moment in time, and that the wildcard "resolves" to (i.e. the resource's * name matches the {@param expression} wildcard). * The {@param context} provides the current time-snapshot view of cluster state, as well as conditions - * on whether to consider alias, datastream, system, and hidden resources. - * It does NOT consider the open or closed status of index resources. + * on whether to consider alias, data stream, system, and hidden resources. */ - private static Stream matchResourcesToWildcard(Context context, String wildcardExpression) { + static Set matchWildcardToResources(Context context, String wildcardExpression) { assert isWildcard(wildcardExpression); final SortedMap indicesLookup = context.getState().getMetadata().getIndicesLookup(); - Stream matchesStream; + Set matchedResources = new HashSet<>(); + // this applies an initial pre-filtering in the case where the expression is a common suffix wildcard, eg "test*" if (Regex.isSuffixMatchPattern(wildcardExpression)) { - // this is an initial pre-filtering in the case where the expression is a common suffix wildcard, eg "test*" - matchesStream = filterIndicesLookupForSuffixWildcard(indicesLookup, wildcardExpression).values().stream(); - } else { - matchesStream = indicesLookup.values().stream(); - if (Regex.isMatchAllPattern(wildcardExpression) == false) { - matchesStream = matchesStream.filter( - indexAbstraction -> Regex.simpleMatch(wildcardExpression, indexAbstraction.getName()) - ); + for (IndexAbstraction ia : filterIndicesLookupForSuffixWildcard(indicesLookup, wildcardExpression).values()) { + maybeAddToResult(context, wildcardExpression, ia, matchedResources); } + return matchedResources; } - if (context.getOptions().ignoreAliases()) { - matchesStream = matchesStream.filter(indexAbstraction -> indexAbstraction.getType() != Type.ALIAS); + // In case of match all it fetches all index abstractions + if (Regex.isMatchAllPattern(wildcardExpression)) { + for (IndexAbstraction ia : indicesLookup.values()) { + maybeAddToResult(context, wildcardExpression, ia, matchedResources); + } + return matchedResources; } - if (context.includeDataStreams() == false) { - matchesStream = matchesStream.filter(indexAbstraction -> indexAbstraction.isDataStreamRelated() == false); + for (IndexAbstraction indexAbstraction : indicesLookup.values()) { + if (Regex.simpleMatch(wildcardExpression, indexAbstraction.getName())) { + maybeAddToResult(context, wildcardExpression, indexAbstraction, matchedResources); + } } - // historic, i.e. not net-new, system indices are included irrespective of the system access predicate - // the system access predicate is based on the endpoint kind and HTTP request headers that identify the stack feature - matchesStream = matchesStream.filter( - indexAbstraction -> indexAbstraction.isSystem() == false - || (indexAbstraction.getType() != Type.DATA_STREAM - && indexAbstraction.getParentDataStream() == null - && context.netNewSystemIndexPredicate.test(indexAbstraction.getName()) == false) - || context.systemIndexAccessPredicate.test(indexAbstraction.getName()) - ); + return matchedResources; + } + + private static void maybeAddToResult( + Context context, + String wildcardExpression, + IndexAbstraction indexAbstraction, + Set matchedResources + ) { + if (shouldExpandToIndexAbstraction(context, wildcardExpression, indexAbstraction)) { + matchedResources.addAll(expandToOpenClosed(context, indexAbstraction)); + } + } + + /** + * Checks if this index abstraction should be included because it matched the wildcard expression. + * @param context the options of this request that influence the decision if this index abstraction should be included in the result + * @param wildcardExpression the wildcard expression that matched this index abstraction + * @param indexAbstraction the index abstraction in question + * @return true, if the index abstraction should be included in the result + */ + private static boolean shouldExpandToIndexAbstraction( + Context context, + String wildcardExpression, + IndexAbstraction indexAbstraction + ) { + if (context.getOptions().ignoreAliases() && indexAbstraction.getType() == Type.ALIAS) { + return false; + } + if (context.includeDataStreams() == false && indexAbstraction.isDataStreamRelated()) { + return false; + } + + if (indexAbstraction.isSystem() + && SystemResourceAccess.shouldExpandToSystemIndexAbstraction(context, indexAbstraction) == false) { + return false; + } + if (context.getOptions().expandWildcardsHidden() == false) { - if (wildcardExpression.startsWith(".")) { - // there is this behavior that hidden indices that start with "." are not hidden if the wildcard expression also - // starts with "." - matchesStream = matchesStream.filter( - indexAbstraction -> indexAbstraction.isHidden() == false || indexAbstraction.getName().startsWith(".") - ); - } else { - matchesStream = matchesStream.filter(indexAbstraction -> indexAbstraction.isHidden() == false); + // there is this behavior that hidden indices that start with "." are not hidden if the wildcard expression also + // starts with "." + if (indexAbstraction.isHidden() + && (wildcardExpression.startsWith(".") && indexAbstraction.getName().startsWith(".")) == false) { + return false; } } - return matchesStream; + return true; } private static Map filterIndicesLookupForSuffixWildcard( @@ -1406,67 +1499,78 @@ private static Map filterIndicesLookupForSuffixWildcar } /** - * Return the {@code Stream} of open and/or closed index names for the given {@param resources}. + * Return the {@code Set} of open and/or closed index names for the given {@param resources}. * Data streams and aliases are interpreted to refer to multiple indices, * then all index resources are filtered by their open/closed status. */ - private static Stream expandToOpenClosed(Context context, Stream resources) { + private static Set expandToOpenClosed(Context context, IndexAbstraction indexAbstraction) { final IndexMetadata.State excludeState = excludeState(context.getOptions()); - return resources.flatMap(indexAbstraction -> { - if (context.isPreserveAliases() && indexAbstraction.getType() == Type.ALIAS) { - return Stream.of(indexAbstraction.getName()); - } else if (context.isPreserveDataStreams() && indexAbstraction.getType() == Type.DATA_STREAM) { - return Stream.of(indexAbstraction.getName()); - } else { - Stream indicesStateStream = Stream.of(); - if (shouldIncludeRegularIndices(context.getOptions())) { - indicesStateStream = indexAbstraction.getIndices().stream().map(context.state.metadata()::index); - } - if (indexAbstraction.getType() == Type.DATA_STREAM && shouldIncludeFailureIndices(context.getOptions())) { - DataStream dataStream = (DataStream) indexAbstraction; - indicesStateStream = Stream.concat( - indicesStateStream, - dataStream.getFailureIndices().getIndices().stream().map(context.state.metadata()::index) - ); + Set resources = new HashSet<>(); + if (context.isPreserveAliases() && indexAbstraction.getType() == Type.ALIAS) { + resources.add(new ResolvedExpression(indexAbstraction.getName())); + } else if (context.isPreserveDataStreams() && indexAbstraction.getType() == Type.DATA_STREAM) { + resources.add(new ResolvedExpression(indexAbstraction.getName())); + } else { + if (shouldIncludeRegularIndices(context.getOptions())) { + for (int i = 0, n = indexAbstraction.getIndices().size(); i < n; i++) { + Index index = indexAbstraction.getIndices().get(i); + IndexMetadata indexMetadata = context.state.metadata().index(index); + if (indexMetadata.getState() != excludeState) { + resources.add(new ResolvedExpression(index.getName())); + } } - if (excludeState != null) { - indicesStateStream = indicesStateStream.filter(indexMeta -> indexMeta.getState() != excludeState); + } + if (indexAbstraction.getType() == Type.DATA_STREAM && shouldIncludeFailureIndices(context.getOptions())) { + DataStream dataStream = (DataStream) indexAbstraction; + for (int i = 0, n = dataStream.getFailureIndices().getIndices().size(); i < n; i++) { + Index index = dataStream.getFailureIndices().getIndices().get(i); + IndexMetadata indexMetadata = context.state.metadata().index(index); + if (indexMetadata.getState() != excludeState) { + resources.add(new ResolvedExpression(index.getName())); + } } - return indicesStateStream.map(indexMeta -> indexMeta.getIndex().getName()); } - }); + } + return resources; } - private static List resolveEmptyOrTrivialWildcard(Context context) { + private static List resolveEmptyOrTrivialWildcard(Context context) { final String[] allIndices = resolveEmptyOrTrivialWildcardToAllIndices(context.getOptions(), context.getState().metadata()); if (context.systemIndexAccessLevel == SystemIndexAccessLevel.ALL) { - return List.of(allIndices); + List result = new ArrayList<>(allIndices.length); + for (int i = 0; i < allIndices.length; i++) { + result.add(new ResolvedExpression(allIndices[i])); + } + return result; } else { return resolveEmptyOrTrivialWildcardWithAllowedSystemIndices(context, allIndices); } } - private static List resolveEmptyOrTrivialWildcardWithAllowedSystemIndices(Context context, String[] allIndices) { - return Arrays.stream(allIndices).filter(name -> { - if (name.startsWith(".")) { - IndexAbstraction abstraction = context.state.metadata().getIndicesLookup().get(name); - assert abstraction != null : "null abstraction for " + name + " but was in array of all indices"; - if (abstraction.isSystem()) { - if (context.netNewSystemIndexPredicate.test(name)) { - if (SystemIndexAccessLevel.BACKWARDS_COMPATIBLE_ONLY.equals(context.systemIndexAccessLevel)) { - return false; - } else { - return context.systemIndexAccessPredicate.test(name); - } - } else if (abstraction.getType() == Type.DATA_STREAM || abstraction.getParentDataStream() != null) { - return context.systemIndexAccessPredicate.test(name); - } - } else { - return true; - } + private static List resolveEmptyOrTrivialWildcardWithAllowedSystemIndices( + Context context, + String[] allIndices + ) { + List filteredIndices = new ArrayList<>(allIndices.length); + for (int i = 0; i < allIndices.length; i++) { + if (shouldIncludeIndexAbstraction(context, allIndices[i])) { + filteredIndices.add(new ResolvedExpression(allIndices[i])); } + } + return filteredIndices; + } + + private static boolean shouldIncludeIndexAbstraction(Context context, String name) { + if (name.startsWith(".") == false) { + return true; + } + + IndexAbstraction abstraction = context.state.metadata().getIndicesLookup().get(name); + assert abstraction != null : "null abstraction for " + name + " but was in array of all indices"; + if (abstraction.isSystem() == false) { return true; - }).toList(); + } + return SystemResourceAccess.isSystemIndexAbstractionAccessible(context, abstraction); } private static String[] resolveEmptyOrTrivialWildcardToAllIndices(IndicesOptions options, Metadata metadata) { @@ -1489,8 +1593,39 @@ private static String[] resolveEmptyOrTrivialWildcardToAllIndices(IndicesOptions return Strings.EMPTY_ARRAY; } } + + static boolean isWildcard(String expression) { + return Regex.isSimpleMatchPattern(expression); + } + + static boolean hasWildcards(String[] expressions) { + for (int i = 0; i < expressions.length; i++) { + if (isWildcard(expressions[i])) { + return true; + } + } + return false; + } } + /** + * @return If the specified string is data math expression then this method returns the resolved expression. + */ + public static String resolveDateMathExpression(String dateExpression) { + return DateMathExpressionResolver.resolveExpression(dateExpression); + } + + /** + * @param time instant to consider when parsing the expression + * @return If the specified string is data math expression then this method returns the resolved expression. + */ + public static String resolveDateMathExpression(String dateExpression, long time) { + return DateMathExpressionResolver.resolveExpression(dateExpression, () -> time); + } + + /** + * Resolves a date math expression based on the requested time. + */ public static final class DateMathExpressionResolver { private static final DateFormatter DEFAULT_DATE_FORMATTER = DateFormatter.forPattern("uuuu.MM.dd"); @@ -1505,28 +1640,19 @@ private DateMathExpressionResolver() { // utility class } - public static List resolve(Context context, List expressions) { - List result = new ArrayList<>(expressions.size()); - for (ExpressionList.Expression expression : new ExpressionList(context, expressions)) { - result.add(resolveExpression(expression, context::getStartTime)); - } - return result; - } - - static String resolveExpression(String expression) { + /** + * Resolves a date math expression using the current time. This method recognises a date math expression iff when they start with + * %3C and end with %3E. Otherwise, it returns the expression intact. + */ + public static String resolveExpression(String expression) { return resolveExpression(expression, System::currentTimeMillis); } - static String resolveExpression(ExpressionList.Expression expression, LongSupplier getTime) { - if (expression.isExclusion()) { - // accepts date-math exclusions that are of the form "-<...{}>", i.e. the "-" is outside the "<>" date-math template - return "-" + resolveExpression(expression.get(), getTime); - } else { - return resolveExpression(expression.get(), getTime); - } - } - - static String resolveExpression(String expression, LongSupplier getTime) { + /** + * Resolves a date math expression using the provided time. This method recognises a date math expression iff when they start with + * %3C and end with %3E. Otherwise, it returns the expression intact. + */ + public static String resolveExpression(String expression, LongSupplier getTime) { if (expression.startsWith(EXPRESSION_LEFT_BOUND) == false || expression.endsWith(EXPRESSION_RIGHT_BOUND) == false) { return expression; } @@ -1675,178 +1801,133 @@ private static String doResolveExpression(String expression, LongSupplier getTim } } - public static final class ExplicitResourceNameFilter { + /** + * In this class we collect the system access relevant code. The helper methods provide the following functionalities: + * - determining the access to a system index abstraction + * - verifying the access to system abstractions and adding the necessary warnings + * - determining the access to a system index based on its name + * WARNING: we have observed differences in how the access is determined. For now this behaviour is documented and preserved. + */ + public static final class SystemResourceAccess { - private ExplicitResourceNameFilter() { + private SystemResourceAccess() { // Utility class } /** - * Returns an expression list with "unavailable" (missing or not acceptable) resource names filtered out. - * Only explicit resource names are considered for filtering. Wildcard and exclusion expressions are kept in. - */ - public static List filterUnavailable(Context context, List expressions) { - ensureRemoteIndicesRequireIgnoreUnavailable(context.getOptions(), expressions); - List result = new ArrayList<>(expressions.size()); - for (ExpressionList.Expression expression : new ExpressionList(context, expressions)) { - validateAliasOrIndex(expression); - if (expression.isWildcard() || expression.isExclusion() || ensureAliasOrIndexExists(context, expression.get())) { - result.add(expression.expression()); - } - } - return result; - } - - /** - * This returns `true` if the given {@param name} is of a resource that exists. - * Otherwise, it returns `false` if the `ignore_unvailable` option is `true`, or, if `false`, it throws a "not found" type of - * exception. + * Checks if this system index abstraction should be included when resolving via {@link + * IndexNameExpressionResolver.WildcardExpressionResolver#resolveEmptyOrTrivialWildcardWithAllowedSystemIndices(Context, String[])}. + * NOTE: it behaves differently than {@link SystemResourceAccess#shouldExpandToSystemIndexAbstraction(Context, IndexAbstraction)} + * because in the case that the access level is BACKWARDS_COMPATIBLE_ONLY it does not include the net-new indices, this is + * questionable. */ - @Nullable - private static boolean ensureAliasOrIndexExists(Context context, String name) { - boolean ignoreUnavailable = context.getOptions().ignoreUnavailable(); - IndexAbstraction indexAbstraction = context.getState().getMetadata().getIndicesLookup().get(name); - if (indexAbstraction == null) { - if (ignoreUnavailable) { - return false; - } else { - throw notFoundException(name); - } - } - // treat aliases as unavailable indices when ignoreAliases is set to true (e.g. delete index and update aliases api) - if (indexAbstraction.getType() == Type.ALIAS && context.getOptions().ignoreAliases()) { - if (ignoreUnavailable) { + public static boolean isSystemIndexAbstractionAccessible(Context context, IndexAbstraction abstraction) { + assert abstraction.isSystem() : "We should only check this for system resources"; + if (context.netNewSystemIndexPredicate.test(abstraction.getName())) { + if (SystemIndexAccessLevel.BACKWARDS_COMPATIBLE_ONLY.equals(context.systemIndexAccessLevel)) { return false; } else { - throw aliasesNotSupportedException(name); - } - } - if (indexAbstraction.isDataStreamRelated() && context.includeDataStreams() == false) { - if (ignoreUnavailable) { - return false; - } else { - IndexNotFoundException infe = notFoundException(name); - // Allows callers to handle IndexNotFoundException differently based on whether data streams were excluded. - infe.addMetadata(EXCLUDED_DATA_STREAMS_KEY, "true"); - throw infe; + return context.systemIndexAccessPredicate.test(abstraction.getName()); } + } else if (abstraction.getType() == Type.DATA_STREAM || abstraction.getParentDataStream() != null) { + return context.systemIndexAccessPredicate.test(abstraction.getName()); } return true; } - private static void validateAliasOrIndex(ExpressionList.Expression expression) { - if (Strings.isEmpty(expression.expression())) { - throw notFoundException(expression.expression()); - } - // Expressions can not start with an underscore. This is reserved for APIs. If the check gets here, the API - // does not exist and the path is interpreted as an expression. If the expression begins with an underscore, - // throw a specific error that is different from the [[IndexNotFoundException]], which is typically thrown - // if the expression can't be found. - if (expression.expression().charAt(0) == '_') { - throw new InvalidIndexNameException(expression.expression(), "must not start with '_'."); - } + /** + * Historic, i.e. not net-new, system indices are included irrespective of the system access predicate + * the system access predicate is based on the endpoint kind and HTTP request headers that identify the stack feature. + * A historic system resource, can only be an index since system data streams were added later. + */ + private static boolean shouldExpandToSystemIndexAbstraction(Context context, IndexAbstraction indexAbstraction) { + assert indexAbstraction.isSystem() : "We should only check this for system resources"; + boolean isHistoric = indexAbstraction.getType() != Type.DATA_STREAM + && indexAbstraction.getParentDataStream() == null + && context.netNewSystemIndexPredicate.test(indexAbstraction.getName()) == false; + return isHistoric || context.systemIndexAccessPredicate.test(indexAbstraction.getName()); } - private static void ensureRemoteIndicesRequireIgnoreUnavailable(IndicesOptions options, List indexExpressions) { - if (options.ignoreUnavailable()) { + /** + * Checks if any system indices that should not have been accessible according to the + * {@link Context#getSystemIndexAccessPredicate()} are accessed, and it performs the following actions: + * - if there are historic (aka not net-new) system indices, then it adds a deprecation warning + * - if it contains net-new system indices or system data streams, it throws an exception. + */ + private static void checkSystemIndexAccess(Context context, ThreadContext threadContext, Index... concreteIndices) { + final Predicate systemIndexAccessPredicate = context.getSystemIndexAccessPredicate(); + if (systemIndexAccessPredicate == Predicates.always()) { return; } - for (String index : indexExpressions) { - if (index.contains(":")) { - failOnRemoteIndicesNotIgnoringUnavailable(indexExpressions); - } - } + doCheckSystemIndexAccess(context, systemIndexAccessPredicate, threadContext, concreteIndices); } - private static void failOnRemoteIndicesNotIgnoringUnavailable(List indexExpressions) { - List crossClusterIndices = new ArrayList<>(); - for (String index : indexExpressions) { - if (index.contains(":")) { - crossClusterIndices.add(index); + private static void doCheckSystemIndexAccess( + Context context, + Predicate systemIndexAccessPredicate, + ThreadContext threadContext, + Index... concreteIndices + ) { + final Metadata metadata = context.getState().metadata(); + final List resolvedSystemIndices = new ArrayList<>(); + final List resolvedNetNewSystemIndices = new ArrayList<>(); + final Set resolvedSystemDataStreams = new HashSet<>(); + final SortedMap indicesLookup = metadata.getIndicesLookup(); + boolean matchedIndex = false; + for (int i = 0; i < concreteIndices.length; i++) { + Index concreteIndex = concreteIndices[i]; + IndexMetadata idxMetadata = metadata.index(concreteIndex); + String name = concreteIndex.getName(); + if (idxMetadata.isSystem() && systemIndexAccessPredicate.test(name) == false) { + matchedIndex = true; + IndexAbstraction indexAbstraction = indicesLookup.get(name); + if (indexAbstraction.getParentDataStream() != null) { + resolvedSystemDataStreams.add(indexAbstraction.getParentDataStream().getName()); + } else if (context.netNewSystemIndexPredicate.test(name)) { + resolvedNetNewSystemIndices.add(name); + } else { + resolvedSystemIndices.add(name); + } } } - throw new IllegalArgumentException( - "Cross-cluster calls are not supported in this context but remote indices were requested: " + crossClusterIndices - ); - } - } - - /** - * Used to iterate expression lists and work out which expression item is a wildcard or an exclusion. - */ - public static final class ExpressionList implements Iterable { - private final List expressionsList; - private final boolean hasWildcard; - - public record Expression(String expression, boolean isWildcard, boolean isExclusion) { - public String get() { - if (isExclusion()) { - // drop the leading "-" if exclusion because it is easier for callers to handle it like this - return expression().substring(1); - } else { - return expression(); - } + if (matchedIndex) { + handleMatchedSystemIndices(resolvedSystemIndices, resolvedSystemDataStreams, resolvedNetNewSystemIndices, threadContext); } } - /** - * Creates the expression iterable that can be used to easily check which expression item is a wildcard or an exclusion (or both). - * The {@param context} is used to check if wildcards ought to be considered or not. - */ - public ExpressionList(Context context, List expressionStrings) { - List expressionsList = new ArrayList<>(expressionStrings.size()); - boolean wildcardSeen = false; - for (String expressionString : expressionStrings) { - boolean isExclusion = expressionString.startsWith("-") && wildcardSeen; - if (context.getOptions().expandWildcardExpressions() && isWildcard(expressionString)) { - wildcardSeen = true; - expressionsList.add(new Expression(expressionString, true, isExclusion)); - } else { - expressionsList.add(new Expression(expressionString, false, isExclusion)); - } + private static void handleMatchedSystemIndices( + List resolvedSystemIndices, + Set resolvedSystemDataStreams, + List resolvedNetNewSystemIndices, + ThreadContext threadContext + ) { + if (resolvedSystemIndices.isEmpty() == false) { + Collections.sort(resolvedSystemIndices); + deprecationLogger.warn( + DeprecationCategory.API, + "open_system_index_access", + "this request accesses system indices: {}, but in a future major version, direct access to system " + + "indices will be prevented by default", + resolvedSystemIndices + ); + } + if (resolvedSystemDataStreams.isEmpty() == false) { + throw SystemIndices.dataStreamAccessException(threadContext, resolvedSystemDataStreams); + } + if (resolvedNetNewSystemIndices.isEmpty() == false) { + throw SystemIndices.netNewSystemIndexAccessException(threadContext, resolvedNetNewSystemIndices); } - this.expressionsList = expressionsList; - this.hasWildcard = wildcardSeen; } /** - * Returns {@code true} if the expression contains any wildcard and the options allow wildcard expansion + * Used in {@link IndexNameExpressionResolver#shouldTrackConcreteIndex(Context, Index)} to exclude net-new indices + * when we are in backwards compatible only access level. + * This also feels questionable as well. */ - public boolean hasWildcard() { - return this.hasWildcard; - } - - @Override - public Iterator iterator() { - return expressionsList.iterator(); + private static boolean isNetNewInBackwardCompatibleMode(Context context, Index index) { + return context.systemIndexAccessLevel == SystemIndexAccessLevel.BACKWARDS_COMPATIBLE_ONLY + && context.netNewSystemIndexPredicate.test(index.getName()); } } - /** - * This is a context for the DateMathExpressionResolver which does not require {@code IndicesOptions} or {@code ClusterState} - * since it uses only the start time to resolve expressions. - */ - public static final class ResolverContext extends Context { - public ResolverContext() { - this(System.currentTimeMillis()); - } - - public ResolverContext(long startTime) { - super(null, null, startTime, false, false, false, false, SystemIndexAccessLevel.ALL, Predicates.never(), Predicates.never()); - } - - @Override - public ClusterState getState() { - throw new UnsupportedOperationException("should never be called"); - } - - @Override - public IndicesOptions getOptions() { - throw new UnsupportedOperationException("should never be called"); - } - } - - private static boolean isWildcard(String expression) { - return Regex.isSimpleMatchPattern(expression); - } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadata.java index 35b7d957bf076..6ddcd6a45e4b6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadata.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexWriteLoad.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexWriteLoad.java index 7e3c7d382f803..459f8e0618f33 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexWriteLoad.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexWriteLoad.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/InferenceFieldMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/InferenceFieldMetadata.java index 2ea74aaed35d1..8917d5a9cbbb5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/InferenceFieldMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/InferenceFieldMetadata.java @@ -1,13 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; +import org.elasticsearch.TransportVersions; import org.elasticsearch.cluster.Diff; import org.elasticsearch.cluster.SimpleDiffable; import org.elasticsearch.common.io.stream.StreamInput; @@ -31,21 +33,33 @@ */ public final class InferenceFieldMetadata implements SimpleDiffable, ToXContentFragment { private static final String INFERENCE_ID_FIELD = "inference_id"; + private static final String SEARCH_INFERENCE_ID_FIELD = "search_inference_id"; private static final String SOURCE_FIELDS_FIELD = "source_fields"; private final String name; private final String inferenceId; + private final String searchInferenceId; private final String[] sourceFields; public InferenceFieldMetadata(String name, String inferenceId, String[] sourceFields) { + this(name, inferenceId, inferenceId, sourceFields); + } + + public InferenceFieldMetadata(String name, String inferenceId, String searchInferenceId, String[] sourceFields) { this.name = Objects.requireNonNull(name); this.inferenceId = Objects.requireNonNull(inferenceId); + this.searchInferenceId = Objects.requireNonNull(searchInferenceId); this.sourceFields = Objects.requireNonNull(sourceFields); } public InferenceFieldMetadata(StreamInput input) throws IOException { this.name = input.readString(); this.inferenceId = input.readString(); + if (input.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { + this.searchInferenceId = input.readString(); + } else { + this.searchInferenceId = this.inferenceId; + } this.sourceFields = input.readStringArray(); } @@ -53,6 +67,9 @@ public InferenceFieldMetadata(StreamInput input) throws IOException { public void writeTo(StreamOutput out) throws IOException { out.writeString(name); out.writeString(inferenceId); + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { + out.writeString(searchInferenceId); + } out.writeStringArray(sourceFields); } @@ -63,12 +80,13 @@ public boolean equals(Object o) { InferenceFieldMetadata that = (InferenceFieldMetadata) o; return Objects.equals(name, that.name) && Objects.equals(inferenceId, that.inferenceId) + && Objects.equals(searchInferenceId, that.searchInferenceId) && Arrays.equals(sourceFields, that.sourceFields); } @Override public int hashCode() { - int result = Objects.hash(name, inferenceId); + int result = Objects.hash(name, inferenceId, searchInferenceId); result = 31 * result + Arrays.hashCode(sourceFields); return result; } @@ -81,6 +99,10 @@ public String getInferenceId() { return inferenceId; } + public String getSearchInferenceId() { + return searchInferenceId; + } + public String[] getSourceFields() { return sourceFields; } @@ -93,6 +115,9 @@ public static Diff readDiffFrom(StreamInput in) throws I public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(name); builder.field(INFERENCE_ID_FIELD, inferenceId); + if (searchInferenceId.equals(inferenceId) == false) { + builder.field(SEARCH_INFERENCE_ID_FIELD, searchInferenceId); + } builder.array(SOURCE_FIELDS_FIELD, sourceFields); return builder.endObject(); } @@ -105,6 +130,7 @@ public static InferenceFieldMetadata fromXContent(XContentParser parser) throws String currentFieldName = null; String inferenceId = null; + String searchInferenceId = null; List inputFields = new ArrayList<>(); while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { @@ -112,6 +138,8 @@ public static InferenceFieldMetadata fromXContent(XContentParser parser) throws } else if (token == XContentParser.Token.VALUE_STRING) { if (INFERENCE_ID_FIELD.equals(currentFieldName)) { inferenceId = parser.text(); + } else if (SEARCH_INFERENCE_ID_FIELD.equals(currentFieldName)) { + searchInferenceId = parser.text(); } } else if (token == XContentParser.Token.START_ARRAY) { if (SOURCE_FIELDS_FIELD.equals(currentFieldName)) { @@ -127,6 +155,11 @@ public static InferenceFieldMetadata fromXContent(XContentParser parser) throws parser.skipChildren(); } } - return new InferenceFieldMetadata(name, inferenceId, inputFields.toArray(String[]::new)); + return new InferenceFieldMetadata( + name, + inferenceId, + searchInferenceId == null ? inferenceId : searchInferenceId, + inputFields.toArray(String[]::new) + ); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ItemUsage.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ItemUsage.java index 2e0c3194fb9ea..2a6873ab37313 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ItemUsage.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ItemUsage.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/LifecycleExecutionState.java b/server/src/main/java/org/elasticsearch/cluster/metadata/LifecycleExecutionState.java index 497b0991517d3..abc0983ccb2d4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/LifecycleExecutionState.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/LifecycleExecutionState.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; @@ -27,6 +28,7 @@ public record LifecycleExecutionState( Boolean isAutoRetryableError, Integer failedStepRetryCount, String stepInfo, + String previousStepInfo, String phaseDefinition, Long lifecycleDate, Long phaseTime, @@ -52,6 +54,7 @@ public record LifecycleExecutionState( private static final String IS_AUTO_RETRYABLE_ERROR = "is_auto_retryable_error"; private static final String FAILED_STEP_RETRY_COUNT = "failed_step_retry_count"; private static final String STEP_INFO = "step_info"; + private static final String PREVIOUS_STEP_INFO = "previous_step_info"; private static final String PHASE_DEFINITION = "phase_definition"; private static final String SNAPSHOT_NAME = "snapshot_name"; private static final String SNAPSHOT_REPOSITORY = "snapshot_repository"; @@ -73,6 +76,7 @@ public static Builder builder(LifecycleExecutionState state) { .setIsAutoRetryableError(state.isAutoRetryableError) .setFailedStepRetryCount(state.failedStepRetryCount) .setStepInfo(state.stepInfo) + .setPreviousStepInfo(state.previousStepInfo) .setPhaseDefinition(state.phaseDefinition) .setIndexCreationDate(state.lifecycleDate) .setPhaseTime(state.phaseTime) @@ -115,6 +119,10 @@ public static LifecycleExecutionState fromCustomMetadata(Map cus if (stepInfo != null) { builder.setStepInfo(stepInfo); } + String previousStepInfo = customData.get(PREVIOUS_STEP_INFO); + if (previousStepInfo != null) { + builder.setPreviousStepInfo(previousStepInfo); + } String phaseDefinition = customData.get(PHASE_DEFINITION); if (phaseDefinition != null) { builder.setPhaseDefinition(phaseDefinition); @@ -223,6 +231,9 @@ public Map asMap() { if (stepInfo != null) { result.put(STEP_INFO, stepInfo); } + if (previousStepInfo != null) { + result.put(PREVIOUS_STEP_INFO, previousStepInfo); + } if (lifecycleDate != null) { result.put(INDEX_CREATION_DATE, String.valueOf(lifecycleDate)); } @@ -262,6 +273,7 @@ public static class Builder { private String step; private String failedStep; private String stepInfo; + private String previousStepInfo; private String phaseDefinition; private Long indexCreationDate; private Long phaseTime; @@ -300,6 +312,11 @@ public Builder setStepInfo(String stepInfo) { return this; } + public Builder setPreviousStepInfo(String previousStepInfo) { + this.previousStepInfo = previousStepInfo; + return this; + } + public Builder setPhaseDefinition(String phaseDefinition) { this.phaseDefinition = phaseDefinition; return this; @@ -369,6 +386,7 @@ public LifecycleExecutionState build() { isAutoRetryableError, failedStepRetryCount, stepInfo, + previousStepInfo, phaseDefinition, indexCreationDate, phaseTime, diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java index f611f10f520f8..746abe7b255a3 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Manifest.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MappingMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MappingMetadata.java index b629ab5d5f710..ee4869508eaf2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MappingMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MappingMetadata.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index 905c3078b3c9c..c67e89f6cd1b5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; @@ -29,7 +30,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.VersionedNamedWriteable; @@ -41,7 +41,6 @@ import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.ChunkedToXContent; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.core.Nullable; @@ -696,6 +695,11 @@ public long version() { return this.version; } + /** + * @return A UUID which identifies this cluster. Nodes record the UUID of the cluster they first join on disk, and will then refuse to + * join clusters with different UUIDs. Note that when the cluster is forming for the first time this value may not yet be committed, + * and therefore it may change. Check {@link #clusterUUIDCommitted()} to verify that the value is committed if needed. + */ public String clusterUUID() { return this.clusterUUID; } @@ -1306,27 +1310,15 @@ public Map templatesV2() { .orElse(Collections.emptyMap()); } - public boolean isTimeSeriesTemplate(ComposableIndexTemplate indexTemplate) { + public IndexMode retrieveIndexModeFromTemplate(ComposableIndexTemplate indexTemplate) { if (indexTemplate.getDataStreamTemplate() == null) { - return false; + return null; } var settings = MetadataIndexTemplateService.resolveSettings(indexTemplate, componentTemplates()); // Not using IndexSettings.MODE.get() to avoid validation that may fail at this point. var rawIndexMode = settings.get(IndexSettings.MODE.getKey()); - var indexMode = rawIndexMode != null ? Enum.valueOf(IndexMode.class, rawIndexMode.toUpperCase(Locale.ROOT)) : null; - if (indexMode == IndexMode.TIME_SERIES) { - // No need to check for the existence of index.routing_path here, because index.mode=time_series can't be specified without it. - // Setting validation takes care of this. - // Also no need to validate that the fields defined in index.routing_path are keyword fields with time_series_dimension - // attribute enabled. This is validated elsewhere (DocumentMapper). - return true; - } - - // in a followup change: check the existence of keyword fields of type keyword and time_series_dimension attribute enabled in - // the template. In this case the index.routing_path setting can be generated from the mapping. - - return false; + return rawIndexMode != null ? Enum.valueOf(IndexMode.class, rawIndexMode.toUpperCase(Locale.ROOT)) : null; } public Map dataStreams() { @@ -1508,49 +1500,44 @@ public static Metadata fromXContent(XContentParser parser) throws IOException { } @Override - public Iterator toXContentChunked(ToXContent.Params p) { - XContentContext context = XContentContext.valueOf(p.param(CONTEXT_MODE_PARAM, CONTEXT_MODE_API)); - final Iterator start = context == XContentContext.API - ? ChunkedToXContentHelper.startObject("metadata") - : Iterators.single((builder, params) -> builder.startObject("meta-data").field("version", version())); - - final Iterator persistentSettings = context != XContentContext.API && persistentSettings().isEmpty() == false - ? Iterators.single((builder, params) -> { - builder.startObject("settings"); - persistentSettings().toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true"))); - return builder.endObject(); + public Iterator toXContentChunked(ToXContent.Params params) { + XContentContext context = XContentContext.valueOf(params.param(CONTEXT_MODE_PARAM, CONTEXT_MODE_API)); + + return ChunkedToXContent.builder(params) + .append( + context == XContentContext.API + ? (b, p) -> b.startObject("metadata") + : (b, p) -> b.startObject("meta-data").field("version", version()) + ) + .append((b, p) -> { + b.field("cluster_uuid", clusterUUID); + b.field("cluster_uuid_committed", clusterUUIDCommitted); + b.startObject("cluster_coordination"); + coordinationMetadata().toXContent(b, p); + return b.endObject(); }) - : Collections.emptyIterator(); - - final Iterator indices = context == XContentContext.API - ? ChunkedToXContentHelper.wrapWithObject("indices", indices().values().iterator()) - : Collections.emptyIterator(); - - return Iterators.concat(start, Iterators.single((builder, params) -> { - builder.field("cluster_uuid", clusterUUID); - builder.field("cluster_uuid_committed", clusterUUIDCommitted); - builder.startObject("cluster_coordination"); - coordinationMetadata().toXContent(builder, params); - return builder.endObject(); - }), - persistentSettings, - ChunkedToXContentHelper.wrapWithObject( - "templates", - Iterators.map( - templates().values().iterator(), - template -> (builder, params) -> IndexTemplateMetadata.Builder.toXContentWithTypes(template, builder, params) - ) - ), - indices, - Iterators.flatMap( - customs.entrySet().iterator(), - entry -> entry.getValue().context().contains(context) - ? ChunkedToXContentHelper.wrapWithObject(entry.getKey(), entry.getValue().toXContentChunked(p)) - : Collections.emptyIterator() - ), - ChunkedToXContentHelper.wrapWithObject("reserved_state", reservedStateMetadata().values().iterator()), - ChunkedToXContentHelper.endObject() - ); + .execute(xb -> { + if (context != XContentContext.API && persistentSettings().isEmpty() == false) { + xb.append((b, p) -> { + b.startObject("settings"); + persistentSettings().toXContent(b, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true"))); + return b.endObject(); + }); + } + }) + .object("templates", templates().values().iterator(), t -> (b, p) -> IndexTemplateMetadata.Builder.toXContentWithTypes(t, b, p)) + .execute(xb -> { + if (context == XContentContext.API) { + xb.xContentObject("indices", indices().values().iterator()); + } + }) + .forEach(customs.entrySet().iterator(), (b, e) -> { + if (e.getValue().context().contains(context)) { + b.xContentObject(e.getKey(), e.getValue()); + } + }) + .xContentObject("reserved_state", reservedStateMetadata().values().iterator()) + .append((b, p) -> b.endObject()); } public Map getMappingsByHash() { @@ -1803,7 +1790,6 @@ public static class Builder { private DiffableStringMap hashesOfConsistentSettings = DiffableStringMap.EMPTY; private final ImmutableOpenMap.Builder indices; - private final ImmutableOpenMap.Builder> aliasedIndices; private final ImmutableOpenMap.Builder templates; private final ImmutableOpenMap.Builder customs; @@ -1833,7 +1819,6 @@ public Builder() { this.hashesOfConsistentSettings = metadata.hashesOfConsistentSettings; this.version = metadata.version; this.indices = ImmutableOpenMap.builder(metadata.indices); - this.aliasedIndices = ImmutableOpenMap.builder(metadata.aliasedIndices); this.templates = ImmutableOpenMap.builder(metadata.templates); this.customs = ImmutableOpenMap.builder(metadata.customs); this.previousIndicesLookup = metadata.indicesLookup; @@ -1846,7 +1831,6 @@ public Builder() { private Builder(Map mappingsByHash, int indexCountHint) { clusterUUID = UNKNOWN_CLUSTER_UUID; indices = ImmutableOpenMap.builder(indexCountHint); - aliasedIndices = ImmutableOpenMap.builder(); templates = ImmutableOpenMap.builder(); customs = ImmutableOpenMap.builder(); reservedStateMetadata = new HashMap<>(); @@ -1861,7 +1845,6 @@ public Builder put(IndexMetadata.Builder indexMetadataBuilder) { dedupeMapping(indexMetadataBuilder); IndexMetadata indexMetadata = indexMetadataBuilder.build(); IndexMetadata previous = indices.put(indexMetadata.getIndex().getName(), indexMetadata); - updateAliases(previous, indexMetadata); if (unsetPreviousIndicesLookup(previous, indexMetadata)) { previousIndicesLookup = null; } @@ -1886,7 +1869,6 @@ public Builder put(IndexMetadata indexMetadata, boolean incrementVersion) { return this; } } - updateAliases(previous, indexMetadata); if (unsetPreviousIndicesLookup(previous, indexMetadata)) { previousIndicesLookup = null; } @@ -1961,8 +1943,7 @@ public IndexMetadata getSafe(Index index) { public Builder remove(String index) { previousIndicesLookup = null; checkForUnusedMappings = true; - IndexMetadata previous = indices.remove(index); - updateAliases(previous, null); + indices.remove(index); return this; } @@ -1972,7 +1953,6 @@ public Builder removeAllIndices() { indices.clear(); mappingsByHash.clear(); - aliasedIndices.clear(); return this; } @@ -1983,67 +1963,6 @@ public Builder indices(Map indices) { return this; } - void updateAliases(IndexMetadata previous, IndexMetadata current) { - if (previous == null && current != null) { - for (var key : current.getAliases().keySet()) { - putAlias(key, current.getIndex()); - } - } else if (previous != null && current == null) { - for (var key : previous.getAliases().keySet()) { - removeAlias(key, previous.getIndex()); - } - } else if (previous != null && current != null) { - if (Objects.equals(previous.getAliases(), current.getAliases())) { - return; - } - - for (var key : current.getAliases().keySet()) { - if (previous.getAliases().containsKey(key) == false) { - putAlias(key, current.getIndex()); - } - } - for (var key : previous.getAliases().keySet()) { - if (current.getAliases().containsKey(key) == false) { - removeAlias(key, current.getIndex()); - } - } - } - } - - private Builder putAlias(String alias, Index index) { - Objects.requireNonNull(alias); - Objects.requireNonNull(index); - - Set indices = new HashSet<>(aliasedIndices.getOrDefault(alias, Set.of())); - if (indices.add(index) == false) { - return this; // indices already contained this index - } - aliasedIndices.put(alias, Collections.unmodifiableSet(indices)); - return this; - } - - private Builder removeAlias(String alias, Index index) { - Objects.requireNonNull(alias); - Objects.requireNonNull(index); - - Set indices = aliasedIndices.get(alias); - if (indices == null || indices.isEmpty()) { - throw new IllegalStateException("Cannot remove non-existent alias [" + alias + "] for index [" + index.getName() + "]"); - } - - indices = new HashSet<>(indices); - if (indices.remove(index) == false) { - throw new IllegalStateException("Cannot remove non-existent alias [" + alias + "] for index [" + index.getName() + "]"); - } - - if (indices.isEmpty()) { - aliasedIndices.remove(alias); // for consistency, we don't store empty sets, so null it out - } else { - aliasedIndices.put(alias, Collections.unmodifiableSet(indices)); - } - return this; - } - public Builder put(IndexTemplateMetadata.Builder template) { return put(template.build()); } @@ -2365,6 +2284,7 @@ public Metadata build(boolean skipNameCollisionChecks) { int totalNumberOfShards = 0; int totalOpenIndexShards = 0; + ImmutableOpenMap.Builder> aliasedIndicesBuilder = ImmutableOpenMap.builder(); final String[] allIndicesArray = new String[indicesMap.size()]; int i = 0; final Set sha256HashesInUse = checkForUnusedMappings ? Sets.newHashSetWithExpectedSize(mappingsByHash.size()) : null; @@ -2396,9 +2316,19 @@ public Metadata build(boolean skipNameCollisionChecks) { sha256HashesInUse.add(mapping.getSha256()); } } + for (var alias : indexMetadata.getAliases().keySet()) { + var indices = aliasedIndicesBuilder.get(alias); + if (indices == null) { + indices = new HashSet<>(); + aliasedIndicesBuilder.put(alias, indices); + } + indices.add(indexMetadata.getIndex()); + } } - - var aliasedIndices = this.aliasedIndices.build(); + for (String alias : aliasedIndicesBuilder.keys()) { + aliasedIndicesBuilder.put(alias, Collections.unmodifiableSet(aliasedIndicesBuilder.get(alias))); + } + var aliasedIndices = aliasedIndicesBuilder.build(); for (var entry : aliasedIndices.entrySet()) { List aliasIndices = entry.getValue().stream().map(idx -> indicesMap.get(idx.getName())).toList(); validateAlias(entry.getKey(), aliasIndices); @@ -2599,7 +2529,6 @@ private static boolean assertContainsIndexIfDataStream(DataStream parent, IndexM assert parent == null || parent.getIndices().stream().anyMatch(index -> indexMetadata.getIndex().getName().equals(index.getName())) || (DataStream.isFailureStoreFeatureFlagEnabled() - && parent.isFailureStoreEnabled() && parent.getFailureIndices() .getIndices() .stream() @@ -2625,7 +2554,7 @@ private static void collectDataStreams( for (Index i : dataStream.getIndices()) { indexToDataStreamLookup.put(i.getName(), dataStream); } - if (DataStream.isFailureStoreFeatureFlagEnabled() && dataStream.isFailureStoreEnabled()) { + if (DataStream.isFailureStoreFeatureFlagEnabled()) { for (Index i : dataStream.getFailureIndices().getIndices()) { indexToDataStreamLookup.put(i.getName(), dataStream); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index fdcf4c7354273..254646f8e71a9 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; @@ -20,7 +21,6 @@ import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; -import org.elasticsearch.cluster.ack.ClusterStateUpdateRequest; import org.elasticsearch.cluster.routing.allocation.allocator.AllocationActionListener; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; @@ -98,7 +98,12 @@ public void createDataStream(CreateDataStreamClusterStateUpdateRequest request, var delegate = new AllocationActionListener<>(listener, threadPool.getThreadContext()); submitUnbatchedTask( "create-data-stream [" + request.name + "]", - new AckedClusterStateUpdateTask(Priority.HIGH, request, delegate.clusterStateUpdate()) { + new AckedClusterStateUpdateTask( + Priority.HIGH, + request.masterNodeTimeout(), + request.ackTimeout(), + delegate.clusterStateUpdate() + ) { @Override public ClusterState execute(ClusterState currentState) throws Exception { // When we're manually creating a data stream (i.e. not an auto creation), we don't need to initialize the failure store @@ -137,13 +142,19 @@ public ClusterState createDataStream( ); } - public static final class CreateDataStreamClusterStateUpdateRequest extends ClusterStateUpdateRequest< - CreateDataStreamClusterStateUpdateRequest> { - - private final boolean performReroute; - private final String name; - private final long startTime; - private final SystemDataStreamDescriptor descriptor; + public record CreateDataStreamClusterStateUpdateRequest( + String name, + long startTime, + @Nullable SystemDataStreamDescriptor systemDataStreamDescriptor, + TimeValue masterNodeTimeout, + TimeValue ackTimeout, + boolean performReroute + ) { + public CreateDataStreamClusterStateUpdateRequest { + Objects.requireNonNull(name); + Objects.requireNonNull(masterNodeTimeout); + Objects.requireNonNull(ackTimeout); + } public CreateDataStreamClusterStateUpdateRequest(String name) { this(name, System.currentTimeMillis(), null, TimeValue.ZERO, TimeValue.ZERO, true); @@ -153,42 +164,14 @@ public CreateDataStreamClusterStateUpdateRequest( String name, SystemDataStreamDescriptor systemDataStreamDescriptor, TimeValue masterNodeTimeout, - TimeValue timeout, + TimeValue ackTimeout, boolean performReroute ) { - this(name, System.currentTimeMillis(), systemDataStreamDescriptor, masterNodeTimeout, timeout, performReroute); - } - - public CreateDataStreamClusterStateUpdateRequest( - String name, - long startTime, - SystemDataStreamDescriptor systemDataStreamDescriptor, - TimeValue masterNodeTimeout, - TimeValue timeout, - boolean performReroute - ) { - this.name = name; - this.startTime = startTime; - this.descriptor = systemDataStreamDescriptor; - this.performReroute = performReroute; - masterNodeTimeout(masterNodeTimeout); - ackTimeout(timeout); + this(name, System.currentTimeMillis(), systemDataStreamDescriptor, masterNodeTimeout, ackTimeout, performReroute); } public boolean isSystem() { - return descriptor != null; - } - - public boolean performReroute() { - return performReroute; - } - - public SystemDataStreamDescriptor getSystemDataStreamDescriptor() { - return descriptor; - } - - long getStartTime() { - return startTime; + return systemDataStreamDescriptor != null; } } @@ -222,7 +205,8 @@ static ClusterState createDataStream( * @param request The create data stream request * @param backingIndices List of backing indices. May be empty * @param writeIndex Write index for the data stream. If null, a new write index will be created. - * @param initializeFailureStore Whether the failure store should be initialized + * @param initializeFailureStore Whether the failure store should be initialized (N.B. if true, failure store index creation will be + * performed regardless of whether the template indicates that the failure store is enabled) * @return Cluster state containing the new data stream */ static ClusterState createDataStream( @@ -237,7 +221,7 @@ static ClusterState createDataStream( boolean initializeFailureStore ) throws Exception { String dataStreamName = request.name; - SystemDataStreamDescriptor systemDataStreamDescriptor = request.getSystemDataStreamDescriptor(); + SystemDataStreamDescriptor systemDataStreamDescriptor = request.systemDataStreamDescriptor(); boolean isSystemDataStreamName = metadataCreateIndexService.getSystemIndices().isSystemDataStream(request.name); assert (isSystemDataStreamName && systemDataStreamDescriptor != null) || (isSystemDataStreamName == false && systemDataStreamDescriptor == null) @@ -278,21 +262,27 @@ static ClusterState createDataStream( // This is not a problem as both have different prefixes (`.ds-` vs `.fs-`) and both will be using the same `generation` field // when rolling over in the future. final long initialGeneration = 1; + ResettableValue dataStreamOptionsTemplate = isSystem + ? MetadataIndexTemplateService.resolveDataStreamOptions(template, systemDataStreamDescriptor.getComponentTemplates()) + : MetadataIndexTemplateService.resolveDataStreamOptions(template, metadata.componentTemplates()); + final DataStreamOptions dataStreamOptions = dataStreamOptionsTemplate.mapAndGet(DataStreamOptions.Template::toDataStreamOptions); // If we need to create a failure store, do so first. Do not reroute during the creation since we will do - // that as part of creating the backing index if required. + // that as part of creating the backing index if required. N.B. This is done if initializeFailureStore, + // regardless of whether the template indicates that the failure store is enabled: it is the caller's + // responsibility to check that before setting. IndexMetadata failureStoreIndex = null; - if (template.getDataStreamTemplate().hasFailureStore() && initializeFailureStore) { + if (initializeFailureStore) { if (isSystem) { throw new IllegalArgumentException("Failure stores are not supported on system data streams"); } - String failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, initialGeneration, request.getStartTime()); + String failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, initialGeneration, request.startTime()); currentState = createFailureStoreIndex( metadataCreateIndexService, "initialize_data_stream", settings, currentState, - request.getStartTime(), + request.startTime(), dataStreamName, template, failureStoreIndexName, @@ -302,7 +292,7 @@ static ClusterState createDataStream( } if (writeIndex == null) { - String firstBackingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, initialGeneration, request.getStartTime()); + String firstBackingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, initialGeneration, request.startTime()); currentState = createBackingIndex( metadataCreateIndexService, currentState, @@ -320,8 +310,7 @@ static ClusterState createDataStream( } assert writeIndex != null; assert writeIndex.mapping() != null : "no mapping found for backing index [" + writeIndex.getIndex().getName() + "]"; - assert template.getDataStreamTemplate().hasFailureStore() == false || initializeFailureStore == false || failureStoreIndex != null - : "failure store should have an initial index"; + assert initializeFailureStore == false || failureStoreIndex != null : "failure store should have an initial index"; assert failureStoreIndex == null || failureStoreIndex.mapping() != null : "no mapping found for failure store [" + failureStoreIndex.getIndex().getName() + "]"; @@ -330,7 +319,7 @@ static ClusterState createDataStream( .collect(Collectors.toCollection(ArrayList::new)); dsBackingIndices.add(writeIndex.getIndex()); boolean hidden = isSystem || template.getDataStreamTemplate().isHidden(); - final IndexMode indexMode = metadata.isTimeSeriesTemplate(template) ? IndexMode.TIME_SERIES : null; + final IndexMode indexMode = metadata.retrieveIndexModeFromTemplate(template); final DataStreamLifecycle lifecycle = isSystem ? MetadataIndexTemplateService.resolveLifecycle(template, systemDataStreamDescriptor.getComponentTemplates()) : MetadataIndexTemplateService.resolveLifecycle(template, metadata.componentTemplates()); @@ -346,7 +335,7 @@ static ClusterState createDataStream( template.getDataStreamTemplate().isAllowCustomRouting(), indexMode, lifecycle == null && isDslOnlyMode ? DataStreamLifecycle.DEFAULT : lifecycle, - template.getDataStreamTemplate().hasFailureStore(), + dataStreamOptions, new DataStream.DataStreamIndices(DataStream.BACKING_INDEX_PREFIX, dsBackingIndices, false, null), // If the failure store shouldn't be initialized on data stream creation, we're marking it for "lazy rollover", which will // initialize the failure store on first write. @@ -391,7 +380,7 @@ private static ClusterState createBackingIndex( firstBackingIndexName ).dataStreamName(dataStreamName) .systemDataStreamDescriptor(systemDataStreamDescriptor) - .nameResolvedInstant(request.getStartTime()) + .nameResolvedInstant(request.startTime()) .performReroute(request.performReroute()) .setMatchingTemplate(template); @@ -442,7 +431,8 @@ public static ClusterState createFailureStoreIndex( .nameResolvedInstant(nameResolvedInstant) .performReroute(false) .setMatchingTemplate(template) - .settings(indexSettings); + .settings(indexSettings) + .isFailureIndex(true); try { currentState = metadataCreateIndexService.applyCreateIndexRequest( diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 17db4f9253824..2a7209a2e5e96 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; @@ -51,6 +52,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.PathUtils; import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.env.Environment; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; @@ -60,11 +62,13 @@ import org.elasticsearch.index.IndexSettingProvider; import org.elasticsearch.index.IndexSettingProviders; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexSortConfig; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperService.MergeReason; +import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.shard.IndexLongFieldRange; import org.elasticsearch.indices.IndexCreationException; @@ -253,12 +257,23 @@ public static void validateIndexOrAliasName(String index, BiFunction listener) { + public void createIndex( + final TimeValue masterNodeTimeout, + final TimeValue ackTimeout, + @Nullable final TimeValue waitForActiveShardsTimeout, + final CreateIndexClusterStateUpdateRequest request, + final ActionListener listener + ) { logger.trace("createIndex[{}]", request); - onlyCreateIndex(request, listener.delegateFailureAndWrap((delegate, response) -> { + onlyCreateIndex(masterNodeTimeout, ackTimeout, request, listener.delegateFailureAndWrap((delegate, response) -> { if (response.isAcknowledged()) { logger.trace( "[{}] index creation acknowledged, waiting for active shards [{}]", @@ -269,7 +284,7 @@ public void createIndex(final CreateIndexClusterStateUpdateRequest request, fina clusterService, new String[] { request.index() }, request.waitForActiveShards(), - request.ackTimeout(), + waitForActiveShardsTimeout, delegate.map(shardsAcknowledged -> { if (shardsAcknowledged == false) { logger.debug( @@ -289,13 +304,23 @@ public void createIndex(final CreateIndexClusterStateUpdateRequest request, fina })); } - private void onlyCreateIndex(final CreateIndexClusterStateUpdateRequest request, final ActionListener listener) { - normalizeRequestSetting(request); + private void onlyCreateIndex( + final TimeValue masterNodeTimeout, + final TimeValue ackTimeout, + final CreateIndexClusterStateUpdateRequest request, + final ActionListener listener + ) { + try { + normalizeRequestSetting(request); + } catch (Exception e) { + listener.onFailure(e); + return; + } var delegate = new AllocationActionListener<>(listener, threadPool.getThreadContext()); submitUnbatchedTask( "create-index [" + request.index() + "], cause [" + request.cause() + "]", - new AckedClusterStateUpdateTask(Priority.URGENT, request, delegate.clusterStateUpdate()) { + new AckedClusterStateUpdateTask(Priority.URGENT, masterNodeTimeout, ackTimeout, delegate.clusterStateUpdate()) { @Override public ClusterState execute(ClusterState currentState) throws Exception { @@ -962,69 +987,82 @@ static Settings aggregateIndexSettings( final Settings.Builder indexSettingsBuilder = Settings.builder(); if (sourceMetadata == null) { - final Settings.Builder additionalIndexSettings = Settings.builder(); final Settings templateAndRequestSettings = Settings.builder().put(combinedTemplateSettings).put(request.settings()).build(); - final boolean timeSeriesTemplate = Optional.of(request) + final IndexMode templateIndexMode = Optional.of(request) + .filter(r -> r.isFailureIndex() == false) .map(CreateIndexClusterStateUpdateRequest::matchingTemplate) - .map(metadata::isTimeSeriesTemplate) - .orElse(false); + .map(metadata::retrieveIndexModeFromTemplate) + .orElse(null); // Loop through all the explicit index setting providers, adding them to the // additionalIndexSettings map + final Settings.Builder additionalIndexSettings = Settings.builder(); final var resolvedAt = Instant.ofEpochMilli(request.getNameResolvedAt()); + Set overrulingSettings = new HashSet<>(); for (IndexSettingProvider provider : indexSettingProviders) { - additionalIndexSettings.put( - provider.getAdditionalIndexSettings( - request.index(), - request.dataStreamName(), - timeSeriesTemplate, - currentState.getMetadata(), - resolvedAt, - templateAndRequestSettings, - combinedTemplateMappings - ) + var newAdditionalSettings = provider.getAdditionalIndexSettings( + request.index(), + request.dataStreamName(), + templateIndexMode, + currentState.getMetadata(), + resolvedAt, + templateAndRequestSettings, + combinedTemplateMappings ); + validateAdditionalSettings(provider, newAdditionalSettings, additionalIndexSettings); + additionalIndexSettings.put(newAdditionalSettings); + if (provider.overrulesTemplateAndRequestSettings()) { + overrulingSettings.addAll(newAdditionalSettings.keySet()); + } } - // For all the explicit settings, we go through the template and request level settings - // and see if either a template or the request has "cancelled out" an explicit default - // setting. For example, if a plugin had as an explicit setting: - // "index.mysetting": "blah - // And either a template or create index request had: - // "index.mysetting": null - // We want to remove the explicit setting not only from the explicitly set settings, but - // also from the template and request settings, so that from the newly create index's - // perspective it is as though the setting has not been set at all (using the default - // value). for (String explicitSetting : additionalIndexSettings.keys()) { - if (templateSettings.keys().contains(explicitSetting) && templateSettings.get(explicitSetting) == null) { - logger.debug( - "removing default [{}] setting as it in set to null in a template for [{}] creation", - explicitSetting, - request.index() - ); - additionalIndexSettings.remove(explicitSetting); + if (overrulingSettings.contains(explicitSetting)) { + // Remove any conflicting template and request settings to use the provided values. templateSettings.remove(explicitSetting); - } - if (requestSettings.keys().contains(explicitSetting) && requestSettings.get(explicitSetting) == null) { - logger.debug( - "removing default [{}] setting as it in set to null in the request for [{}] creation", - explicitSetting, - request.index() - ); - additionalIndexSettings.remove(explicitSetting); requestSettings.remove(explicitSetting); + } else { + // For all the explicit settings, we go through the template and request level settings + // and see if either a template or the request has "cancelled out" an explicit default + // setting. For example, if a plugin had as an explicit setting: + // "index.mysetting": "blah + // And either a template or create index request had: + // "index.mysetting": null + // We want to remove the explicit setting not only from the explicitly set settings, but + // also from the template and request settings, so that from the newly create index's + // perspective it is as though the setting has not been set at all (using the default + // value). + if (templateSettings.keys().contains(explicitSetting) && templateSettings.get(explicitSetting) == null) { + logger.debug( + "removing default [{}] setting as it is set to null in a template for [{}] creation", + explicitSetting, + request.index() + ); + additionalIndexSettings.remove(explicitSetting); + templateSettings.remove(explicitSetting); + } + if (requestSettings.keys().contains(explicitSetting) && requestSettings.get(explicitSetting) == null) { + logger.debug( + "removing default [{}] setting as it is set to null in the request for [{}] creation", + explicitSetting, + request.index() + ); + additionalIndexSettings.remove(explicitSetting); + requestSettings.remove(explicitSetting); + } } } // Finally, we actually add the explicit defaults prior to the template settings and the // request settings, so that the precedence goes: - // Explicit Defaults -> Template -> Request -> Necessary Settings (# of shards, uuid, etc) + // Explicit Defaults -> Template -> Request -> Filter out failure store settings -> Necessary Settings (# of shards, uuid, etc) indexSettingsBuilder.put(additionalIndexSettings.build()); indexSettingsBuilder.put(templateSettings.build()); } - + if (request.isFailureIndex()) { + DataStreamFailureStoreDefinition.filterUserDefinedSettings(indexSettingsBuilder); + } // now, put the request settings, so they override templates indexSettingsBuilder.put(requestSettings.build()); @@ -1093,6 +1131,29 @@ static Settings aggregateIndexSettings( return indexSettings; } + /** + * Validates whether additional settings don't have keys that are already defined in all additional settings. + * + * @param provider The {@link IndexSettingProvider} that produced additionalSettings + * @param additionalSettings The settings produced by the specified provider + * @param allAdditionalSettings A settings builder containing all additional settings produced by any {@link IndexSettingProvider} + * that already executed + * @throws IllegalArgumentException If keys in additionalSettings are already defined in allAdditionalSettings + */ + public static void validateAdditionalSettings( + IndexSettingProvider provider, + Settings additionalSettings, + Settings.Builder allAdditionalSettings + ) throws IllegalArgumentException { + for (String settingName : additionalSettings.keySet()) { + if (allAdditionalSettings.keys().contains(settingName)) { + var name = provider.getClass().getSimpleName(); + var message = Strings.format("additional index setting [%s] added by [%s] is already present", settingName, name); + throw new IllegalArgumentException(message); + } + } + } + private static void validateSoftDeleteSettings(Settings indexSettings) { if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(indexSettings) == false && IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(indexSettings).onOrAfter(IndexVersions.V_8_0_0)) { @@ -1265,7 +1326,7 @@ static IndexMetadata buildIndexMetadata( ) { IndexMetadata.Builder indexMetadataBuilder = createIndexMetadataBuilder(indexName, sourceMetadata, indexSettings, routingNumShards); indexMetadataBuilder.system(isSystem); - if (minClusterTransportVersion.before(TransportVersions.EVENT_INGESTED_RANGE_IN_CLUSTER_STATE)) { + if (minClusterTransportVersion.before(TransportVersions.V_8_15_0)) { // promote to UNKNOWN for older versions since they don't know how to handle event.ingested in cluster state indexMetadataBuilder.eventIngestedRange(IndexLongFieldRange.UNKNOWN, minClusterTransportVersion); } @@ -1332,7 +1393,7 @@ private static void updateIndexMappingsAndBuildSortOrder( MapperService mapperService = indexService.mapperService(); IndexMode indexMode = indexService.getIndexSettings() != null ? indexService.getIndexSettings().getMode() : IndexMode.STANDARD; List allMappings = new ArrayList<>(); - final CompressedXContent defaultMapping = indexMode.getDefaultMapping(); + final CompressedXContent defaultMapping = indexMode.getDefaultMapping(indexService.getIndexSettings()); if (defaultMapping != null) { allMappings.add(defaultMapping); } @@ -1508,6 +1569,16 @@ static void validateCloneIndex( IndexMetadata.selectCloneShard(0, sourceMetadata, INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings)); } + private static final Set UNMODIFIABLE_SETTINGS_DURING_RESIZE = Set.of( + IndexSettings.MODE.getKey(), + SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), + IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), + IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), + IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), + IndexSortConfig.INDEX_SORT_MODE_SETTING.getKey(), + IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey() + ); + static IndexMetadata validateResize( Metadata metadata, ClusterBlocks clusterBlocks, @@ -1545,6 +1616,11 @@ static IndexMetadata validateResize( // of if the source shards are divisible by the number of target shards IndexMetadata.getRoutingFactor(sourceMetadata.getNumberOfShards(), INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings)); } + for (String setting : UNMODIFIABLE_SETTINGS_DURING_RESIZE) { + if (targetIndexSettings.hasValue(setting)) { + throw new IllegalArgumentException("can't change setting [" + setting + "] during resize"); + } + } return sourceMetadata; } @@ -1580,23 +1656,11 @@ static void prepareResizeIndexSettings( throw new IllegalStateException("unknown resize type is " + type); } - final Settings.Builder builder = Settings.builder(); + final Settings.Builder builder; if (copySettings) { - // copy all settings and non-copyable settings and settings that have already been set (e.g., from the request) - for (final String key : sourceMetadata.getSettings().keySet()) { - final Setting setting = indexScopedSettings.get(key); - if (setting == null) { - assert indexScopedSettings.isPrivateSetting(key) : key; - } else if (setting.getProperties().contains(Setting.Property.NotCopyableOnResize)) { - continue; - } - // do not override settings that have already been set (for example, from the request) - if (indexSettingsBuilder.keys().contains(key)) { - continue; - } - builder.copy(key, sourceMetadata.getSettings()); - } + builder = copySettingsFromSource(true, sourceMetadata.getSettings(), indexScopedSettings, indexSettingsBuilder); } else { + builder = Settings.builder(); final Predicate sourceSettingsPredicate = (s) -> (s.startsWith("index.similarity.") || s.startsWith("index.analysis.") || s.startsWith("index.sort.") @@ -1614,6 +1678,36 @@ static void prepareResizeIndexSettings( } } + public static Settings.Builder copySettingsFromSource( + boolean copyPrivateSettings, + Settings sourceSettings, + IndexScopedSettings indexScopedSettings, + Settings.Builder indexSettingsBuilder + ) { + final Settings.Builder builder = Settings.builder(); + for (final String key : sourceSettings.keySet()) { + final Setting setting = indexScopedSettings.get(key); + if (setting == null) { + assert indexScopedSettings.isPrivateSetting(key) : key; + if (copyPrivateSettings == false) { + continue; + } + } else if (setting.getProperties().contains(Setting.Property.NotCopyableOnResize)) { + continue; + } else if (setting.isPrivateIndex()) { + if (copyPrivateSettings == false) { + continue; + } + } + // do not override settings that have already been set (for example, from the request) + if (indexSettingsBuilder.keys().contains(key)) { + continue; + } + builder.copy(key, sourceSettings); + } + return builder; + } + /** * Returns a default number of routing shards based on the number of shards of the index. The default number of routing shards will * allow any index to be split at least once and at most 10 times by a factor of two. The closer the number or shards gets to 1024 diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java index 9e8a99351d84a..db3973c1a15a8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; @@ -44,6 +45,7 @@ public class MetadataDataStreamsService { private final DataStreamGlobalRetentionSettings globalRetentionSettings; private final MasterServiceTaskQueue updateLifecycleTaskQueue; private final MasterServiceTaskQueue setRolloverOnWriteTaskQueue; + private final MasterServiceTaskQueue updateOptionsTaskQueue; public MetadataDataStreamsService( ClusterService clusterService, @@ -92,6 +94,20 @@ public Tuple executeTask( Priority.NORMAL, rolloverOnWriteExecutor ); + ClusterStateTaskExecutor updateOptionsExecutor = new SimpleBatchedAckListenerTaskExecutor<>() { + + @Override + public Tuple executeTask( + UpdateOptionsTask modifyOptionsTask, + ClusterState clusterState + ) { + return new Tuple<>( + updateDataStreamOptions(clusterState, modifyOptionsTask.getDataStreamNames(), modifyOptionsTask.getOptions()), + modifyOptionsTask + ); + } + }; + this.updateOptionsTaskQueue = clusterService.createTaskQueue("modify-data-stream-options", Priority.NORMAL, updateOptionsExecutor); } public void modifyDataStream(final ModifyDataStreamsAction.Request request, final ActionListener listener) { @@ -146,6 +162,39 @@ public void removeLifecycle( ); } + /** + * Submits the task to set the provided data stream options to the requested data streams. + */ + public void setDataStreamOptions( + final List dataStreamNames, + DataStreamOptions options, + TimeValue ackTimeout, + TimeValue masterTimeout, + final ActionListener listener + ) { + updateOptionsTaskQueue.submitTask( + "set-data-stream-options", + new UpdateOptionsTask(dataStreamNames, options, ackTimeout, listener), + masterTimeout + ); + } + + /** + * Submits the task to remove the data stream options from the requested data streams. + */ + public void removeDataStreamOptions( + List dataStreamNames, + TimeValue ackTimeout, + TimeValue masterTimeout, + ActionListener listener + ) { + updateOptionsTaskQueue.submitTask( + "delete-data-stream-options", + new UpdateOptionsTask(dataStreamNames, null, ackTimeout, listener), + masterTimeout + ); + } + @SuppressForbidden(reason = "legacy usage of unbatched task") // TODO add support for batching here private void submitUnbatchedTask(@SuppressWarnings("SameParameterValue") String source, ClusterStateUpdateTask task) { clusterService.submitUnbatchedStateUpdateTask(source, task); @@ -227,6 +276,24 @@ ClusterState updateDataLifecycle(ClusterState currentState, List dataStr return ClusterState.builder(currentState).metadata(builder.build()).build(); } + /** + * Creates an updated cluster state in which the requested data streams have the data stream options provided. + * Visible for testing. + */ + ClusterState updateDataStreamOptions( + ClusterState currentState, + List dataStreamNames, + @Nullable DataStreamOptions dataStreamOptions + ) { + Metadata metadata = currentState.metadata(); + Metadata.Builder builder = Metadata.builder(metadata); + for (var dataStreamName : dataStreamNames) { + var dataStream = validateDataStream(metadata, dataStreamName); + builder.put(dataStream.copy().setDataStreamOptions(dataStreamOptions).build()); + } + return ClusterState.builder(currentState).metadata(builder.build()).build(); + } + /** * Creates an updated cluster state in which the requested data stream has the flag {@link DataStream#rolloverOnWrite()} * set to the value of the parameter rolloverOnWrite @@ -371,6 +438,34 @@ public DataStreamLifecycle getDataLifecycle() { } } + /** + * A cluster state update task that consists of the cluster state request and the listeners that need to be notified upon completion. + */ + static class UpdateOptionsTask extends AckedBatchedClusterStateUpdateTask { + + private final List dataStreamNames; + private final DataStreamOptions options; + + UpdateOptionsTask( + List dataStreamNames, + @Nullable DataStreamOptions options, + TimeValue ackTimeout, + ActionListener listener + ) { + super(ackTimeout, listener); + this.dataStreamNames = dataStreamNames; + this.options = options; + } + + public List getDataStreamNames() { + return dataStreamNames; + } + + public DataStreamOptions getOptions() { + return options; + } + } + /** * A cluster state update task that consists of the cluster state request and the listeners that need to be notified upon completion. */ diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexService.java index a87912f3ffc8d..5d1a037d6bc3e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexService.java @@ -1,22 +1,26 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexClusterStateUpdateRequest; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateAckListener; import org.elasticsearch.cluster.ClusterStateTaskExecutor; +import org.elasticsearch.cluster.ClusterStateTaskListener; import org.elasticsearch.cluster.RestoreInProgress; import org.elasticsearch.cluster.SimpleBatchedAckListenerTaskExecutor; import org.elasticsearch.cluster.block.ClusterBlocks; +import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; @@ -24,7 +28,7 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; import org.elasticsearch.injection.guice.Inject; @@ -32,10 +36,10 @@ import org.elasticsearch.snapshots.SnapshotInProgressException; import org.elasticsearch.snapshots.SnapshotsService; -import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Objects; import java.util.Set; import static org.elasticsearch.cluster.routing.allocation.allocator.AllocationActionListener.rerouteCompletionIsNotRequired; @@ -47,22 +51,19 @@ public class MetadataDeleteIndexService { private static final Logger logger = LogManager.getLogger(MetadataDeleteIndexService.class); - private final Settings settings; - // package private for tests - final ClusterStateTaskExecutor executor; - private final MasterServiceTaskQueue taskQueue; + final ClusterStateTaskExecutor executor; + private final MasterServiceTaskQueue taskQueue; @Inject public MetadataDeleteIndexService(Settings settings, ClusterService clusterService, AllocationService allocationService) { - this.settings = settings; executor = new SimpleBatchedAckListenerTaskExecutor<>() { @Override public Tuple executeTask( - DeleteIndexClusterStateUpdateRequest task, + DeleteIndicesClusterStateUpdateTask task, ClusterState clusterState ) { - return Tuple.tuple(MetadataDeleteIndexService.deleteIndices(clusterState, Sets.newHashSet(task.indices()), settings), task); + return Tuple.tuple(MetadataDeleteIndexService.deleteIndices(clusterState, task.indices, settings), task); } @Override @@ -80,11 +81,64 @@ public ClusterState afterBatchExecution(ClusterState clusterState, boolean clust taskQueue = clusterService.createTaskQueue("delete-index", Priority.URGENT, executor); } - public void deleteIndices(final DeleteIndexClusterStateUpdateRequest request) { - if (request.indices() == null || request.indices().length == 0) { - throw new IllegalArgumentException("Index name is required"); + public void deleteIndices( + TimeValue masterNodeTimeout, + TimeValue ackTimeout, + Set indices, + ActionListener listener + ) { + if (indices == null || indices.isEmpty()) { + throw new IllegalArgumentException("Indices are required"); + } + taskQueue.submitTask( + "delete-index " + indices, + new DeleteIndicesClusterStateUpdateTask(indices, ackTimeout, listener), + masterNodeTimeout + ); + } + + // package private for tests + static class DeleteIndicesClusterStateUpdateTask implements ClusterStateTaskListener, ClusterStateAckListener { + + private final Set indices; + private final TimeValue ackTimeout; + private final ActionListener listener; + + DeleteIndicesClusterStateUpdateTask(Set indices, TimeValue ackTimeout, ActionListener listener) { + this.indices = Objects.requireNonNull(indices); + this.ackTimeout = Objects.requireNonNull(ackTimeout); + this.listener = Objects.requireNonNull(listener); + } + + @Override + public boolean mustAck(DiscoveryNode discoveryNode) { + return true; + } + + @Override + public void onAllNodesAcked() { + listener.onResponse(AcknowledgedResponse.TRUE); + } + + @Override + public void onAckFailure(Exception e) { + listener.onResponse(AcknowledgedResponse.FALSE); + } + + @Override + public void onAckTimeout() { + listener.onResponse(AcknowledgedResponse.FALSE); + } + + @Override + public TimeValue ackTimeout() { + return ackTimeout; + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); } - taskQueue.submitTask("delete-index " + Arrays.toString(request.indices()), request, request.masterNodeTimeout()); } /** diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataFeatures.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataFeatures.java index 6b09b467a017c..89f776a7ada0f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataFeatures.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataFeatures.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java index 41ef8ed6aa470..f926e2b6ebf35 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; @@ -281,7 +282,7 @@ public boolean mustAck(DiscoveryNode discoveryNode) { @Override public void onAllNodesAcked() { - listener.onResponse(IndicesAliasesResponse.build(request.getActionResults())); + listener.onResponse(IndicesAliasesResponse.build(request.actionResults())); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java index 272c107883043..95d1c37ec41ae 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; @@ -469,7 +470,7 @@ public void addIndexBlock(AddIndexBlockClusterStateUpdateRequest request, Action } addBlocksQueue.submitTask( - "add-index-block-[" + request.getBlock().name + "]-" + Arrays.toString(concreteIndices), + "add-index-block-[" + request.block().name + "]-" + Arrays.toString(concreteIndices), new AddBlocksTask(request, listener), request.masterNodeTimeout() ); @@ -479,7 +480,7 @@ private class AddBlocksExecutor extends SimpleBatchedExecutor> executeTask(AddBlocksTask task, ClusterState clusterState) { - return addIndexBlock(task.request.indices(), clusterState, task.request.getBlock()); + return addIndexBlock(task.request.indices(), clusterState, task.request.block()); } @Override @@ -496,7 +497,7 @@ public void taskSucceeded(AddBlocksTask task, Map blockedIn .delegateFailure( (delegate2, verifyResults) -> finalizeBlocksQueue.submitTask( "finalize-index-block-[" - + task.request.getBlock().name + + task.request.block().name + "]-[" + blockedIndices.keySet().stream().map(Index::getName).collect(Collectors.joining(", ")) + "]", @@ -528,7 +529,7 @@ public Tuple> executeTask(FinalizeBlocksTask clusterState, task.blockedIndices, task.verifyResults, - task.request.getBlock() + task.request.block() ); assert finalizeResult.v2().size() == task.verifyResults.size(); return finalizeResult; @@ -796,9 +797,7 @@ private void sendVerifyShardBlockRequest( block, parentTaskId ); - if (request.ackTimeout() != null) { - shardRequest.timeout(request.ackTimeout()); - } + shardRequest.timeout(request.ackTimeout()); client.executeLocally(TransportVerifyShardIndexBlockAction.TYPE, shardRequest, listener); } } @@ -1121,6 +1120,7 @@ private ClusterState openIndices(final Index[] indices, final ClusterState curre final Metadata.Builder metadata = Metadata.builder(currentState.metadata()); final ClusterBlocks.Builder blocks = ClusterBlocks.builder(currentState.blocks()); final IndexVersion minIndexCompatibilityVersion = currentState.getNodes().getMinSupportedIndexVersion(); + final IndexVersion minReadOnlyIndexCompatibilityVersion = currentState.getNodes().getMinReadOnlySupportedIndexVersion(); for (IndexMetadata indexMetadata : indicesToOpen) { final Index index = indexMetadata.getIndex(); @@ -1138,7 +1138,11 @@ private ClusterState openIndices(final Index[] indices, final ClusterState curre // The index might be closed because we couldn't import it due to an old incompatible // version, so we need to verify its compatibility. - newIndexMetadata = indexMetadataVerifier.verifyIndexMetadata(newIndexMetadata, minIndexCompatibilityVersion); + newIndexMetadata = indexMetadataVerifier.verifyIndexMetadata( + newIndexMetadata, + minIndexCompatibilityVersion, + minReadOnlyIndexCompatibilityVersion + ); try { indicesService.verifyIndexMetadata(newIndexMetadata, newIndexMetadata); } catch (Exception e) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index ff23f50ef7afe..7f8b87d2d3f48 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; @@ -311,12 +312,7 @@ public ClusterState addComponentTemplate( } } - final Template finalTemplate = new Template( - finalSettings, - wrappedMappings, - template.template().aliases(), - template.template().lifecycle() - ); + final Template finalTemplate = Template.builder(template.template()).settings(finalSettings).mappings(wrappedMappings).build(); final ComponentTemplate finalComponentTemplate = new ComponentTemplate( finalTemplate, template.version(), @@ -347,6 +343,7 @@ public ClusterState addComponentTemplate( composableTemplate, globalRetentionSettings.get() ); + validateDataStreamOptions(tempStateWithComponentTemplateAdded.metadata(), composableTemplateName, composableTemplate); validateIndexTemplateV2(composableTemplateName, composableTemplate, tempStateWithComponentTemplateAdded); } catch (Exception e) { if (validationFailure == null) { @@ -628,7 +625,7 @@ public ClusterState addIndexTemplateV2( // adjusted (to add _doc) and it should be validated CompressedXContent mappings = innerTemplate.mappings(); CompressedXContent wrappedMappings = wrapMappingsIfNecessary(mappings, xContentRegistry); - final Template finalTemplate = new Template(finalSettings, wrappedMappings, innerTemplate.aliases(), innerTemplate.lifecycle()); + final Template finalTemplate = Template.builder(innerTemplate).settings(finalSettings).mappings(wrappedMappings).build(); finalIndexTemplate = template.toBuilder().template(finalTemplate).build(); } @@ -689,49 +686,44 @@ public static Map> v2TemplateOverlaps( return overlaps; } - private void validateIndexTemplateV2(String name, ComposableIndexTemplate indexTemplate, ClusterState currentState) { + // Visibility for testing + void validateIndexTemplateV2(String name, ComposableIndexTemplate indexTemplate, ClusterState currentState) { // Workaround for the fact that start_time and end_time are injected by the MetadataCreateDataStreamService upon creation, // but when validating templates that create data streams the MetadataCreateDataStreamService isn't used. - var finalTemplate = Optional.ofNullable(indexTemplate.template()); - var finalSettings = Settings.builder(); + var finalTemplate = indexTemplate.template(); final var now = Instant.now(); final var metadata = currentState.getMetadata(); final var combinedMappings = collectMappings(indexTemplate, metadata.componentTemplates(), "tmp_idx"); final var combinedSettings = resolveSettings(indexTemplate, metadata.componentTemplates()); // First apply settings sourced from index setting providers: + var finalSettings = Settings.builder(); for (var provider : indexSettingProviders) { - finalSettings.put( - provider.getAdditionalIndexSettings( - "validate-index-name", - indexTemplate.getDataStreamTemplate() != null ? "validate-data-stream-name" : null, - indexTemplate.getDataStreamTemplate() != null && metadata.isTimeSeriesTemplate(indexTemplate), - currentState.getMetadata(), - now, - combinedSettings, - combinedMappings - ) + var newAdditionalSettings = provider.getAdditionalIndexSettings( + "validate-index-name", + indexTemplate.getDataStreamTemplate() != null ? "validate-data-stream-name" : null, + metadata.retrieveIndexModeFromTemplate(indexTemplate), + currentState.getMetadata(), + now, + combinedSettings, + combinedMappings ); + MetadataCreateIndexService.validateAdditionalSettings(provider, newAdditionalSettings, finalSettings); + finalSettings.put(newAdditionalSettings); } // Then apply setting from component templates: finalSettings.put(combinedSettings); // Then finally apply settings resolved from index template: - finalSettings.put(finalTemplate.map(Template::settings).orElse(Settings.EMPTY)); - - var templateToValidate = indexTemplate.toBuilder() - .template( - new Template( - finalSettings.build(), - finalTemplate.map(Template::mappings).orElse(null), - finalTemplate.map(Template::aliases).orElse(null), - finalTemplate.map(Template::lifecycle).orElse(null) - ) - ) - .build(); + if (finalTemplate != null && finalTemplate.settings() != null) { + finalSettings.put(finalTemplate.settings()); + } + + var templateToValidate = indexTemplate.toBuilder().template(Template.builder(finalTemplate).settings(finalSettings)).build(); validate(name, templateToValidate); validateDataStreamsStillReferenced(currentState, name, templateToValidate); validateLifecycle(currentState.metadata(), name, templateToValidate, globalRetentionSettings.get()); + validateDataStreamOptions(currentState.metadata(), name, templateToValidate); if (templateToValidate.isDeprecated() == false) { validateUseOfDeprecatedComponentTemplates(name, templateToValidate, currentState.metadata().componentTemplates()); @@ -788,7 +780,7 @@ private void validateUseOfDeprecatedIngestPipelines(String name, IngestMetadata private void emitWarningIfPipelineIsDeprecated(String name, Map pipelines, String pipelineName) { Optional.ofNullable(pipelineName) .map(pipelines::get) - .filter(p -> Boolean.TRUE.equals(p.getConfigAsMap().get("deprecated"))) + .filter(p -> Boolean.TRUE.equals(p.getConfig().get("deprecated"))) .ifPresent( p -> deprecationLogger.warn( DeprecationCategory.TEMPLATES, @@ -825,6 +817,20 @@ static void validateLifecycle( } } + // Visible for testing + static void validateDataStreamOptions(Metadata metadata, String indexTemplateName, ComposableIndexTemplate template) { + ResettableValue dataStreamOptions = resolveDataStreamOptions(template, metadata.componentTemplates()); + if (dataStreamOptions.get() != null) { + if (template.getDataStreamTemplate() == null) { + throw new IllegalArgumentException( + "index template [" + + indexTemplateName + + "] specifies data stream options that can only be used in combination with a data stream" + ); + } + } + } + /** * Validate that by changing or adding {@code newTemplate}, there are * no unreferenced data streams. Note that this scenario is still possible @@ -1206,6 +1212,42 @@ static ClusterState innerPutTemplate( return ClusterState.builder(currentState).metadata(builder).build(); } + /** + * A private, local alternative to elements.stream().anyMatch(predicate) for micro-optimization reasons. + */ + private static boolean anyMatch(final List elements, final Predicate predicate) { + for (T e : elements) { + if (predicate.test(e)) { + return true; + } + } + return false; + } + + /** + * A private, local alternative to elements.stream().noneMatch(predicate) for micro-optimization reasons. + */ + private static boolean noneMatch(final List elements, final Predicate predicate) { + for (T e : elements) { + if (predicate.test(e)) { + return false; + } + } + return true; + } + + /** + * A private, local alternative to elements.stream().filter(predicate).findFirst() for micro-optimization reasons. + */ + private static Optional findFirst(final List elements, final Predicate predicate) { + for (T e : elements) { + if (predicate.test(e)) { + return Optional.of(e); + } + } + return Optional.empty(); + } + /** * Finds index templates whose index pattern matched with the given index name. In the case of * hidden indices, a template with a match all pattern or global template will not be returned. @@ -1225,15 +1267,14 @@ public static List findV1Templates(Metadata metadata, Str final List matchedTemplates = new ArrayList<>(); for (IndexTemplateMetadata template : metadata.templates().values()) { if (isHidden == null || isHidden == Boolean.FALSE) { - final boolean matched = template.patterns().stream().anyMatch(patternMatchPredicate); - if (matched) { + if (anyMatch(template.patterns(), patternMatchPredicate)) { matchedTemplates.add(template); } } else { assert isHidden == Boolean.TRUE; - final boolean isNotMatchAllTemplate = template.patterns().stream().noneMatch(Regex::isMatchAllPattern); + final boolean isNotMatchAllTemplate = noneMatch(template.patterns(), Regex::isMatchAllPattern); if (isNotMatchAllTemplate) { - if (template.patterns().stream().anyMatch(patternMatchPredicate)) { + if (anyMatch(template.patterns(), patternMatchPredicate)) { matchedTemplates.add(template); } } @@ -1244,19 +1285,21 @@ public static List findV1Templates(Metadata metadata, Str // this is complex but if the index is not hidden in the create request but is hidden as the result of template application, // then we need to exclude global templates if (isHidden == null) { - final Optional templateWithHiddenSetting = matchedTemplates.stream() - .filter(template -> IndexMetadata.INDEX_HIDDEN_SETTING.exists(template.settings())) - .findFirst(); + final Optional templateWithHiddenSetting = findFirst( + matchedTemplates, + template -> IndexMetadata.INDEX_HIDDEN_SETTING.exists(template.settings()) + ); if (templateWithHiddenSetting.isPresent()) { final boolean templatedIsHidden = IndexMetadata.INDEX_HIDDEN_SETTING.get(templateWithHiddenSetting.get().settings()); if (templatedIsHidden) { // remove the global templates - matchedTemplates.removeIf(current -> current.patterns().stream().anyMatch(Regex::isMatchAllPattern)); + matchedTemplates.removeIf(current -> anyMatch(current.patterns(), Regex::isMatchAllPattern)); } // validate that hidden didn't change - final Optional templateWithHiddenSettingPostRemoval = matchedTemplates.stream() - .filter(template -> IndexMetadata.INDEX_HIDDEN_SETTING.exists(template.settings())) - .findFirst(); + final Optional templateWithHiddenSettingPostRemoval = findFirst( + matchedTemplates, + template -> IndexMetadata.INDEX_HIDDEN_SETTING.exists(template.settings()) + ); if (templateWithHiddenSettingPostRemoval.isEmpty() || templateWithHiddenSetting.get() != templateWithHiddenSettingPostRemoval.get()) { throw new IllegalStateException( @@ -1319,14 +1362,13 @@ static List> findV2CandidateTemplates(Met * built with a template that none of its indices match. */ if (isHidden == false || template.getDataStreamTemplate() != null) { - final boolean matched = template.indexPatterns().stream().anyMatch(patternMatchPredicate); - if (matched) { + if (anyMatch(template.indexPatterns(), patternMatchPredicate)) { candidates.add(Tuple.tuple(name, template)); } } else { - final boolean isNotMatchAllTemplate = template.indexPatterns().stream().noneMatch(Regex::isMatchAllPattern); + final boolean isNotMatchAllTemplate = noneMatch(template.indexPatterns(), Regex::isMatchAllPattern); if (isNotMatchAllTemplate) { - if (template.indexPatterns().stream().anyMatch(patternMatchPredicate)) { + if (anyMatch(template.indexPatterns(), patternMatchPredicate)) { candidates.add(Tuple.tuple(name, template)); } } @@ -1340,7 +1382,7 @@ static List> findV2CandidateTemplates(Met // Checks if a global template specifies the `index.hidden` setting. This check is important because a global // template shouldn't specify the `index.hidden` setting, we leave it up to the caller to handle this situation. private static boolean isGlobalAndHasIndexHiddenSetting(Metadata metadata, ComposableIndexTemplate template, String templateName) { - return template.indexPatterns().stream().anyMatch(Regex::isMatchAllPattern) + return anyMatch(template.indexPatterns(), Regex::isMatchAllPattern) && IndexMetadata.INDEX_HIDDEN_SETTING.exists(resolveSettings(metadata, templateName)); } @@ -1531,7 +1573,7 @@ static List> resolveAliases( public static DataStreamLifecycle resolveLifecycle(final Metadata metadata, final String templateName) { final ComposableIndexTemplate template = metadata.templatesV2().get(templateName); assert template != null - : "attempted to resolve settings for a template [" + templateName + "] that did not exist in the cluster state"; + : "attempted to resolve lifecycle for a template [" + templateName + "] that did not exist in the cluster state"; if (template == null) { return null; } @@ -1623,6 +1665,81 @@ public static DataStreamLifecycle composeDataLifecycles(List} object + */ + public static ResettableValue resolveDataStreamOptions(final Metadata metadata, final String templateName) { + final ComposableIndexTemplate template = metadata.templatesV2().get(templateName); + assert template != null + : "attempted to resolve data stream options for a template [" + templateName + "] that did not exist in the cluster state"; + if (template == null) { + return ResettableValue.undefined(); + } + return resolveDataStreamOptions(template, metadata.componentTemplates()); + } + + /** + * Resolve the provided v2 template and component templates into a {@link ResettableValue} object + */ + public static ResettableValue resolveDataStreamOptions( + ComposableIndexTemplate template, + Map componentTemplates + ) { + Objects.requireNonNull(template, "attempted to resolve data stream for a null template"); + Objects.requireNonNull(componentTemplates, "attempted to resolve data stream options with null component templates"); + + List> dataStreamOptionsList = new ArrayList<>(); + for (String componentTemplateName : template.composedOf()) { + if (componentTemplates.containsKey(componentTemplateName) == false) { + continue; + } + ResettableValue dataStreamOptions = componentTemplates.get(componentTemplateName) + .template() + .resettableDataStreamOptions(); + if (dataStreamOptions.isDefined()) { + dataStreamOptionsList.add(dataStreamOptions); + } + } + // The actual index template's data stream options have the highest precedence. + if (template.template() != null && template.template().resettableDataStreamOptions().isDefined()) { + dataStreamOptionsList.add(template.template().resettableDataStreamOptions()); + } + return composeDataStreamOptions(dataStreamOptionsList); + } + + /** + * This method composes a series of data streams options to a final one. Since currently the data stream options + * contains only the failure store configuration which also contains only one field, the composition is a bit trivial. + * But we introduce the mechanics that will help extend it really easily. + * @param dataStreamOptionsList a sorted list of data stream options in the order that they will be composed + * @return the final data stream option configuration + */ + public static ResettableValue composeDataStreamOptions( + List> dataStreamOptionsList + ) { + if (dataStreamOptionsList.isEmpty()) { + return ResettableValue.undefined(); + } + DataStreamOptions.Template.Builder builder = null; + for (ResettableValue current : dataStreamOptionsList) { + if (current.isDefined() == false) { + continue; + } + if (current.shouldReset()) { + builder = null; + } else { + DataStreamOptions.Template currentTemplate = current.get(); + if (builder == null) { + builder = DataStreamOptions.Template.builder(currentTemplate); + } else { + // Currently failure store has only one field that needs to be defined so the composing of the failure store is trivial + builder.updateFailureStore(currentTemplate.failureStore()); + } + } + } + return builder == null ? ResettableValue.undefined() : ResettableValue.create(builder.build()); + } + /** * Given a state and a composable template, validate that the final composite template * generated by the composable template and all of its component templates contains valid diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java index 1c956c7f78f32..5d45bf1ce127e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java index 13c2fabd6b3df..39acc6d3f6311 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; @@ -17,7 +18,6 @@ import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; -import org.elasticsearch.cluster.ack.ClusterStateUpdateRequest; import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest; import org.elasticsearch.cluster.routing.allocation.allocator.AllocationActionListener; import org.elasticsearch.cluster.service.ClusterService; @@ -29,6 +29,7 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.DocumentMapper; @@ -39,6 +40,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; @@ -104,7 +106,12 @@ public void migrateToDataStream( var delegate = new AllocationActionListener<>(listener, threadContext); submitUnbatchedTask( "migrate-to-data-stream [" + request.aliasName + "]", - new AckedClusterStateUpdateTask(Priority.HIGH, request, delegate.clusterStateUpdate()) { + new AckedClusterStateUpdateTask( + Priority.HIGH, + request.masterNodeTimeout(), + request.ackTimeout(), + delegate.clusterStateUpdate() + ) { @Override public ClusterState execute(ClusterState currentState) throws Exception { @@ -224,7 +231,7 @@ static void prepareBackingIndex( Settings nodeSettings ) throws IOException { MappingMetadata mm = im.mapping(); - if (mm == null) { + if (mm == null || mm.equals(MappingMetadata.EMPTY_MAPPINGS)) { throw new IllegalArgumentException("backing index [" + im.getIndex().getName() + "] must have mappings for a timestamp field"); } @@ -244,9 +251,11 @@ static void prepareBackingIndex( DataStreamFailureStoreDefinition.applyFailureStoreSettings(nodeSettings, settingsUpdate); } - imb.settings(settingsUpdate.build()) - .settingsVersion(im.getSettingsVersion() + 1) - .mappingVersion(im.getMappingVersion() + 1) + Settings maybeUpdatedSettings = settingsUpdate.build(); + if (IndexSettings.same(im.getSettings(), maybeUpdatedSettings) == false) { + imb.settings(maybeUpdatedSettings).settingsVersion(im.getSettingsVersion() + 1); + } + imb.mappingVersion(im.getMappingVersion() + 1) .mappingsUpdatedVersion(IndexVersion.current()) .putMapping(new MappingMetadata(mapper)); b.put(imb); @@ -277,15 +286,11 @@ static void validateBackingIndices(ClusterState currentState, String dataStreamN } } - @SuppressWarnings("rawtypes") - public static final class MigrateToDataStreamClusterStateUpdateRequest extends ClusterStateUpdateRequest { - - private final String aliasName; - - public MigrateToDataStreamClusterStateUpdateRequest(String aliasName, TimeValue masterNodeTimeout, TimeValue timeout) { - this.aliasName = aliasName; - masterNodeTimeout(masterNodeTimeout); - ackTimeout(timeout); + public record MigrateToDataStreamClusterStateUpdateRequest(String aliasName, TimeValue masterNodeTimeout, TimeValue ackTimeout) { + public MigrateToDataStreamClusterStateUpdateRequest { + Objects.requireNonNull(aliasName); + Objects.requireNonNull(masterNodeTimeout); + Objects.requireNonNull(ackTimeout); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsService.java index 3272462dd3725..4fcbd4165423b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; @@ -175,7 +176,7 @@ ClusterState execute(ClusterState currentState) { } final Settings closedSettings = settingsForClosedIndices.build(); final Settings openSettings = settingsForOpenIndices.build(); - final boolean preserveExisting = request.isPreserveExisting(); + final boolean preserveExisting = request.onExisting() == UpdateSettingsClusterStateUpdateRequest.OnExisting.PRESERVE; RoutingTable.Builder routingTableBuilder = null; Metadata.Builder metadataBuilder = Metadata.builder(currentState.metadata()); @@ -198,7 +199,7 @@ ClusterState execute(ClusterState currentState) { } if (skippedSettings.isEmpty() == false && openIndices.isEmpty() == false) { - if (request.reopenShards()) { + if (request.onStaticSetting() == UpdateSettingsClusterStateUpdateRequest.OnStaticSetting.REOPEN_INDICES) { // We have non-dynamic settings and open indices. We will unassign all of the shards in these indices so that the new // changed settings are applied when the shards are re-assigned. routingTableBuilder = RoutingTable.builder( diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadata.java index 38eefa4085527..272f321e386fc 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadata.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; @@ -16,7 +17,7 @@ import org.elasticsearch.cluster.SimpleDiffable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.core.Nullable; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ParseField; @@ -189,8 +190,8 @@ public int hashCode() { } @Override - public Iterator toXContentChunked(ToXContent.Params ignored) { - return ChunkedToXContentHelper.xContentValuesMap(NODES_FIELD.getPreferredName(), nodes); + public Iterator toXContentChunked(ToXContent.Params params) { + return ChunkedToXContent.builder(params).xContentObjectFields(NODES_FIELD.getPreferredName(), nodes); } /** diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ProcessClusterEventTimeoutException.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ProcessClusterEventTimeoutException.java index 0d167e0b5cae5..1182faea81ed6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ProcessClusterEventTimeoutException.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ProcessClusterEventTimeoutException.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetadata.java index 9b07fbadb2328..7ffa85a7d42e6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetadata.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoryMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoryMetadata.java index 95bb9d1431f3d..0b9c359006b23 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoryMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoryMetadata.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; @@ -45,7 +46,11 @@ public class RepositoryMetadata implements Writeable { * @param settings repository settings */ public RepositoryMetadata(String name, String type, Settings settings) { - this(name, RepositoryData.MISSING_UUID, type, settings, RepositoryData.UNKNOWN_REPO_GEN, RepositoryData.EMPTY_REPO_GEN); + this(name, RepositoryData.MISSING_UUID, type, settings); + } + + public RepositoryMetadata(String name, String uuid, String type, Settings settings) { + this(name, uuid, type, settings, RepositoryData.UNKNOWN_REPO_GEN, RepositoryData.EMPTY_REPO_GEN); } public RepositoryMetadata(RepositoryMetadata metadata, long generation, long pendingGeneration) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ReservedStateErrorMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ReservedStateErrorMetadata.java index 92ff6b4a2c567..9ca86a6fa8459 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ReservedStateErrorMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ReservedStateErrorMetadata.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ReservedStateHandlerMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ReservedStateHandlerMetadata.java index b78a17c11dfb8..79ff6b79b0633 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ReservedStateHandlerMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ReservedStateHandlerMetadata.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ReservedStateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ReservedStateMetadata.java index 5df045df4ecd8..a0b35f7cfc3eb 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ReservedStateMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ReservedStateMetadata.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; @@ -90,6 +91,21 @@ public Set conflicts(String handlerName, Set modified) { return Collections.unmodifiableSet(intersect); } + /** + * Get the reserved keys for the handler name + * + * @param handlerName handler name to get keys for + * @return set of keys for that handler + */ + public Set keys(String handlerName) { + ReservedStateHandlerMetadata handlerMetadata = handlers.get(handlerName); + if (handlerMetadata == null || handlerMetadata.keys().isEmpty()) { + return Collections.emptySet(); + } + + return Collections.unmodifiableSet(handlerMetadata.keys()); + } + /** * Reads an {@link ReservedStateMetadata} from a {@link StreamInput} * diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ResettableValue.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ResettableValue.java new file mode 100644 index 0000000000000..4f38d2b8386a6 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ResettableValue.java @@ -0,0 +1,216 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.cluster.metadata; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; +import java.util.function.BiFunction; +import java.util.function.Function; + +/** + * This class holds a value of type @{param T} that can be in one of 3 states: + * - It has a concrete value, or + * - It is missing, or + * - It is meant to reset any other when it is composed with it. + * It is mainly used in template composition to capture the case when the user wished to reset any previous values. + * @param + */ +public class ResettableValue { + private static final ResettableValue RESET = new ResettableValue<>(true, null); + private static final ResettableValue UNDEFINED = new ResettableValue<>(false, null); + private static final String DISPLAY_RESET_VALUES = "display_reset"; + private static final Map HIDE_RESET_VALUES_PARAMS = Map.of(DISPLAY_RESET_VALUES, "false"); + + private final T value; + private final boolean isDefined; + + /** + * @return the reset state, meaning that this value is explicitly requested to be reset + */ + public static ResettableValue reset() { + @SuppressWarnings("unchecked") + ResettableValue t = (ResettableValue) RESET; + return t; + } + + /** + * @return the undefined state, meaning that this value has not been specified + */ + public static ResettableValue undefined() { + @SuppressWarnings("unchecked") + ResettableValue t = (ResettableValue) UNDEFINED; + return t; + } + + /** + * Wraps a value, if the value is null, it returns {@link #undefined()} + */ + public static ResettableValue create(T value) { + if (value == null) { + return undefined(); + } + return new ResettableValue<>(true, value); + } + + private ResettableValue(boolean isDefined, T value) { + this.isDefined = isDefined; + this.value = value; + } + + /** + * @return true if the state of this is reset + */ + public boolean shouldReset() { + return isDefined && value == null; + } + + /** + * @return true when the value is defined, either with a concrete value or reset. + */ + public boolean isDefined() { + return isDefined; + } + + /** + * @return the concrete value or null if it is in undefined or reset states. + */ + @Nullable + public T get() { + return value; + } + + /** + * Writes a single optional explicitly nullable value. This method is in direct relation with the + * {@link #read(StreamInput, Writeable.Reader)} which reads the respective value. It's the + * responsibility of the caller to preserve order of the fields and their backwards compatibility. + * + * @throws IOException + */ + static void write(StreamOutput out, ResettableValue value, Writeable.Writer writer) throws IOException { + out.writeBoolean(value.isDefined); + if (value.isDefined) { + out.writeBoolean(value.shouldReset()); + if (value.shouldReset() == false) { + writer.write(out, value.get()); + } + } + } + + /** + * Reads a single optional and explicitly nullable value. This method is in direct relation with the + * {@link #write(StreamOutput, ResettableValue, Writeable.Writer)} which writes the respective value. It's the + * responsibility of the caller to preserve order of the fields and their backwards compatibility. + * + * @throws IOException + */ + @Nullable + static ResettableValue read(StreamInput in, Writeable.Reader reader) throws IOException { + boolean isDefined = in.readBoolean(); + if (isDefined == false) { + return ResettableValue.undefined(); + } + boolean shouldReset = in.readBoolean(); + if (shouldReset) { + return ResettableValue.reset(); + } + T value = reader.read(in); + return ResettableValue.create(value); + } + + /** + * Gets the value and applies the function {@param f} when the value is not null. Slightly more efficient than + * this.map(f).get(). + */ + public U mapAndGet(Function f) { + if (isDefined() == false || shouldReset()) { + return null; + } else { + return f.apply(value); + } + } + + public ResettableValue map(Function mapper) { + Objects.requireNonNull(mapper); + if (isDefined == false) { + return ResettableValue.undefined(); + } + if (shouldReset()) { + return reset(); + } + return ResettableValue.create(mapper.apply(value)); + } + + /** + * Ιt merges the values of the ResettableValue's when they are defined using the provided mergeFunction. + */ + public static ResettableValue merge(ResettableValue initial, ResettableValue update, BiFunction mergeFunction) { + if (update.shouldReset()) { + return undefined(); + } + if (update.isDefined() == false) { + return initial; + } + if (initial.isDefined() == false || initial.shouldReset()) { + return update; + } + // Because we checked that's defined and not in reset state, we can directly apply the merge function. + return ResettableValue.create(mergeFunction.apply(initial.value, update.value)); + } + + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params, String field) throws IOException { + return toXContent(builder, params, field, Function.identity()); + } + + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params, String field, Function transformValue) + throws IOException { + if (isDefined) { + if (value != null) { + builder.field(field, transformValue.apply(value)); + } else if (ResettableValue.shouldDisplayResetValue(params)) { + builder.nullField(field); + } + } + return builder; + } + + public static boolean shouldDisplayResetValue(ToXContent.Params params) { + return params.paramAsBoolean(DISPLAY_RESET_VALUES, true); + } + + public static ToXContent.Params hideResetValues(ToXContent.Params params) { + return new ToXContent.DelegatingMapParams(HIDE_RESET_VALUES_PARAMS, params); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ResettableValue that = (ResettableValue) o; + return isDefined == that.isDefined && Objects.equals(value, that.value); + } + + @Override + public int hashCode() { + return Objects.hash(value, isDefined); + } + + @Override + public String toString() { + return "ResettableValue{" + "value=" + value + ", isDefined=" + isDefined + '}'; + } +} diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownPersistentTasksStatus.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownPersistentTasksStatus.java index e053e29e1c1eb..c8d0b153c60c2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownPersistentTasksStatus.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownPersistentTasksStatus.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownPluginsStatus.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownPluginsStatus.java index 71452745d51fc..8e796fa5fd1a5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownPluginsStatus.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownPluginsStatus.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownShardMigrationStatus.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownShardMigrationStatus.java index 508f8346a875d..a224b3ae82312 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownShardMigrationStatus.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownShardMigrationStatus.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/SingleNodeShutdownMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/SingleNodeShutdownMetadata.java index 3453b3b6d70ff..ebd7a7db13f82 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/SingleNodeShutdownMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/SingleNodeShutdownMetadata.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; @@ -27,6 +28,7 @@ import java.util.Locale; import java.util.Objects; +import static org.elasticsearch.TransportVersions.NODE_SHUTDOWN_EPHEMERAL_ID_ADDED; import static org.elasticsearch.core.Strings.format; /** @@ -39,6 +41,7 @@ public class SingleNodeShutdownMetadata implements SimpleDiffable new SingleNodeShutdownMetadata( (String) a[0], - Type.valueOf((String) a[1]), - (String) a[2], - (long) a[3], - (boolean) a[4], - (TimeValue) a[5], - (String) a[6], - (TimeValue) a[7] + (String) a[1], + Type.valueOf((String) a[2]), + (String) a[3], + (long) a[4], + (boolean) a[5], + (TimeValue) a[6], + (String) a[7], + (TimeValue) a[8] ) ); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), NODE_ID_FIELD); + PARSER.declareField( + ConstructingObjectParser.optionalConstructorArg(), + (p, c) -> p.textOrNull(), + NODE_EPHEMERAL_ID_FIELD, + ObjectParser.ValueType.STRING_OR_NULL + ); PARSER.declareString(ConstructingObjectParser.constructorArg(), TYPE_FIELD); PARSER.declareString(ConstructingObjectParser.constructorArg(), REASON_FIELD); PARSER.declareLong(ConstructingObjectParser.constructorArg(), STARTED_AT_MILLIS_FIELD); @@ -90,6 +100,8 @@ public static SingleNodeShutdownMetadata parse(XContentParser parser) { public static final TimeValue DEFAULT_RESTART_SHARD_ALLOCATION_DELAY = TimeValue.timeValueMinutes(5); private final String nodeId; + @Nullable + private final String nodeEphemeralId; private final Type type; private final String reason; private final long startedAtMillis; @@ -109,6 +121,7 @@ public static SingleNodeShutdownMetadata parse(XContentParser parser) { */ private SingleNodeShutdownMetadata( String nodeId, + @Nullable String nodeEphemeralId, Type type, String reason, long startedAtMillis, @@ -118,6 +131,7 @@ private SingleNodeShutdownMetadata( @Nullable TimeValue gracePeriod ) { this.nodeId = Objects.requireNonNull(nodeId, "node ID must not be null"); + this.nodeEphemeralId = nodeEphemeralId; this.type = Objects.requireNonNull(type, "shutdown type must not be null"); this.reason = Objects.requireNonNull(reason, "shutdown reason must not be null"); this.startedAtMillis = startedAtMillis; @@ -156,6 +170,11 @@ private SingleNodeShutdownMetadata( public SingleNodeShutdownMetadata(StreamInput in) throws IOException { this.nodeId = in.readString(); + if (in.getTransportVersion().onOrAfter(NODE_SHUTDOWN_EPHEMERAL_ID_ADDED)) { + this.nodeEphemeralId = in.readOptionalString(); + } else { + this.nodeEphemeralId = null; // empty when talking to old nodes, meaning the persistent node id is the only differentiator + } this.type = in.readEnum(Type.class); this.reason = in.readString(); this.startedAtMillis = in.readVLong(); @@ -180,6 +199,15 @@ public String getNodeId() { return nodeId; } + /** + * @return The ephemeral ID of the node this {@link SingleNodeShutdownMetadata} concerns, or + * {@code null} if the ephemeral id is unknown. + */ + @Nullable + public String getNodeEphemeralId() { + return nodeEphemeralId; + } + /** * @return The type of shutdown this is (shutdown vs. permanent). */ @@ -240,6 +268,9 @@ public TimeValue getGracePeriod() { @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(nodeId); + if (out.getTransportVersion().onOrAfter(NODE_SHUTDOWN_EPHEMERAL_ID_ADDED)) { + out.writeOptionalString(nodeEphemeralId); + } if ((out.getTransportVersion().before(REPLACE_SHUTDOWN_TYPE_ADDED_VERSION) && this.type == SingleNodeShutdownMetadata.Type.REPLACE) || (out.getTransportVersion().before(SIGTERM_ADDED_VERSION) && this.type == Type.SIGTERM)) { out.writeEnum(SingleNodeShutdownMetadata.Type.REMOVE); @@ -263,9 +294,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); { builder.field(NODE_ID_FIELD.getPreferredName(), nodeId); + builder.field(NODE_EPHEMERAL_ID_FIELD.getPreferredName(), nodeEphemeralId); builder.field(TYPE_FIELD.getPreferredName(), type); builder.field(REASON_FIELD.getPreferredName(), reason); - builder.timeField(STARTED_AT_MILLIS_FIELD.getPreferredName(), STARTED_AT_READABLE_FIELD, startedAtMillis); + builder.timestampFieldsFromUnixEpochMillis( + STARTED_AT_MILLIS_FIELD.getPreferredName(), + STARTED_AT_READABLE_FIELD, + startedAtMillis + ); builder.field(NODE_SEEN_FIELD.getPreferredName(), nodeSeen); if (allocationDelay != null) { builder.field(ALLOCATION_DELAY_FIELD.getPreferredName(), allocationDelay.getStringRep()); @@ -290,6 +326,7 @@ public boolean equals(Object o) { return getStartedAtMillis() == that.getStartedAtMillis() && getNodeSeen() == that.getNodeSeen() && getNodeId().equals(that.getNodeId()) + && Objects.equals(getNodeEphemeralId(), that.getNodeEphemeralId()) && getType() == that.getType() && getReason().equals(that.getReason()) && Objects.equals(getAllocationDelay(), that.getAllocationDelay()) @@ -301,6 +338,7 @@ && getReason().equals(that.getReason()) public int hashCode() { return Objects.hash( getNodeId(), + getNodeEphemeralId(), getType(), getReason(), getStartedAtMillis(), @@ -317,6 +355,8 @@ public String toString() { stringBuilder.append("{") .append("nodeId=[") .append(nodeId) + .append("], nodeEphemeralId=[") + .append(nodeEphemeralId) .append(']') .append(", type=[") .append(type) @@ -345,6 +385,7 @@ public static Builder builder(SingleNodeShutdownMetadata original) { return builder(); } return new Builder().setNodeId(original.getNodeId()) + .setNodeEphemeralId(original.getNodeEphemeralId()) .setType(original.getType()) .setReason(original.getReason()) .setStartedAtMillis(original.getStartedAtMillis()) @@ -354,6 +395,7 @@ public static Builder builder(SingleNodeShutdownMetadata original) { public static class Builder { private String nodeId; + private String nodeEphemeralId; private Type type; private String reason; private long startedAtMillis = -1; @@ -373,6 +415,15 @@ public Builder setNodeId(String nodeId) { return this; } + /** + * @param nodeEphemeralId The node ephemeral ID this metadata refers to. + * @return This builder. + */ + public Builder setNodeEphemeralId(String nodeEphemeralId) { + this.nodeEphemeralId = nodeEphemeralId; + return this; + } + /** * @param type The type of shutdown. * @return This builder. @@ -439,6 +490,7 @@ public SingleNodeShutdownMetadata build() { return new SingleNodeShutdownMetadata( nodeId, + nodeEphemeralId, type, reason, startedAtMillis, diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeService.java index 1cdfbd40e5214..228bb3b222a57 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeService.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java index 0a045261e07b8..7d354768ca987 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.cluster.metadata; @@ -46,12 +47,19 @@ public class Template implements SimpleDiffable